1//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for SI
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUIGroupLP.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPULaneMaskUtils.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUSelectionDAGInfo.h"
21#include "AMDGPUTargetMachine.h"
22#include "GCNSubtarget.h"
23#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
24#include "SIMachineFunctionInfo.h"
25#include "SIRegisterInfo.h"
26#include "llvm/ADT/APFloat.h"
27#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/FloatingPointMode.h"
29#include "llvm/ADT/Statistic.h"
30#include "llvm/Analysis/OptimizationRemarkEmitter.h"
31#include "llvm/Analysis/UniformityAnalysis.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/ByteProvider.h"
34#include "llvm/CodeGen/FunctionLoweringInfo.h"
35#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
36#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
37#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
38#include "llvm/CodeGen/MachineFrameInfo.h"
39#include "llvm/CodeGen/MachineFunction.h"
40#include "llvm/CodeGen/MachineLoopInfo.h"
41#include "llvm/CodeGen/PseudoSourceValueManager.h"
42#include "llvm/CodeGen/SDPatternMatch.h"
43#include "llvm/IR/DiagnosticInfo.h"
44#include "llvm/IR/IRBuilder.h"
45#include "llvm/IR/IntrinsicInst.h"
46#include "llvm/IR/IntrinsicsAMDGPU.h"
47#include "llvm/IR/IntrinsicsR600.h"
48#include "llvm/IR/MDBuilder.h"
49#include "llvm/Support/AMDGPUAddrSpace.h"
50#include "llvm/Support/CommandLine.h"
51#include "llvm/Support/KnownBits.h"
52#include "llvm/Support/ModRef.h"
53#include "llvm/TargetParser/AtomicScope.h"
54#include "llvm/Transforms/Utils/LowerAtomic.h"
55#include <optional>
56
57using namespace llvm;
58using namespace llvm::SDPatternMatch;
59
60#define DEBUG_TYPE "si-lower"
61
62STATISTIC(NumTailCalls, "Number of tail calls");
63
64static cl::opt<bool>
65 DisableLoopAlignment("amdgpu-disable-loop-alignment",
66 cl::desc("Do not align and prefetch loops"),
67 cl::init(Val: false));
68
69static cl::opt<bool> UseDivergentRegisterIndexing(
70 "amdgpu-use-divergent-register-indexing", cl::Hidden,
71 cl::desc("Use indirect register addressing for divergent indexes"),
72 cl::init(Val: false));
73
74static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF) {
75 return MF.getInfo<SIMachineFunctionInfo>()->getMode().getDenormalFPEnv();
76}
77
78static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
79 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
80 return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
81}
82
83static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF) {
84 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
85 return Info->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign();
86}
87
88static unsigned findFirstFreeSGPR(CCState &CCInfo) {
89 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
90 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
91 if (!CCInfo.isAllocated(Reg: AMDGPU::SGPR0 + Reg)) {
92 return AMDGPU::SGPR0 + Reg;
93 }
94 }
95 llvm_unreachable("Cannot allocate sgpr");
96}
97
98SITargetLowering::SITargetLowering(const TargetMachine &TM,
99 const GCNSubtarget &STI)
100 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI) {
101 addRegisterClass(VT: MVT::i1, RC: &AMDGPU::VReg_1RegClass);
102 addRegisterClass(VT: MVT::i64, RC: &AMDGPU::SReg_64RegClass);
103
104 addRegisterClass(VT: MVT::i32, RC: &AMDGPU::SReg_32RegClass);
105
106 const SIRegisterInfo *TRI = STI.getRegisterInfo();
107 const TargetRegisterClass *V32RegClass =
108 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 32);
109 addRegisterClass(VT: MVT::f32, RC: V32RegClass);
110
111 addRegisterClass(VT: MVT::v2i32, RC: &AMDGPU::SReg_64RegClass);
112
113 const TargetRegisterClass *V64RegClass =
114 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 64);
115
116 addRegisterClass(VT: MVT::f64, RC: V64RegClass);
117 addRegisterClass(VT: MVT::v2f32, RC: V64RegClass);
118 addRegisterClass(VT: MVT::Untyped, RC: V64RegClass);
119
120 addRegisterClass(VT: MVT::v3i32, RC: &AMDGPU::SGPR_96RegClass);
121 addRegisterClass(VT: MVT::v3f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 96));
122
123 addRegisterClass(VT: MVT::v2i64, RC: &AMDGPU::SGPR_128RegClass);
124 addRegisterClass(VT: MVT::v2f64, RC: &AMDGPU::SGPR_128RegClass);
125
126 addRegisterClass(VT: MVT::v4i32, RC: &AMDGPU::SGPR_128RegClass);
127 addRegisterClass(VT: MVT::v4f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 128));
128
129 addRegisterClass(VT: MVT::v5i32, RC: &AMDGPU::SGPR_160RegClass);
130 addRegisterClass(VT: MVT::v5f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 160));
131
132 addRegisterClass(VT: MVT::v6i32, RC: &AMDGPU::SGPR_192RegClass);
133 addRegisterClass(VT: MVT::v6f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
134
135 addRegisterClass(VT: MVT::v3i64, RC: &AMDGPU::SGPR_192RegClass);
136 addRegisterClass(VT: MVT::v3f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
137
138 addRegisterClass(VT: MVT::v7i32, RC: &AMDGPU::SGPR_224RegClass);
139 addRegisterClass(VT: MVT::v7f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 224));
140
141 addRegisterClass(VT: MVT::v8i32, RC: &AMDGPU::SGPR_256RegClass);
142 addRegisterClass(VT: MVT::v8f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
143
144 addRegisterClass(VT: MVT::v4i64, RC: &AMDGPU::SGPR_256RegClass);
145 addRegisterClass(VT: MVT::v4f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
146
147 addRegisterClass(VT: MVT::v9i32, RC: &AMDGPU::SGPR_288RegClass);
148 addRegisterClass(VT: MVT::v9f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 288));
149
150 addRegisterClass(VT: MVT::v10i32, RC: &AMDGPU::SGPR_320RegClass);
151 addRegisterClass(VT: MVT::v10f32,
152 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 320));
153
154 addRegisterClass(VT: MVT::v11i32, RC: &AMDGPU::SGPR_352RegClass);
155 addRegisterClass(VT: MVT::v11f32,
156 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 352));
157
158 addRegisterClass(VT: MVT::v12i32, RC: &AMDGPU::SGPR_384RegClass);
159 addRegisterClass(VT: MVT::v12f32,
160 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 384));
161
162 addRegisterClass(VT: MVT::v16i32, RC: &AMDGPU::SGPR_512RegClass);
163 addRegisterClass(VT: MVT::v16f32,
164 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
165
166 addRegisterClass(VT: MVT::v8i64, RC: &AMDGPU::SGPR_512RegClass);
167 addRegisterClass(VT: MVT::v8f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
168
169 addRegisterClass(VT: MVT::v16i64, RC: &AMDGPU::SGPR_1024RegClass);
170 addRegisterClass(VT: MVT::v16f64,
171 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
172
173 if (Subtarget->has16BitInsts()) {
174 if (Subtarget->useRealTrue16Insts()) {
175 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::VGPR_16RegClass);
176 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::VGPR_16RegClass);
177 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::VGPR_16RegClass);
178 } else {
179 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::SReg_32RegClass);
180 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::SReg_32RegClass);
181 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::SReg_32RegClass);
182 }
183
184 // Unless there are also VOP3P operations, not operations are really legal.
185 addRegisterClass(VT: MVT::v2i16, RC: &AMDGPU::SReg_32RegClass);
186 addRegisterClass(VT: MVT::v2f16, RC: &AMDGPU::SReg_32RegClass);
187 addRegisterClass(VT: MVT::v2bf16, RC: &AMDGPU::SReg_32RegClass);
188 addRegisterClass(VT: MVT::v4i16, RC: &AMDGPU::SReg_64RegClass);
189 addRegisterClass(VT: MVT::v4f16, RC: &AMDGPU::SReg_64RegClass);
190 addRegisterClass(VT: MVT::v4bf16, RC: &AMDGPU::SReg_64RegClass);
191 addRegisterClass(VT: MVT::v8i16, RC: &AMDGPU::SGPR_128RegClass);
192 addRegisterClass(VT: MVT::v8f16, RC: &AMDGPU::SGPR_128RegClass);
193 addRegisterClass(VT: MVT::v8bf16, RC: &AMDGPU::SGPR_128RegClass);
194 addRegisterClass(VT: MVT::v16i16, RC: &AMDGPU::SGPR_256RegClass);
195 addRegisterClass(VT: MVT::v16f16, RC: &AMDGPU::SGPR_256RegClass);
196 addRegisterClass(VT: MVT::v16bf16, RC: &AMDGPU::SGPR_256RegClass);
197 addRegisterClass(VT: MVT::v32i16, RC: &AMDGPU::SGPR_512RegClass);
198 addRegisterClass(VT: MVT::v32f16, RC: &AMDGPU::SGPR_512RegClass);
199 addRegisterClass(VT: MVT::v32bf16, RC: &AMDGPU::SGPR_512RegClass);
200 }
201
202 addRegisterClass(VT: MVT::v32i32, RC: &AMDGPU::VReg_1024RegClass);
203 addRegisterClass(VT: MVT::v32f32,
204 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
205
206 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
207
208 setMinFunctionAlignment(Align(4));
209 setPrefFunctionAlignment(Align(STI.getInstCacheLineSize()));
210
211 // The boolean content concept here is too inflexible. Compares only ever
212 // really produce a 1-bit result. Any copy/extend from these will turn into a
213 // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
214 // it's what most targets use.
215 setBooleanContents(ZeroOrOneBooleanContent);
216 setBooleanVectorContents(ZeroOrOneBooleanContent);
217
218 // We need to custom lower vector stores from local memory
219 setOperationAction(Ops: ISD::LOAD,
220 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
221 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
222 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
223 MVT::i1, MVT::v32i32},
224 Action: Custom);
225
226 setOperationAction(Ops: ISD::STORE,
227 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
228 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
229 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
230 MVT::i1, MVT::v32i32},
231 Action: Custom);
232
233 if (isTypeLegal(VT: MVT::bf16)) {
234 for (unsigned Opc :
235 {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
236 ISD::FREM, ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
237 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FSQRT, ISD::FCBRT,
238 ISD::FSIN, ISD::FCOS, ISD::FPOW, ISD::FPOWI,
239 ISD::FLDEXP, ISD::FFREXP, ISD::FLOG, ISD::FLOG2,
240 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10,
241 ISD::FCEIL, ISD::FTRUNC, ISD::FRINT, ISD::FNEARBYINT,
242 ISD::FROUND, ISD::FROUNDEVEN, ISD::FFLOOR, ISD::FCANONICALIZE,
243 ISD::SETCC}) {
244 setOperationAction(Op: Opc, VT: MVT::bf16, Action: Promote);
245 }
246
247 // Only targets with packed bf16 instructions, e.g. gfx13.
248 if (Subtarget->hasBF16PackedInsts()) {
249 // Don't use Expand for fsub - the DAG combiner will undo fadd+fneg back
250 // to fsub, causing a libcall (which doesn't exist for bf16). Instead,
251 // directly expand to widened v2bf16 operations.
252 setOperationAction(Op: ISD::FSUB, VT: MVT::bf16, Action: Custom);
253 setOperationAction(Ops: {ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}, VT: MVT::bf16,
254 Action: Promote);
255 // Promote scalar operations to a v2bf16 operation with an unused high
256 // lane.
257 for (unsigned Opc :
258 {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMAXNUM, ISD::FMINNUM,
259 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FCANONICALIZE})
260 AddPromotedToType(Opc, OrigVT: MVT::bf16, DestVT: MVT::v2bf16);
261 }
262
263 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Expand);
264
265 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Promote);
266 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::bf16, DestVT: MVT::i16);
267
268 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Legal);
269 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Legal);
270 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Legal);
271
272 // We only need to custom lower because we can't specify an action for bf16
273 // sources.
274 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
275 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
276 }
277
278 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
279 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i16, Action: Expand);
280 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Expand);
281 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i16, Action: Expand);
282 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i16, Action: Expand);
283 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i16, Action: Expand);
284 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
285 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Expand);
286 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i8, Action: Expand);
287 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i8, Action: Expand);
288 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i8, Action: Expand);
289 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
290 setTruncStoreAction(ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Expand);
291 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Expand);
292 setTruncStoreAction(ValVT: MVT::v16i16, MemVT: MVT::v16i8, Action: Expand);
293 setTruncStoreAction(ValVT: MVT::v32i16, MemVT: MVT::v32i8, Action: Expand);
294
295 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
296 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
297 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i8, Action: Expand);
298 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i8, Action: Expand);
299 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i16, Action: Expand);
300 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i32, Action: Expand);
301 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i32, Action: Expand);
302
303 setOperationAction(Ops: ISD::GlobalAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
304 setOperationAction(Ops: ISD::BlockAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
305 setOperationAction(Ops: ISD::ExternalSymbol, VTs: {MVT::i32, MVT::i64}, Action: Custom);
306
307 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Promote);
308 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Custom);
309 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Promote);
310 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::f64, DestVT: MVT::i64);
311
312 setOperationAction(Ops: ISD::FSQRT, VTs: {MVT::f32, MVT::f64}, Action: Custom);
313
314 setOperationAction(Ops: ISD::SELECT_CC,
315 VTs: {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Action: Expand);
316
317 setOperationAction(Op: ISD::SETCC, VT: MVT::i1, Action: Promote);
318 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::v2i1, MVT::v4i1}, Action: Expand);
319 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::i1, DestVT: MVT::i32);
320
321 setOperationAction(Ops: ISD::TRUNCATE,
322 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
323 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
324 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
325 Action: Expand);
326 setOperationAction(Ops: ISD::FP_ROUND,
327 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
328 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
329 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
330 Action: Expand);
331
332 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG,
333 VTs: {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
334 MVT::v3i16, MVT::v4i16, MVT::Other},
335 Action: Custom);
336
337 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
338 setOperationAction(Ops: ISD::BR_CC,
339 VTs: {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Action: Expand);
340
341 setOperationAction(Ops: {ISD::ABS, ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Legal);
342 setOperationAction(Ops: {ISD::UADDO, ISD::USUBO}, VT: MVT::i64, Action: Legal);
343
344 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i32, Action: Legal);
345 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i64, Action: Legal);
346
347 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, VT: MVT::i64,
348 Action: Expand);
349
350 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
351
352 // We only support LOAD/STORE and vector manipulation ops for vectors
353 // with > 4 elements.
354 for (MVT VT :
355 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
356 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
357 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
358 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
359 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
360 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
361 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
362 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
363 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
364 switch (Op) {
365 case ISD::LOAD:
366 case ISD::STORE:
367 case ISD::ATOMIC_LOAD:
368 case ISD::ATOMIC_STORE:
369 case ISD::BUILD_VECTOR:
370 case ISD::BITCAST:
371 case ISD::UNDEF:
372 case ISD::POISON:
373 case ISD::EXTRACT_VECTOR_ELT:
374 case ISD::INSERT_VECTOR_ELT:
375 case ISD::SCALAR_TO_VECTOR:
376 case ISD::IS_FPCLASS:
377 break;
378 case ISD::EXTRACT_SUBVECTOR:
379 case ISD::INSERT_SUBVECTOR:
380 case ISD::CONCAT_VECTORS:
381 setOperationAction(Op, VT, Action: Custom);
382 break;
383 default:
384 setOperationAction(Op, VT, Action: Expand);
385 break;
386 }
387 }
388 }
389
390 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f32, Action: Expand);
391
392 // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
393 // is expanded to avoid having two separate loops in case the index is a VGPR.
394
395 // Most operations are naturally 32-bit vector operations. We only support
396 // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
397 for (MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
398 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
399 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
400
401 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
402 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
403
404 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
405 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
406
407 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
408 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
409 }
410
411 for (MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
412 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
413 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
414
415 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
416 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
417
418 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
419 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
420
421 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
422 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
423 }
424
425 for (MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
426 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
427 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
428
429 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
430 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
431
432 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
433 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
434
435 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
436 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
437 }
438
439 for (MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
440 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
441 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
442
443 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
444 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
445
446 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
447 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
448
449 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
450 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
451 }
452
453 for (MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
454 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
455 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
456
457 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
458 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
459
460 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
461 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
462
463 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
464 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
465 }
466
467 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
468 VTs: {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
469 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
470 Action: Custom);
471
472 if (Subtarget->hasPkMovB32()) {
473 // TODO: 16-bit element vectors should be legal with even aligned elements.
474 // TODO: Can be legal with wider source types than the result with
475 // subregister extracts.
476 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2i32, MVT::v2f32}, Action: Legal);
477 }
478
479 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT: MVT::v2i32, Action: Legal);
480 // Prevent SELECT v2i32 from being implemented with the above bitwise ops and
481 // instead lower to cndmask in SITargetLowering::LowerSELECT().
482 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i32, Action: Custom);
483 // Enable MatchRotate to produce ISD::ROTR, which is later transformed to
484 // alignbit.
485 setOperationAction(Op: ISD::ROTR, VT: MVT::v2i32, Action: Custom);
486
487 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
488 Action: Custom);
489
490 // Avoid stack access for these.
491 // TODO: Generalize to more vector types.
492 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
493 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
494 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
495 Action: Custom);
496
497 // Deal with vec3 vector operations when widened to vec4.
498 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
499 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
500
501 // Deal with vec5/6/7 vector operations when widened to vec8.
502 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
503 VTs: {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
504 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
505 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
506 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
507 Action: Custom);
508
509 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
510 // and output demarshalling
511 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP, VTs: {MVT::i32, MVT::i64}, Action: Custom);
512
513 // We can't return success/failure, only the old value,
514 // let LLVM add the comparison
515 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VTs: {MVT::i32, MVT::i64},
516 Action: Expand);
517
518 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
519
520 setOperationAction(Ops: ISD::BITREVERSE, VTs: {MVT::i32, MVT::i64}, Action: Legal);
521
522 // FIXME: This should be narrowed to i32, but that only happens if i64 is
523 // illegal.
524 // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
525 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i64, MVT::i32}, Action: Legal);
526
527 // On SI this is s_memtime and s_memrealtime on VI.
528 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
529
530 if (Subtarget->hasSMemRealTime() ||
531 Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
532 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
533 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Custom);
534
535 if (Subtarget->has16BitInsts()) {
536 setOperationAction(Ops: {ISD::FPOW, ISD::FPOWI}, VT: MVT::f16, Action: Promote);
537 setOperationAction(Ops: {ISD::FLOG, ISD::FEXP, ISD::FLOG10}, VT: MVT::f16, Action: Custom);
538 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Legal);
539 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Legal);
540 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f16, Action: Legal);
541 } else {
542 setOperationAction(Op: ISD::FSQRT, VT: MVT::f16, Action: Custom);
543 }
544
545 if (Subtarget->hasMadMacF32Insts())
546 setOperationAction(Op: ISD::FMAD, VT: MVT::f32, Action: Legal);
547
548 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
549 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
550 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
551
552 // We only really have 32-bit BFE instructions (and 16-bit on VI).
553 //
554 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
555 // effort to match them now. We want this to be false for i64 cases when the
556 // extraction isn't restricted to the upper or lower half. Ideally we would
557 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
558 // span the midpoint are probably relatively rare, so don't worry about them
559 // for now.
560 setHasExtractBitsInsn(true);
561
562 // Clamp modifier on add/sub
563 if (Subtarget->hasIntClamp())
564 setOperationAction(Ops: {ISD::UADDSAT, ISD::USUBSAT}, VT: MVT::i32, Action: Legal);
565
566 if (Subtarget->hasAddNoCarryInsts())
567 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT}, VTs: {MVT::i16, MVT::i32},
568 Action: Legal);
569
570 // Do not have s_{min|max}_*f64 instruction f64 will only be lowered to
571 // v_{min|max}_*f64
572 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
573 setOperationAction(
574 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
575 VTs: {MVT::f64, MVT::f32}, Action: Legal);
576 } else {
577 setOperationAction(
578 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
579 VTs: {MVT::f64, MVT::f32}, Action: Custom);
580 // These are really only legal for ieee_mode functions. We should be
581 // avoiding them for functions that don't have ieee_mode enabled, so just
582 // say they are legal.
583 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
584 VTs: {MVT::f64, MVT::f32}, Action: Legal);
585 }
586
587 if (Subtarget->haveRoundOpsF64())
588 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, VT: MVT::f64,
589 Action: Legal);
590 else
591 setOperationAction(Ops: {ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
592 VT: MVT::f64, Action: Custom);
593
594 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
595 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VTs: {MVT::f32, MVT::f64},
596 Action: Legal);
597 setOperationAction(Ops: ISD::FFREXP, VTs: {MVT::f32, MVT::f64}, Action: Custom);
598
599 setOperationAction(Ops: {ISD::FSIN, ISD::FCOS, ISD::FDIV}, VT: MVT::f32, Action: Custom);
600 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Custom);
601
602 setOperationAction(Ops: ISD::BF16_TO_FP, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
603 setOperationAction(Ops: ISD::FP_TO_BF16, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
604
605 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i32,
606 Action: Custom);
607 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i16,
608 Action: Custom);
609 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i1,
610 Action: Custom);
611
612 // Custom lower these because we can't specify a rule based on an illegal
613 // source bf16.
614 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f32, Action: Custom);
615 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f64, Action: Custom);
616
617 if (Subtarget->has16BitInsts()) {
618 setOperationAction(Ops: {ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
619 ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
620 VT: MVT::i16, Action: Legal);
621
622 AddPromotedToType(Opc: ISD::SIGN_EXTEND, OrigVT: MVT::i16, DestVT: MVT::i32);
623
624 setOperationAction(Ops: {ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
625 VT: MVT::i16, Action: Expand);
626
627 setOperationAction(Ops: {ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
628 ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
629 ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
630 ISD::CTPOP},
631 VT: MVT::i16, Action: Promote);
632
633 setOperationAction(Op: ISD::LOAD, VT: MVT::i16, Action: Custom);
634
635 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
636
637 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::i16, Action: Promote);
638 AddPromotedToType(Opc: ISD::FP16_TO_FP, OrigVT: MVT::i16, DestVT: MVT::i32);
639 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::i16, Action: Promote);
640 AddPromotedToType(Opc: ISD::FP_TO_FP16, OrigVT: MVT::i16, DestVT: MVT::i32);
641
642 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i16, Action: Custom);
643 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i32, Action: Custom);
644 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i16, Action: Custom);
645 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i1, Action: Custom);
646
647 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i32, Action: Custom);
648
649 // F16 - Constant Actions.
650 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
651 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
652
653 // F16 - Load/Store Actions.
654 setOperationAction(Op: ISD::LOAD, VT: MVT::f16, Action: Promote);
655 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
656 setOperationAction(Op: ISD::STORE, VT: MVT::f16, Action: Promote);
657 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
658
659 // BF16 - Load/Store Actions.
660 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Promote);
661 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
662 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Promote);
663 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
664
665 if (!Subtarget->useRealTrue16Insts() ||
666 !Subtarget->d16PreservesUnusedBits()) {
667 for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
668 setOperationAction(Op: ISD::ATOMIC_LOAD, VT, Action: Promote);
669 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: VT, DestVT: MVT::i32);
670 }
671 }
672
673 if (!Subtarget->useRealTrue16Insts()) {
674 for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
675 setOperationAction(Op: ISD::ATOMIC_STORE, VT, Action: Promote);
676 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: VT, DestVT: MVT::i32);
677 }
678 }
679
680 // F16 - VOP1 Actions.
681 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
682 ISD::FSIN, ISD::FROUND},
683 VT: MVT::f16, Action: Custom);
684
685 // BF16 - VOP1 Actions.
686 if (Subtarget->hasBF16TransInsts())
687 setOperationAction(Ops: {ISD::FCOS, ISD::FSIN, ISD::FDIV}, VT: MVT::bf16, Action: Custom);
688
689 // F16 - VOP2 Actions.
690 setOperationAction(Ops: {ISD::BR_CC, ISD::SELECT_CC}, VTs: {MVT::f16, MVT::bf16},
691 Action: Expand);
692 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VT: MVT::f16, Action: Custom);
693 setOperationAction(Op: ISD::FFREXP, VT: MVT::f16, Action: Custom);
694 setOperationAction(Op: ISD::FDIV, VT: MVT::f16, Action: Custom);
695
696 // F16 - VOP3 Actions.
697 setOperationAction(Op: ISD::FMA, VT: MVT::f16, Action: Legal);
698 if (STI.hasMadF16())
699 setOperationAction(Op: ISD::FMAD, VT: MVT::f16, Action: Legal);
700
701 for (MVT VT :
702 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
703 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
704 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
705 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
706 switch (Op) {
707 case ISD::LOAD:
708 case ISD::STORE:
709 case ISD::ATOMIC_LOAD:
710 case ISD::ATOMIC_STORE:
711 case ISD::BUILD_VECTOR:
712 case ISD::BITCAST:
713 case ISD::UNDEF:
714 case ISD::POISON:
715 case ISD::EXTRACT_VECTOR_ELT:
716 case ISD::INSERT_VECTOR_ELT:
717 case ISD::INSERT_SUBVECTOR:
718 case ISD::SCALAR_TO_VECTOR:
719 case ISD::IS_FPCLASS:
720 break;
721 case ISD::EXTRACT_SUBVECTOR:
722 case ISD::CONCAT_VECTORS:
723 case ISD::FSIN:
724 case ISD::FCOS:
725 setOperationAction(Op, VT, Action: Custom);
726 break;
727 default:
728 setOperationAction(Op, VT, Action: Expand);
729 break;
730 }
731 }
732 }
733
734 // v_perm_b32 can handle either of these.
735 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::v2i16}, Action: Legal);
736 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i16, Action: Custom);
737
738 // Legalize vector types for sat conversions to select v_cvt_pk_[iu]16_f32.
739 if (Subtarget->hasVCvtPkIU16F32())
740 setOperationAction(
741 Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT},
742 VTs: {MVT::v2i16, MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16},
743 Action: Custom);
744
745 // XXX - Do these do anything? Vector constants turn into build_vector.
746 setOperationAction(Ops: ISD::Constant, VTs: {MVT::v2i16, MVT::v2f16}, Action: Legal);
747
748 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON},
749 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Legal);
750
751 setOperationAction(Op: ISD::STORE, VT: MVT::v2i16, Action: Promote);
752 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
753 setOperationAction(Op: ISD::STORE, VT: MVT::v2f16, Action: Promote);
754 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
755
756 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i16, Action: Promote);
757 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
758 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f16, Action: Promote);
759 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
760
761 setOperationAction(Op: ISD::AND, VT: MVT::v2i16, Action: Promote);
762 AddPromotedToType(Opc: ISD::AND, OrigVT: MVT::v2i16, DestVT: MVT::i32);
763 setOperationAction(Op: ISD::OR, VT: MVT::v2i16, Action: Promote);
764 AddPromotedToType(Opc: ISD::OR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
765 setOperationAction(Op: ISD::XOR, VT: MVT::v2i16, Action: Promote);
766 AddPromotedToType(Opc: ISD::XOR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
767
768 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i16, Action: Promote);
769 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
770 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f16, Action: Promote);
771 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
772 setOperationAction(Op: ISD::LOAD, VT: MVT::v4bf16, Action: Promote);
773 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
774
775 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
776 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
777 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
778 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
779 setOperationAction(Op: ISD::STORE, VT: MVT::v4bf16, Action: Promote);
780 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
781
782 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i16, Action: Promote);
783 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
784 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f16, Action: Promote);
785 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
786 setOperationAction(Op: ISD::LOAD, VT: MVT::v8bf16, Action: Promote);
787 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
788
789 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
790 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
791 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
792 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
793
794 setOperationAction(Op: ISD::STORE, VT: MVT::v8i16, Action: Promote);
795 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
796 setOperationAction(Op: ISD::STORE, VT: MVT::v8f16, Action: Promote);
797 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
798 setOperationAction(Op: ISD::STORE, VT: MVT::v8bf16, Action: Promote);
799 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
800
801 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i16, Action: Promote);
802 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
803 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f16, Action: Promote);
804 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
805 setOperationAction(Op: ISD::LOAD, VT: MVT::v16bf16, Action: Promote);
806 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
807
808 setOperationAction(Op: ISD::STORE, VT: MVT::v16i16, Action: Promote);
809 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
810 setOperationAction(Op: ISD::STORE, VT: MVT::v16f16, Action: Promote);
811 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
812 setOperationAction(Op: ISD::STORE, VT: MVT::v16bf16, Action: Promote);
813 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
814
815 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i16, Action: Promote);
816 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
817 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f16, Action: Promote);
818 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
819 setOperationAction(Op: ISD::LOAD, VT: MVT::v32bf16, Action: Promote);
820 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
821
822 setOperationAction(Op: ISD::STORE, VT: MVT::v32i16, Action: Promote);
823 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
824 setOperationAction(Op: ISD::STORE, VT: MVT::v32f16, Action: Promote);
825 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
826 setOperationAction(Op: ISD::STORE, VT: MVT::v32bf16, Action: Promote);
827 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
828
829 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
830 VT: MVT::v2i32, Action: Expand);
831 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
832
833 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
834 VT: MVT::v4i32, Action: Expand);
835
836 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
837 VT: MVT::v8i32, Action: Expand);
838
839 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
840 Action: Subtarget->hasVOP3PInsts() ? Legal : Custom);
841
842 setOperationAction(Ops: ISD::FNEG, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
843 // This isn't really legal, but this avoids the legalizer unrolling it (and
844 // allows matching fneg (fabs x) patterns)
845 setOperationAction(Ops: ISD::FABS, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
846
847 // Can do this in one BFI plus a constant materialize.
848 setOperationAction(Ops: ISD::FCOPYSIGN,
849 VTs: {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
850 MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
851 MVT::v32f16, MVT::v32bf16},
852 Action: Custom);
853 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
854 setOperationAction(
855 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
856 VT: MVT::f16, Action: Legal);
857
858 setOperationAction(
859 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
860 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, Action: Custom);
861 } else {
862 setOperationAction(
863 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
864 VT: MVT::f16, Action: Custom);
865
866 setOperationAction(Ops: {ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, VT: MVT::f16,
867 Action: Legal);
868
869 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
870 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
871 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
872 Action: Custom);
873
874 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM},
875 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
876 Action: Expand);
877 }
878
879 for (MVT Vec16 :
880 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
881 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
882 setOperationAction(
883 Ops: {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
884 VT: Vec16, Action: Custom);
885 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec16, Action: Expand);
886 }
887 }
888
889 if (Subtarget->hasVOP3PInsts()) {
890 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
891 ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
892 ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
893 VT: MVT::v2i16, Action: Legal);
894
895 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
896 ISD::FCANONICALIZE},
897 VT: MVT::v2f16, Action: Legal);
898
899 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT,
900 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Custom);
901
902 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
903 VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
904 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
905 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
906 Action: Custom);
907
908 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
909 // Split vector operations.
910 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
911 ISD::MUL, ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN,
912 ISD::UMAX, ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
913 ISD::SSUBSAT},
914 VT, Action: Custom);
915
916 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
917 // Split vector operations.
918 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
919 ISD::FCANONICALIZE},
920 VT, Action: Custom);
921
922 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
923 setOperationAction(
924 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
925 VT: MVT::v2f16, Action: Legal);
926 } else {
927 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE}, VT: MVT::v2f16,
928 Action: Legal);
929
930 setOperationAction(
931 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
932 VTs: {MVT::v2f16, MVT::v4f16}, Action: Custom);
933 }
934 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f16, Action: Custom);
935 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
936 Action: Custom);
937
938 if (Subtarget->hasBF16PackedInsts()) {
939 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMAXNUM, ISD::FMINNUM,
940 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FMA,
941 ISD::FNEG, ISD::FABS, ISD::FCANONICALIZE},
942 VT: MVT::v2bf16, Action: Legal);
943
944 for (MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
945 // Split vector operations.
946 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE,
947 ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM,
948 ISD::FMAXIMUMNUM, ISD::FNEG, ISD::FABS},
949 VT, Action: Custom);
950 }
951
952 if (Subtarget->hasAnyPackedFP32Ops()) {
953 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
954 VT: MVT::v2f32, Action: Legal);
955 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
956 VTs: {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
957 Action: Custom);
958 }
959 if (Subtarget->hasAnyPackedFP64Ops()) {
960 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG,
961 ISD::FCANONICALIZE, ISD::BUILD_VECTOR},
962 VT: MVT::v2f64, Action: Legal);
963 setOperationAction(
964 Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FCANONICALIZE},
965 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Action: Custom);
966
967 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
968 setOperationAction(
969 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
970 VT: MVT::v2f64, Action: Legal);
971
972 setOperationAction(
973 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
974 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Action: Custom);
975 } else {
976 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE}, VT: MVT::v2f64,
977 Action: Legal);
978 setOperationAction(
979 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
980 VT: MVT::v2f64, Action: Custom);
981 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINNUM,
982 ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
983 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64},
984 Action: Custom);
985 }
986 }
987
988 if (Subtarget->hasAnyPackedU64Ops()) {
989 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL, ISD::BUILD_VECTOR},
990 VT: MVT::v2i64, Action: Legal);
991 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL},
992 VTs: {MVT::v4i64, MVT::v8i64, MVT::v16i64, MVT::v32i64},
993 Action: Custom);
994 }
995 }
996
997 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v4f16, Action: Custom);
998
999 if (Subtarget->has16BitInsts()) {
1000 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i16, Action: Promote);
1001 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2i16, DestVT: MVT::i32);
1002 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f16, Action: Promote);
1003 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f16, DestVT: MVT::i32);
1004 setOperationAction(Op: ISD::SELECT, VT: MVT::v2bf16, Action: Promote);
1005 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2bf16, DestVT: MVT::i32);
1006 } else {
1007 // Legalization hack.
1008 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v2i16, MVT::v2f16}, Action: Custom);
1009
1010 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v2f16, Action: Custom);
1011 }
1012
1013 setOperationAction(Ops: ISD::SELECT,
1014 VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
1015 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1016 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
1017 MVT::v32f16, MVT::v32bf16},
1018 Action: Custom);
1019
1020 setOperationAction(Ops: {ISD::SMULO, ISD::UMULO}, VT: MVT::i64, Action: Custom);
1021
1022 if (Subtarget->useVMulU64Inst())
1023 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Legal);
1024 else if (Subtarget->hasScalarSMulU64())
1025 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
1026
1027 if (Subtarget->hasMad64_32())
1028 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: MVT::i32, Action: Custom);
1029
1030 if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
1031 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1032
1033 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
1034 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM},
1035 VTs: {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Action: Legal);
1036 } else {
1037 // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum
1038 if (Subtarget->hasMinimum3Maximum3F32())
1039 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
1040
1041 if (Subtarget->hasMinimum3Maximum3PKF16()) {
1042 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::v2f16, Action: Legal);
1043
1044 // If only the vector form is available, we need to widen to a vector.
1045 if (!Subtarget->hasMinimum3Maximum3F16())
1046 setOperationPromotedToType(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, OrigVT: MVT::f16,
1047 DestVT: MVT::v2f16);
1048 }
1049 }
1050
1051 if (Subtarget->hasVOP3PInsts()) {
1052 // We want to break these into v2f16 pieces, not scalarize.
1053 setOperationAction(Ops: {ISD::FMINIMUM, ISD::FMAXIMUM},
1054 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
1055 Action: Custom);
1056 }
1057
1058 if (Subtarget->useMinMaxI64Insts())
1059 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i64,
1060 Action: Legal);
1061
1062 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
1063 VTs: {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
1064 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
1065 MVT::i8},
1066 Action: Custom);
1067
1068 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
1069 VTs: {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
1070 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
1071 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
1072 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1073 Action: Custom);
1074
1075 // The s_buffer_load intrinsics accept any result type in IR, but only a few
1076 // of them can be selected. Mark the remaining illegal result types Custom so
1077 // ReplaceNodeResults gets a chance to diagnose them instead of letting the
1078 // type legalizer abort. Its INTRINSIC_WO_CHAIN case dispatches on the
1079 // intrinsic ID, but INTRINSIC_W_CHAIN does not, so remember the types added
1080 // here to keep other chained intrinsics on generic legalization.
1081 for (MVT VT : MVT::all_valuetypes()) {
1082 if (VT.isScalableVector() || isTypeLegal(VT))
1083 continue;
1084 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT, Action: Custom);
1085 if (getOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT) != Custom) {
1086 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT, Action: Custom);
1087 SBufferLoadDiagnosticVTs.set(position: VT.SimpleTy);
1088 }
1089 }
1090
1091 setOperationAction(Ops: ISD::INTRINSIC_VOID,
1092 VTs: {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
1093 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
1094 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1095 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1096 Action: Custom);
1097
1098 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
1099 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
1100 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
1101 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i64, Action: Custom);
1102 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i64, Action: Custom);
1103
1104 // TODO: Could move this to custom lowering, could benefit from combines on
1105 // extract of relevant bits.
1106 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
1107
1108 setOperationAction(Op: ISD::MUL, VT: MVT::i1, Action: Promote);
1109
1110 if (Subtarget->hasBF16ConversionInsts()) {
1111 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND},
1112 VTs: {MVT::bf16, MVT::v2bf16}, Action: Custom);
1113 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Legal);
1114 }
1115
1116 if (Subtarget->hasBF16TransInsts()) {
1117 setOperationAction(Ops: {ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, VT: MVT::bf16, Action: Legal);
1118 }
1119
1120 const bool HasE5M3ConversionInsts =
1121 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
1122 if (Subtarget->hasOCPFP8ConversionInsts() || HasE5M3ConversionInsts) {
1123 setOperationAction(Ops: ISD::CONVERT_FROM_ARBITRARY_FP, VTs: {MVT::f32, MVT::v2f32},
1124 Action: Custom);
1125 setOperationAction(Op: ISD::CONVERT_FROM_ARBITRARY_FP, VT: MVT::v2i8, Action: Custom);
1126
1127 // i8 result promotes to i16, wider vectors split down to v2i8, and v2i8 is
1128 // handled in ReplaceNodeResults before the legalizer splits it per lane.
1129 setOperationAction(Ops: ISD::CONVERT_TO_ARBITRARY_FP, VTs: {MVT::i16, MVT::v2i8},
1130 Action: Custom);
1131 }
1132
1133 if (Subtarget->hasFP8F16ConversionInsts()) {
1134 setOperationAction(Ops: ISD::CONVERT_FROM_ARBITRARY_FP, VTs: {MVT::f16, MVT::v2f16},
1135 Action: Custom);
1136 }
1137
1138 if (Subtarget->hasCvtPkF16F32Inst()) {
1139 setOperationAction(Ops: ISD::FP_ROUND,
1140 VTs: {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
1141 Action: Custom);
1142 }
1143
1144 setTargetDAGCombine({ISD::ADD,
1145 ISD::PTRADD,
1146 ISD::SUB,
1147 ISD::MUL,
1148 ISD::FADD,
1149 ISD::FSUB,
1150 ISD::FDIV,
1151 ISD::FMUL,
1152 ISD::FMINNUM,
1153 ISD::FMAXNUM,
1154 ISD::FMINNUM_IEEE,
1155 ISD::FMAXNUM_IEEE,
1156 ISD::FMINIMUM,
1157 ISD::FMAXIMUM,
1158 ISD::FMINIMUMNUM,
1159 ISD::FMAXIMUMNUM,
1160 ISD::FMA,
1161 ISD::ABS,
1162 ISD::SMIN,
1163 ISD::SMAX,
1164 ISD::UMIN,
1165 ISD::UMAX,
1166 ISD::SETCC,
1167 ISD::SELECT,
1168 ISD::SMIN,
1169 ISD::SMAX,
1170 ISD::UMIN,
1171 ISD::UMAX,
1172 ISD::USUBSAT,
1173 ISD::UADDSAT,
1174 ISD::AND,
1175 ISD::OR,
1176 ISD::XOR,
1177 ISD::SHL,
1178 ISD::SRL,
1179 ISD::SRA,
1180 ISD::FSHR,
1181 ISD::SINT_TO_FP,
1182 ISD::UINT_TO_FP,
1183 ISD::FCANONICALIZE,
1184 ISD::SCALAR_TO_VECTOR,
1185 ISD::ZERO_EXTEND,
1186 ISD::SIGN_EXTEND,
1187 ISD::SIGN_EXTEND_INREG,
1188 ISD::ANY_EXTEND,
1189 ISD::EXTRACT_VECTOR_ELT,
1190 ISD::INSERT_VECTOR_ELT,
1191 ISD::FCOPYSIGN});
1192
1193 if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
1194 setTargetDAGCombine(ISD::FP_ROUND);
1195
1196 // All memory operations. Some folding on the pointer operand is done to help
1197 // matching the constant offsets in the addressing modes.
1198 setTargetDAGCombine({ISD::LOAD,
1199 ISD::STORE,
1200 ISD::ATOMIC_LOAD,
1201 ISD::ATOMIC_STORE,
1202 ISD::ATOMIC_CMP_SWAP,
1203 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
1204 ISD::ATOMIC_SWAP,
1205 ISD::ATOMIC_LOAD_ADD,
1206 ISD::ATOMIC_LOAD_SUB,
1207 ISD::ATOMIC_LOAD_AND,
1208 ISD::ATOMIC_LOAD_OR,
1209 ISD::ATOMIC_LOAD_XOR,
1210 ISD::ATOMIC_LOAD_NAND,
1211 ISD::ATOMIC_LOAD_MIN,
1212 ISD::ATOMIC_LOAD_MAX,
1213 ISD::ATOMIC_LOAD_UMIN,
1214 ISD::ATOMIC_LOAD_UMAX,
1215 ISD::ATOMIC_LOAD_FADD,
1216 ISD::ATOMIC_LOAD_FMIN,
1217 ISD::ATOMIC_LOAD_FMAX,
1218 ISD::ATOMIC_LOAD_UINC_WRAP,
1219 ISD::ATOMIC_LOAD_UDEC_WRAP,
1220 ISD::ATOMIC_LOAD_USUB_COND,
1221 ISD::ATOMIC_LOAD_USUB_SAT,
1222 ISD::INTRINSIC_VOID,
1223 ISD::INTRINSIC_W_CHAIN});
1224
1225 // FIXME: In other contexts we pretend this is a per-function property.
1226 setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
1227
1228 setSchedulingPreference(Sched::RegPressure);
1229}
1230
1231const GCNSubtarget *SITargetLowering::getSubtarget() const { return Subtarget; }
1232
1233ArrayRef<MCPhysReg> SITargetLowering::getRoundingControlRegisters() const {
1234 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1235 return RCRegs;
1236}
1237
1238//===----------------------------------------------------------------------===//
1239// TargetLowering queries
1240//===----------------------------------------------------------------------===//
1241
1242// v_mad_mix* support a conversion from f16 to f32.
1243//
1244// There is only one special case when denormals are enabled we don't currently,
1245// where this is OK to use.
1246bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
1247 EVT DestVT, EVT SrcVT) const {
1248 return DestVT.getScalarType() == MVT::f32 &&
1249 ((((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
1250 (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
1251 SrcVT.getScalarType() == MVT::f16) ||
1252 (Opcode == ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
1253 SrcVT.getScalarType() == MVT::bf16)) &&
1254 // TODO: This probably only requires no input flushing?
1255 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
1256}
1257
1258bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
1259 LLT DestTy, LLT SrcTy) const {
1260 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
1261 (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
1262 DestTy.getScalarSizeInBits() == 32 &&
1263 SrcTy.getScalarSizeInBits() == 16 &&
1264 // TODO: This probably only requires no input flushing?
1265 denormalModeIsFlushAllF32(MF: *MI.getMF());
1266}
1267
1268bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
1269 // SI has some legal vector types, but no legal vector operations. Say no
1270 // shuffles are legal in order to prefer scalarizing some vector operations.
1271 return false;
1272}
1273
1274MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1275 CallingConv::ID CC,
1276 EVT VT) const {
1277 if (CC == CallingConv::AMDGPU_KERNEL)
1278 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1279
1280 if (VT.isVector()) {
1281 EVT ScalarVT = VT.getScalarType();
1282 unsigned Size = ScalarVT.getSizeInBits();
1283 if (Size == 16) {
1284 return Subtarget->has16BitInsts()
1285 ? MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), NumElements: 2)
1286 : MVT::i32;
1287 }
1288
1289 if (Size < 16)
1290 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
1291 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
1292 }
1293
1294 if (!Subtarget->has16BitInsts() && VT.getSizeInBits() == 16)
1295 return MVT::i32;
1296
1297 if (VT.getSizeInBits() > 32)
1298 return MVT::i32;
1299
1300 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1301}
1302
1303unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1304 CallingConv::ID CC,
1305 EVT VT) const {
1306 if (CC == CallingConv::AMDGPU_KERNEL)
1307 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1308
1309 if (VT.isVector()) {
1310 unsigned NumElts = VT.getVectorNumElements();
1311 EVT ScalarVT = VT.getScalarType();
1312 unsigned Size = ScalarVT.getSizeInBits();
1313
1314 // FIXME: Should probably promote 8-bit vectors to i16.
1315 if (Size == 16)
1316 return (NumElts + 1) / 2;
1317
1318 if (Size <= 32)
1319 return NumElts;
1320
1321 if (Size > 32)
1322 return NumElts * ((Size + 31) / 32);
1323 } else if (VT.getSizeInBits() > 32)
1324 return (VT.getSizeInBits() + 31) / 32;
1325
1326 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1327}
1328
1329unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
1330 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1331 unsigned &NumIntermediates, MVT &RegisterVT) const {
1332 if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
1333 unsigned NumElts = VT.getVectorNumElements();
1334 EVT ScalarVT = VT.getScalarType();
1335 unsigned Size = ScalarVT.getSizeInBits();
1336 // FIXME: We should fix the ABI to be the same on targets without 16-bit
1337 // support, but unless we can properly handle 3-vectors, it will be still be
1338 // inconsistent.
1339 if (Size == 16) {
1340 MVT SimpleIntermediateVT =
1341 MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), EC: ElementCount::getFixed(MinVal: 2));
1342 IntermediateVT = SimpleIntermediateVT;
1343 RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
1344 NumIntermediates = (NumElts + 1) / 2;
1345 return (NumElts + 1) / 2;
1346 }
1347
1348 if (Size == 32) {
1349 RegisterVT = ScalarVT.getSimpleVT();
1350 IntermediateVT = RegisterVT;
1351 NumIntermediates = NumElts;
1352 return NumIntermediates;
1353 }
1354
1355 if (Size < 16 && Subtarget->has16BitInsts()) {
1356 // FIXME: Should probably form v2i16 pieces
1357 RegisterVT = MVT::i16;
1358 IntermediateVT = ScalarVT;
1359 NumIntermediates = NumElts;
1360 return NumIntermediates;
1361 }
1362
1363 if (Size != 16 && Size <= 32) {
1364 RegisterVT = MVT::i32;
1365 IntermediateVT = ScalarVT;
1366 NumIntermediates = NumElts;
1367 return NumIntermediates;
1368 }
1369
1370 if (Size > 32) {
1371 RegisterVT = MVT::i32;
1372 IntermediateVT = RegisterVT;
1373 NumIntermediates = NumElts * ((Size + 31) / 32);
1374 return NumIntermediates;
1375 }
1376 }
1377
1378 return TargetLowering::getVectorTypeBreakdownForCallingConv(
1379 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1380}
1381
1382static EVT memVTFromLoadIntrData(const SITargetLowering &TLI,
1383 const DataLayout &DL, Type *Ty,
1384 unsigned MaxNumLanes) {
1385 assert(MaxNumLanes != 0);
1386
1387 LLVMContext &Ctx = Ty->getContext();
1388 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1389 unsigned NumElts = std::min(a: MaxNumLanes, b: VT->getNumElements());
1390 return EVT::getVectorVT(Context&: Ctx, VT: TLI.getValueType(DL, Ty: VT->getElementType()),
1391 NumElements: NumElts);
1392 }
1393
1394 return TLI.getValueType(DL, Ty);
1395}
1396
1397// Peek through TFE struct returns to only use the data size.
1398static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI,
1399 const DataLayout &DL, Type *Ty,
1400 unsigned MaxNumLanes) {
1401 auto *ST = dyn_cast<StructType>(Val: Ty);
1402 if (!ST)
1403 return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
1404
1405 // TFE intrinsics return an aggregate type.
1406 assert(ST->getNumContainedTypes() == 2 &&
1407 ST->getContainedType(1)->isIntegerTy(32));
1408 return memVTFromLoadIntrData(TLI, DL, Ty: ST->getContainedType(i: 0), MaxNumLanes);
1409}
1410
1411/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its
1412/// in-memory representation. This return value is a custom type because there
1413/// is no MVT::i160 and adding one breaks integer promotion logic. While this
1414/// could cause issues during codegen, these address space 7 pointers will be
1415/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer
1416/// in order to allow pre-codegen passes that query TargetTransformInfo, often
1417/// for cost modeling, to work. (This also sets us up decently for doing the
1418/// buffer lowering in GlobalISel if SelectionDAG ever goes away.)
1419MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
1420 if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
1421 return MVT::amdgpuBufferFatPointer;
1422 if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1423 DL.getPointerSizeInBits(AS) == 192)
1424 return MVT::amdgpuBufferStridedPointer;
1425 return AMDGPUTargetLowering::getPointerTy(DL, AS);
1426}
1427/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
1428/// v8i32 when padding is added.
1429/// The in-memory representation of a p9 is {p8, i32, i32}, which is
1430/// also v8i32 with padding.
1431MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
1432 if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
1433 DL.getPointerSizeInBits(AS) == 160) ||
1434 (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1435 DL.getPointerSizeInBits(AS) == 192))
1436 return MVT::v8i32;
1437 return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
1438}
1439
1440static unsigned getIntrMemWidth(unsigned IntrID) {
1441 switch (IntrID) {
1442 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1443 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1444 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1445 return 8;
1446 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1447 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1448 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1449 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1450 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1451 case Intrinsic::amdgcn_flat_load_monitor_b32:
1452 case Intrinsic::amdgcn_global_load_monitor_b32:
1453 return 32;
1454 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1455 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1456 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1457 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1458 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1459 case Intrinsic::amdgcn_flat_load_monitor_b64:
1460 case Intrinsic::amdgcn_global_load_monitor_b64:
1461 return 64;
1462 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1463 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1464 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1465 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
1466 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
1467 case Intrinsic::amdgcn_flat_load_monitor_b128:
1468 case Intrinsic::amdgcn_global_load_monitor_b128:
1469 return 128;
1470 default:
1471 llvm_unreachable("Unknown width");
1472 }
1473}
1474
1475static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI,
1476 unsigned ArgIdx) {
1477 Value *OrderingArg = CI.getArgOperand(i: ArgIdx);
1478 unsigned Ord = cast<ConstantInt>(Val: OrderingArg)->getZExtValue();
1479 switch (AtomicOrderingCABI(Ord)) {
1480 case AtomicOrderingCABI::acquire:
1481 return AtomicOrdering::Acquire;
1482 break;
1483 case AtomicOrderingCABI::release:
1484 return AtomicOrdering::Release;
1485 break;
1486 case AtomicOrderingCABI::seq_cst:
1487 return AtomicOrdering::SequentiallyConsistent;
1488 break;
1489 default:
1490 return AtomicOrdering::Monotonic;
1491 }
1492}
1493
1494static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
1495 MDNode *ScopeMD = cast<MDNode>(
1496 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ArgIdx))->getMetadata());
1497 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1498 return CI.getContext().getOrInsertSyncScopeID(SSN: Scope);
1499}
1500
1501void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
1502 const CallBase &CI,
1503 MachineFunction &MF,
1504 unsigned IntrID) const {
1505 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
1506 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
1507 Flags |= MachineMemOperand::MOInvariant;
1508 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
1509 Flags |= MachineMemOperand::MONonTemporal;
1510 Flags |= getTargetMMOFlags(I: CI);
1511
1512 if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
1513 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
1514 AttributeSet Attr =
1515 Intrinsic::getFnAttributes(C&: CI.getContext(), id: (Intrinsic::ID)IntrID);
1516 MemoryEffects ME = Attr.getMemoryEffects();
1517 if (ME.doesNotAccessMemory())
1518 return;
1519
1520 bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
1521 if (!IsSPrefetch) {
1522 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1523 if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
1524 Flags |= MachineMemOperand::MOVolatile;
1525 }
1526
1527 Flags |= MachineMemOperand::MODereferenceable;
1528
1529 IntrinsicInfo Info;
1530 // TODO: Should images get their own address space?
1531 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1532
1533 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
1534 if (RsrcIntr->IsImage) {
1535 const AMDGPU::ImageDimIntrinsicInfo *Intr =
1536 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID);
1537 BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Intr->BaseOpcode);
1538 Info.align.reset();
1539 }
1540
1541 Value *RsrcArg = CI.getArgOperand(i: RsrcIntr->RsrcArg);
1542 if (auto *RsrcPtrTy = dyn_cast<PointerType>(Val: RsrcArg->getType())) {
1543 if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
1544 // We conservatively set the memory operand of a buffer intrinsic to the
1545 // base resource pointer, so that we can access alias information about
1546 // those pointers. Cases like "this points at the same value
1547 // but with a different offset" are handled in
1548 // areMemAccessesTriviallyDisjoint.
1549 Info.ptrVal = RsrcArg;
1550 }
1551
1552 if (ME.onlyReadsMemory()) {
1553 if (RsrcIntr->IsImage) {
1554 unsigned MaxNumLanes = 4;
1555
1556 if (!BaseOpcode->Gather4) {
1557 // If this isn't a gather, we may have excess loaded elements in the
1558 // IR type. Check the dmask for the real number of elements loaded.
1559 unsigned DMask =
1560 cast<ConstantInt>(Val: CI.getArgOperand(i: 0))->getZExtValue();
1561 MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1562 }
1563
1564 Info.memVT = memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(),
1565 Ty: CI.getType(), MaxNumLanes);
1566 } else {
1567 Info.memVT =
1568 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1569 MaxNumLanes: std::numeric_limits<unsigned>::max());
1570 }
1571
1572 // FIXME: What does alignment mean for an image?
1573 Info.opc = ISD::INTRINSIC_W_CHAIN;
1574 Info.flags = Flags | MachineMemOperand::MOLoad;
1575 } else if (ME.onlyWritesMemory()) {
1576 Info.opc = ISD::INTRINSIC_VOID;
1577
1578 Type *DataTy = CI.getArgOperand(i: 0)->getType();
1579 if (RsrcIntr->IsImage) {
1580 unsigned DMask = cast<ConstantInt>(Val: CI.getArgOperand(i: 1))->getZExtValue();
1581 unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1582 Info.memVT = memVTFromLoadIntrData(TLI: *this, DL: MF.getDataLayout(), Ty: DataTy,
1583 MaxNumLanes: DMaskLanes);
1584 } else
1585 Info.memVT = getValueType(DL: MF.getDataLayout(), Ty: DataTy);
1586
1587 Info.flags = Flags | MachineMemOperand::MOStore;
1588 } else {
1589 // Atomic, NoReturn Sampler or prefetch
1590 Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
1591 : ISD::INTRINSIC_W_CHAIN;
1592
1593 switch (IntrID) {
1594 default:
1595 Info.flags = Flags | MachineMemOperand::MOLoad;
1596 if (!IsSPrefetch)
1597 Info.flags |= MachineMemOperand::MOStore;
1598
1599 if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
1600 // Fake memory access type for no return sampler intrinsics
1601 Info.memVT = MVT::i32;
1602 } else {
1603 // XXX - Should this be volatile without known ordering?
1604 Info.flags |= MachineMemOperand::MOVolatile;
1605 Info.memVT = MVT::getVT(Ty: CI.getArgOperand(i: 0)->getType());
1606 }
1607 break;
1608 case Intrinsic::amdgcn_raw_buffer_load_lds:
1609 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
1610 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
1611 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
1612 case Intrinsic::amdgcn_struct_buffer_load_lds:
1613 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
1614 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
1615 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
1616 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1617
1618 // Entry 0: Load from buffer.
1619 // Don't set an offset, since the pointer value always represents the
1620 // base of the buffer.
1621 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1622 Info.flags = Flags | MachineMemOperand::MOLoad;
1623 Infos.push_back(Elt: Info);
1624
1625 // Entry 1: Store to LDS.
1626 // Instruction offset is applied, and an additional per-lane offset
1627 // which we simulate using a larger memory type.
1628 Info.memVT = EVT::getIntegerVT(
1629 Context&: CI.getContext(), BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1630 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1631 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 2))
1632 ->getZExtValue();
1633 Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
1634 Info.flags = Flags | MachineMemOperand::MOStore;
1635 Infos.push_back(Elt: Info);
1636 return;
1637 }
1638 case Intrinsic::amdgcn_raw_atomic_buffer_load:
1639 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
1640 case Intrinsic::amdgcn_struct_atomic_buffer_load:
1641 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
1642 Info.memVT =
1643 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1644 MaxNumLanes: std::numeric_limits<unsigned>::max());
1645 Info.flags = Flags | MachineMemOperand::MOLoad;
1646 Infos.push_back(Elt: Info);
1647 return;
1648 }
1649 }
1650 }
1651 Infos.push_back(Elt: Info);
1652 return;
1653 }
1654
1655 IntrinsicInfo Info;
1656 switch (IntrID) {
1657 case Intrinsic::amdgcn_ds_ordered_add:
1658 case Intrinsic::amdgcn_ds_ordered_swap: {
1659 Info.opc = ISD::INTRINSIC_W_CHAIN;
1660 Info.memVT = MVT::getVT(Ty: CI.getType());
1661 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1662 Info.align.reset();
1663 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1664
1665 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 4));
1666 if (!Vol->isZero())
1667 Info.flags |= MachineMemOperand::MOVolatile;
1668
1669 Infos.push_back(Elt: Info);
1670 return;
1671 }
1672 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1673 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1674 Info.opc = ISD::INTRINSIC_W_CHAIN;
1675 Info.memVT = MVT::getVT(Ty: CI.getOperand(i_nocapture: 0)->getType());
1676 Info.ptrVal = nullptr;
1677 Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
1678 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1679 Infos.push_back(Elt: Info);
1680 return;
1681 }
1682 case Intrinsic::amdgcn_ds_append:
1683 case Intrinsic::amdgcn_ds_consume: {
1684 Info.opc = ISD::INTRINSIC_W_CHAIN;
1685 Info.memVT = MVT::getVT(Ty: CI.getType());
1686 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1687 Info.align.reset();
1688 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1689
1690 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
1691 if (!Vol->isZero())
1692 Info.flags |= MachineMemOperand::MOVolatile;
1693
1694 Infos.push_back(Elt: Info);
1695 return;
1696 }
1697 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1698 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
1699 Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
1700 ? ISD::INTRINSIC_W_CHAIN
1701 : ISD::INTRINSIC_VOID;
1702 Info.memVT = MVT::getVT(Ty: CI.getType());
1703 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1704 Info.memVT = MVT::i64;
1705 Info.size = 8;
1706 Info.align.reset();
1707 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1708 Info.order = AtomicOrdering::Monotonic;
1709 Infos.push_back(Elt: Info);
1710 return;
1711 }
1712 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
1713 case Intrinsic::amdgcn_image_bvh_intersect_ray:
1714 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
1715 Info.opc = ISD::INTRINSIC_W_CHAIN;
1716 Info.memVT =
1717 MVT::getVT(Ty: IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
1718 ? CI.getType()
1719 : cast<StructType>(Val: CI.getType())
1720 ->getElementType(N: 0)); // XXX: what is correct VT?
1721
1722 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1723 Info.align.reset();
1724 Info.flags = Flags | MachineMemOperand::MOLoad |
1725 MachineMemOperand::MODereferenceable;
1726 Infos.push_back(Elt: Info);
1727 return;
1728 }
1729 case Intrinsic::amdgcn_global_atomic_fmin_num:
1730 case Intrinsic::amdgcn_global_atomic_fmax_num:
1731 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1732 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1733 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
1734 Info.opc = ISD::INTRINSIC_W_CHAIN;
1735 Info.memVT = MVT::getVT(Ty: CI.getType());
1736 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1737 Info.align.reset();
1738 Info.flags =
1739 Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1740 MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile;
1741 Infos.push_back(Elt: Info);
1742 return;
1743 }
1744 case Intrinsic::amdgcn_cluster_load_b32:
1745 case Intrinsic::amdgcn_cluster_load_b64:
1746 case Intrinsic::amdgcn_cluster_load_b128:
1747 case Intrinsic::amdgcn_ds_load_tr6_b96:
1748 case Intrinsic::amdgcn_ds_load_tr4_b64:
1749 case Intrinsic::amdgcn_ds_load_tr8_b64:
1750 case Intrinsic::amdgcn_ds_load_tr16_b128:
1751 case Intrinsic::amdgcn_global_load_tr6_b96:
1752 case Intrinsic::amdgcn_global_load_tr4_b64:
1753 case Intrinsic::amdgcn_global_load_tr_b64:
1754 case Intrinsic::amdgcn_global_load_tr_b128:
1755 case Intrinsic::amdgcn_ds_read_tr4_b64:
1756 case Intrinsic::amdgcn_ds_read_tr6_b96:
1757 case Intrinsic::amdgcn_ds_read_tr8_b64:
1758 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1759 Info.opc = ISD::INTRINSIC_W_CHAIN;
1760 Info.memVT = MVT::getVT(Ty: CI.getType());
1761 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1762 Info.align.reset();
1763 Info.flags = Flags | MachineMemOperand::MOLoad;
1764 Infos.push_back(Elt: Info);
1765 return;
1766 }
1767 case Intrinsic::amdgcn_flat_load_monitor_b32:
1768 case Intrinsic::amdgcn_flat_load_monitor_b64:
1769 case Intrinsic::amdgcn_flat_load_monitor_b128:
1770 case Intrinsic::amdgcn_global_load_monitor_b32:
1771 case Intrinsic::amdgcn_global_load_monitor_b64:
1772 case Intrinsic::amdgcn_global_load_monitor_b128: {
1773 Info.opc = ISD::INTRINSIC_W_CHAIN;
1774 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1775 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1776 Info.align.reset();
1777 Info.flags = MachineMemOperand::MOLoad;
1778 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1779 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1780 Infos.push_back(Elt: Info);
1781 return;
1782 }
1783 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1784 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1785 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
1786 Info.opc = ISD::INTRINSIC_W_CHAIN;
1787 Info.memVT = MVT::getVT(Ty: CI.getType());
1788 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1789 Info.align.reset();
1790 Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
1791 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1792 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1793 Infos.push_back(Elt: Info);
1794 return;
1795 }
1796 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1797 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1798 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
1799 Info.opc = ISD::INTRINSIC_VOID;
1800 Info.memVT = MVT::getVT(Ty: CI.getArgOperand(i: 1)->getType());
1801 Info.ptrVal = CI.getArgOperand(i: 0);
1802 Info.align.reset();
1803 Info.flags = (MachineMemOperand::MOStore | MOCooperative);
1804 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 2);
1805 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 3);
1806 Infos.push_back(Elt: Info);
1807 return;
1808 }
1809 case Intrinsic::amdgcn_ds_gws_init:
1810 case Intrinsic::amdgcn_ds_gws_barrier:
1811 case Intrinsic::amdgcn_ds_gws_sema_v:
1812 case Intrinsic::amdgcn_ds_gws_sema_br:
1813 case Intrinsic::amdgcn_ds_gws_sema_p:
1814 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1815 Info.opc = ISD::INTRINSIC_VOID;
1816
1817 const GCNTargetMachine &TM =
1818 static_cast<const GCNTargetMachine &>(getTargetMachine());
1819
1820 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1821 Info.ptrVal = MFI->getGWSPSV(TM);
1822
1823 // This is an abstract access, but we need to specify a type and size.
1824 Info.memVT = MVT::i32;
1825 Info.size = 4;
1826 Info.align = Align(4);
1827
1828 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1829 Info.flags = Flags | MachineMemOperand::MOLoad;
1830 else
1831 Info.flags = Flags | MachineMemOperand::MOStore;
1832 Infos.push_back(Elt: Info);
1833 return;
1834 }
1835 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1836 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1837 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1838 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1839 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1840 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1841 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1842 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
1843 // Entry 0: Load from source (global/flat).
1844 Info.opc = ISD::INTRINSIC_VOID;
1845 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1846 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1847 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1848 Info.flags = Flags | MachineMemOperand::MOLoad;
1849 Infos.push_back(Elt: Info);
1850
1851 // Entry 1: Store to LDS (same offset).
1852 Info.flags = Flags | MachineMemOperand::MOStore;
1853 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1854 Infos.push_back(Elt: Info);
1855 return;
1856 }
1857 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1858 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1859 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1860 case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
1861 // Entry 0: Load from LDS.
1862 Info.opc = ISD::INTRINSIC_VOID;
1863 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1864 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1865 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1866 Info.flags = Flags | MachineMemOperand::MOLoad;
1867 Infos.push_back(Elt: Info);
1868
1869 // Entry 1: Store to global (same offset).
1870 Info.flags = Flags | MachineMemOperand::MOStore;
1871 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1872 Infos.push_back(Elt: Info);
1873 return;
1874 }
1875 case Intrinsic::amdgcn_av_load_b128:
1876 case Intrinsic::amdgcn_av_store_b128: {
1877 bool IsStore = IntrID == Intrinsic::amdgcn_av_store_b128;
1878 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
1879 Info.memVT = MVT::v4i32;
1880 Info.ptrVal = CI.getArgOperand(i: 0);
1881 Info.align = Align(16);
1882 Info.flags |=
1883 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
1884 // Pretend to be atomic so that SIMemoryLegalizer::expandStore sets cache
1885 // flags appropriately.
1886 Info.order = AtomicOrdering::Monotonic;
1887
1888 LLVMContext &Ctx = CI.getContext();
1889 unsigned ScopeIdx = CI.arg_size() - 1;
1890 MDNode *ScopeMD = cast<MDNode>(
1891 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ScopeIdx))->getMetadata());
1892 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1893 Info.ssid = Ctx.getOrInsertSyncScopeID(SSN: Scope);
1894 Infos.push_back(Elt: Info);
1895 return;
1896 }
1897 case Intrinsic::amdgcn_load_to_lds:
1898 case Intrinsic::amdgcn_load_async_to_lds:
1899 case Intrinsic::amdgcn_global_load_lds:
1900 case Intrinsic::amdgcn_global_load_async_lds: {
1901 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1902 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1903 bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
1904 if (IsVolatile)
1905 Flags |= MachineMemOperand::MOVolatile;
1906
1907 // Entry 0: Load from source (global/flat).
1908 Info.opc = ISD::INTRINSIC_VOID;
1909 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1910 Info.ptrVal = CI.getArgOperand(i: 0); // Source pointer
1911 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 3))->getSExtValue();
1912 Info.flags = Flags | MachineMemOperand::MOLoad;
1913 Infos.push_back(Elt: Info);
1914
1915 // Entry 1: Store to LDS.
1916 // Same offset from the instruction, but an additional per-lane offset is
1917 // added. Represent that using a wider memory type.
1918 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(),
1919 BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1920 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1921 Info.flags = Flags | MachineMemOperand::MOStore;
1922 Infos.push_back(Elt: Info);
1923 return;
1924 }
1925 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
1926 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
1927 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
1928 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1929 Info.opc = ISD::INTRINSIC_W_CHAIN;
1930
1931 const GCNTargetMachine &TM =
1932 static_cast<const GCNTargetMachine &>(getTargetMachine());
1933
1934 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1935 Info.ptrVal = MFI->getGWSPSV(TM);
1936
1937 // This is an abstract access, but we need to specify a type and size.
1938 Info.memVT = MVT::i32;
1939 Info.size = 4;
1940 Info.align = Align(4);
1941
1942 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1943 Infos.push_back(Elt: Info);
1944 return;
1945 }
1946 case Intrinsic::amdgcn_s_prefetch_data:
1947 case Intrinsic::amdgcn_s_prefetch_inst:
1948 case Intrinsic::amdgcn_flat_prefetch:
1949 case Intrinsic::amdgcn_global_prefetch: {
1950 Info.opc = ISD::INTRINSIC_VOID;
1951 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: 8);
1952 Info.ptrVal = CI.getArgOperand(i: 0);
1953 Info.flags = Flags | MachineMemOperand::MOLoad;
1954 Infos.push_back(Elt: Info);
1955 return;
1956 }
1957 default:
1958 return;
1959 }
1960}
1961
1962bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
1963 SmallVectorImpl<Value *> &Ops,
1964 Type *&AccessTy) const {
1965 Value *Ptr = nullptr;
1966 switch (II->getIntrinsicID()) {
1967 case Intrinsic::amdgcn_cluster_load_b128:
1968 case Intrinsic::amdgcn_cluster_load_b64:
1969 case Intrinsic::amdgcn_cluster_load_b32:
1970 case Intrinsic::amdgcn_ds_append:
1971 case Intrinsic::amdgcn_ds_consume:
1972 case Intrinsic::amdgcn_ds_load_tr8_b64:
1973 case Intrinsic::amdgcn_ds_load_tr16_b128:
1974 case Intrinsic::amdgcn_ds_load_tr4_b64:
1975 case Intrinsic::amdgcn_ds_load_tr6_b96:
1976 case Intrinsic::amdgcn_ds_read_tr4_b64:
1977 case Intrinsic::amdgcn_ds_read_tr6_b96:
1978 case Intrinsic::amdgcn_ds_read_tr8_b64:
1979 case Intrinsic::amdgcn_ds_read_tr16_b64:
1980 case Intrinsic::amdgcn_ds_ordered_add:
1981 case Intrinsic::amdgcn_ds_ordered_swap:
1982 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1983 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
1984 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1985 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1986 case Intrinsic::amdgcn_global_atomic_fmax_num:
1987 case Intrinsic::amdgcn_global_atomic_fmin_num:
1988 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1989 case Intrinsic::amdgcn_global_load_tr_b64:
1990 case Intrinsic::amdgcn_global_load_tr_b128:
1991 case Intrinsic::amdgcn_global_load_tr4_b64:
1992 case Intrinsic::amdgcn_global_load_tr6_b96:
1993 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1994 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1995 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1996 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1997 case Intrinsic::amdgcn_av_load_b128:
1998 case Intrinsic::amdgcn_av_store_b128:
1999 Ptr = II->getArgOperand(i: 0);
2000 break;
2001 case Intrinsic::amdgcn_load_to_lds:
2002 case Intrinsic::amdgcn_load_async_to_lds:
2003 case Intrinsic::amdgcn_global_load_lds:
2004 case Intrinsic::amdgcn_global_load_async_lds:
2005 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
2006 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
2007 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
2008 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
2009 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
2010 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
2011 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
2012 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
2013 Ptr = II->getArgOperand(i: 1);
2014 break;
2015 default:
2016 return false;
2017 }
2018 AccessTy = II->getType();
2019 Ops.push_back(Elt: Ptr);
2020 return true;
2021}
2022
2023bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM,
2024 unsigned AddrSpace) const {
2025 if (!Subtarget->hasFlatInstOffsets()) {
2026 // Flat instructions do not have offsets, and only have the register
2027 // address.
2028 return AM.BaseOffs == 0 && AM.Scale == 0;
2029 }
2030
2031 using AMDGPU::FlatAddrSpace;
2032 FlatAddrSpace FlatVariant =
2033 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ? FlatAddrSpace::FlatGlobal
2034 : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? FlatAddrSpace::FlatScratch
2035 : FlatAddrSpace::FLAT;
2036
2037 return AM.Scale == 0 &&
2038 (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
2039 Offset: AM.BaseOffs, AddrSpace, FlatVariant));
2040}
2041
2042bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
2043 if (Subtarget->hasFlatGlobalInsts())
2044 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS);
2045
2046 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
2047 // Assume the we will use FLAT for all global memory accesses
2048 // on VI.
2049 // FIXME: This assumption is currently wrong. On VI we still use
2050 // MUBUF instructions for the r + i addressing mode. As currently
2051 // implemented, the MUBUF instructions only work on buffer < 4GB.
2052 // It may be possible to support > 4GB buffers with MUBUF instructions,
2053 // by setting the stride value in the resource descriptor which would
2054 // increase the size limit to (stride * 4GB). However, this is risky,
2055 // because it has never been validated.
2056 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2057 }
2058
2059 return isLegalMUBUFAddressingMode(AM);
2060}
2061
2062bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
2063 // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
2064 // additionally can do r + r + i with addr64. 32-bit has more addressing
2065 // mode options. Depending on the resource constant, it can also do
2066 // (i64 r0) + (i32 r1) * (i14 i).
2067 //
2068 // Private arrays end up using a scratch buffer most of the time, so also
2069 // assume those use MUBUF instructions. Scratch loads / stores are currently
2070 // implemented as mubuf instructions with offen bit set, so slightly
2071 // different than the normal addr64.
2072 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2073 if (!TII->isLegalMUBUFImmOffset(Imm: AM.BaseOffs))
2074 return false;
2075
2076 // FIXME: Since we can split immediate into soffset and immediate offset,
2077 // would it make sense to allow any immediate?
2078
2079 switch (AM.Scale) {
2080 case 0: // r + i or just i, depending on HasBaseReg.
2081 return true;
2082 case 1:
2083 return true; // We have r + r or r + i.
2084 case 2:
2085 if (AM.HasBaseReg) {
2086 // Reject 2 * r + r.
2087 return false;
2088 }
2089
2090 // Allow 2 * r as r + r
2091 // Or 2 * r + i is allowed as r + r + i.
2092 return true;
2093 default: // Don't allow n * r
2094 return false;
2095 }
2096}
2097
2098bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
2099 const AddrMode &AM, Type *Ty,
2100 unsigned AS,
2101 Instruction *I) const {
2102 // No global is ever allowed as a base.
2103 if (AM.BaseGV)
2104 return false;
2105
2106 if (AS == AMDGPUAS::GLOBAL_ADDRESS)
2107 return isLegalGlobalAddressingMode(AM);
2108
2109 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
2110 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
2111 AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
2112 AS == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2113 // If the offset isn't a multiple of 4, it probably isn't going to be
2114 // correctly aligned.
2115 // FIXME: Can we get the real alignment here?
2116 if (AM.BaseOffs % 4 != 0)
2117 return isLegalMUBUFAddressingMode(AM);
2118
2119 if (!Subtarget->hasScalarSubwordLoads()) {
2120 // There are no SMRD extloads, so if we have to do a small type access we
2121 // will use a MUBUF load.
2122 // FIXME?: We also need to do this if unaligned, but we don't know the
2123 // alignment here.
2124 if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
2125 return isLegalGlobalAddressingMode(AM);
2126 }
2127
2128 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
2129 // SMRD instructions have an 8-bit, dword offset on SI.
2130 if (!isUInt<8>(x: AM.BaseOffs / 4))
2131 return false;
2132 } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
2133 // On CI+, this can also be a 32-bit literal constant offset. If it fits
2134 // in 8-bits, it can use a smaller encoding.
2135 if (!isUInt<32>(x: AM.BaseOffs / 4))
2136 return false;
2137 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
2138 // On VI, these use the SMEM format and the offset is 20-bit in bytes.
2139 if (!isUInt<20>(x: AM.BaseOffs))
2140 return false;
2141 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
2142 // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
2143 // for S_BUFFER_* instructions).
2144 if (!isInt<21>(x: AM.BaseOffs))
2145 return false;
2146 } else {
2147 // On GFX12, all offsets are signed 24-bit in bytes.
2148 if (!isInt<24>(x: AM.BaseOffs))
2149 return false;
2150 }
2151
2152 if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
2153 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
2154 AM.BaseOffs < 0) {
2155 // Scalar (non-buffer) loads can only use a negative offset if
2156 // soffset+offset is non-negative. Since the compiler can only prove that
2157 // in a few special cases, it is safer to claim that negative offsets are
2158 // not supported.
2159 return false;
2160 }
2161
2162 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2163 return true;
2164
2165 if (AM.Scale == 1 && AM.HasBaseReg)
2166 return true;
2167
2168 return false;
2169 }
2170
2171 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
2172 return Subtarget->hasFlatScratchEnabled()
2173 ? isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS)
2174 : isLegalMUBUFAddressingMode(AM);
2175
2176 if (AS == AMDGPUAS::LOCAL_ADDRESS ||
2177 (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
2178 // Basic, single offset DS instructions allow a 16-bit unsigned immediate
2179 // field.
2180 // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
2181 // an 8-bit dword offset but we don't know the alignment here.
2182 if (!isUInt<16>(x: AM.BaseOffs))
2183 return false;
2184
2185 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2186 return true;
2187
2188 if (AM.Scale == 1 && AM.HasBaseReg)
2189 return true;
2190
2191 return false;
2192 }
2193
2194 if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
2195 // For an unknown address space, this usually means that this is for some
2196 // reason being used for pure arithmetic, and not based on some addressing
2197 // computation. We don't have instructions that compute pointers with any
2198 // addressing modes, so treat them as having no offset like flat
2199 // instructions.
2200 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2201 }
2202
2203 // Assume a user alias of global for unknown address spaces.
2204 return isLegalGlobalAddressingMode(AM);
2205}
2206
2207bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
2208 const MachineFunction &MF) const {
2209 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
2210 return (MemVT.getSizeInBits() <= 4 * 32);
2211 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
2212 unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
2213 return (MemVT.getSizeInBits() <= MaxPrivateBits);
2214 }
2215 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
2216 return (MemVT.getSizeInBits() <= 2 * 32);
2217 return true;
2218}
2219
2220bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
2221 unsigned Size, unsigned AddrSpace, Align Alignment,
2222 MachineMemOperand::Flags Flags, unsigned *IsFast) const {
2223 if (IsFast)
2224 *IsFast = 0;
2225
2226 if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
2227 AddrSpace == AMDGPUAS::REGION_ADDRESS) {
2228 // Check if alignment requirements for ds_read/write instructions are
2229 // disabled.
2230 if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
2231 return false;
2232
2233 Align RequiredAlignment(
2234 PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))); // Natural alignment.
2235 if (Subtarget->hasLDSMisalignedBugInWGPMode() && Size > 32 &&
2236 Alignment < RequiredAlignment)
2237 return false;
2238
2239 // Either, the alignment requirements are "enabled", or there is an
2240 // unaligned LDS access related hardware bug though alignment requirements
2241 // are "disabled". In either case, we need to check for proper alignment
2242 // requirements.
2243 //
2244 switch (Size) {
2245 case 64:
2246 // SI has a hardware bug in the LDS / GDS bounds checking: if the base
2247 // address is negative, then the instruction is incorrectly treated as
2248 // out-of-bounds even if base + offsets is in bounds. Split vectorized
2249 // loads here to avoid emitting ds_read2_b32. We may re-combine the
2250 // load later in the SILoadStoreOptimizer.
2251 if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
2252 return false;
2253
2254 // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
2255 // can do a 4 byte aligned, 8 byte access in a single operation using
2256 // ds_read2/write2_b32 with adjacent offsets.
2257 RequiredAlignment = Align(4);
2258
2259 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2260 // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
2261 // ds_write2_b32 depending on the alignment. In either case with either
2262 // alignment there is no faster way of doing this.
2263
2264 // The numbers returned here and below are not additive, it is a 'speed
2265 // rank'. They are just meant to be compared to decide if a certain way
2266 // of lowering an operation is faster than another. For that purpose
2267 // naturally aligned operation gets it bitsize to indicate that "it
2268 // operates with a speed comparable to N-bit wide load". With the full
2269 // alignment ds128 is slower than ds96 for example. If underaligned it
2270 // is comparable to a speed of a single dword access, which would then
2271 // mean 32 < 128 and it is faster to issue a wide load regardless.
2272 // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
2273 // wider load which will not be aligned anymore the latter is slower.
2274 if (IsFast)
2275 *IsFast = (Alignment >= RequiredAlignment) ? 64
2276 : (Alignment < Align(4)) ? 32
2277 : 1;
2278 return true;
2279 }
2280
2281 break;
2282 case 96:
2283 if (!Subtarget->hasDS96AndDS128())
2284 return false;
2285
2286 // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
2287 // gfx8 and older.
2288
2289 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2290 // Naturally aligned access is fastest. However, also report it is Fast
2291 // if memory is aligned less than DWORD. A narrow load or store will be
2292 // be equally slow as a single ds_read_b96/ds_write_b96, but there will
2293 // be more of them, so overall we will pay less penalty issuing a single
2294 // instruction.
2295
2296 // See comment on the values above.
2297 if (IsFast)
2298 *IsFast = (Alignment >= RequiredAlignment) ? 96
2299 : (Alignment < Align(4)) ? 32
2300 : 1;
2301 return true;
2302 }
2303
2304 break;
2305 case 128:
2306 if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
2307 return false;
2308
2309 // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
2310 // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a
2311 // single operation using ds_read2/write2_b64.
2312 RequiredAlignment = Align(8);
2313
2314 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2315 // Naturally aligned access is fastest. However, also report it is Fast
2316 // if memory is aligned less than DWORD. A narrow load or store will be
2317 // be equally slow as a single ds_read_b128/ds_write_b128, but there
2318 // will be more of them, so overall we will pay less penalty issuing a
2319 // single instruction.
2320
2321 // See comment on the values above.
2322 if (IsFast)
2323 *IsFast = (Alignment >= RequiredAlignment) ? 128
2324 : (Alignment < Align(4)) ? 32
2325 : 1;
2326 return true;
2327 }
2328
2329 break;
2330 default:
2331 if (Size > 32)
2332 return false;
2333
2334 break;
2335 }
2336
2337 // See comment on the values above.
2338 // Note that we have a single-dword or sub-dword here, so if underaligned
2339 // it is a slowest possible access, hence returned value is 0.
2340 if (IsFast)
2341 *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
2342
2343 return Alignment >= RequiredAlignment ||
2344 Subtarget->hasUnalignedDSAccessEnabled();
2345 }
2346
2347 // FIXME: We have to be conservative here and assume that flat operations
2348 // will access scratch. If we had access to the IR function, then we
2349 // could determine if any private memory was used in the function.
2350 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
2351 AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
2352 bool AlignedBy4 = Alignment >= Align(4);
2353 if (Subtarget->hasUnalignedScratchAccessEnabled()) {
2354 if (IsFast)
2355 *IsFast = AlignedBy4 ? Size : 1;
2356 return true;
2357 }
2358
2359 if (IsFast)
2360 *IsFast = AlignedBy4;
2361
2362 return AlignedBy4;
2363 }
2364
2365 // So long as they are correct, wide global memory operations perform better
2366 // than multiple smaller memory ops -- even when misaligned
2367 if (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace)) {
2368 if (IsFast)
2369 *IsFast = Size;
2370
2371 return Alignment >= Align(4) ||
2372 Subtarget->hasUnalignedBufferAccessEnabled();
2373 }
2374
2375 // Ensure robust out-of-bounds guarantees for buffer accesses are met when the
2376 // "amdgpu.buffer.oob.mode" module flag has not enabled relaxed untyped-buffer
2377 // OOB semantics. Normally hardware will ensure proper
2378 // out-of-bounds behavior, but in the edge case where an access starts
2379 // out-of-bounds and then enters in-bounds, the entire access would be treated
2380 // as out-of-bounds. Prevent misaligned memory accesses by requiring the
2381 // natural alignment of buffer accesses.
2382 if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
2383 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
2384 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2385 if (!Subtarget->hasRelaxedBufferOOBMode() &&
2386 Alignment < Align(PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))))
2387 return false;
2388 }
2389
2390 // Smaller than dword value must be aligned.
2391 if (Size < 32)
2392 return false;
2393
2394 // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
2395 // byte-address are ignored, thus forcing Dword alignment.
2396 // This applies to private, global, and constant memory.
2397 if (IsFast)
2398 *IsFast = 1;
2399
2400 return Size >= 32 && Alignment >= Align(4);
2401}
2402
2403bool SITargetLowering::allowsMisalignedMemoryAccesses(
2404 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2405 unsigned *IsFast) const {
2406 return allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace,
2407 Alignment, Flags, IsFast);
2408}
2409
2410EVT SITargetLowering::getOptimalMemOpType(
2411 LLVMContext &Context, const MemOp &Op,
2412 const AttributeList &FuncAttributes) const {
2413 // FIXME: Should account for address space here.
2414
2415 // The default fallback uses the private pointer size as a guess for a type to
2416 // use. Make sure we switch these to 64-bit accesses.
2417
2418 if (Op.size() >= 16 &&
2419 Op.isDstAligned(AlignCheck: Align(4))) // XXX: Should only do for global
2420 return MVT::v4i32;
2421
2422 if (Op.size() >= 8 && Op.isDstAligned(AlignCheck: Align(4)))
2423 return MVT::v2i32;
2424
2425 // Use the default.
2426 return MVT::Other;
2427}
2428
2429bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
2430 const MemSDNode *MemNode = cast<MemSDNode>(Val: N);
2431 return MemNode->getMemOperand()->getFlags() & MONoClobber;
2432}
2433
2434bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
2435 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
2436 AS == AMDGPUAS::PRIVATE_ADDRESS;
2437}
2438
2439bool SITargetLowering::isFreeAddrSpaceCast(const DataLayout &DL, unsigned SrcAS,
2440 unsigned DestAS) const {
2441 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
2442 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2443 Subtarget->hasGloballyAddressableScratch()) {
2444 // Flat -> private requires subtracting src_flat_scratch_base_lo.
2445 return false;
2446 }
2447
2448 // Flat -> private/local is a simple truncate.
2449 // Flat -> global is no-op
2450 return true;
2451 }
2452
2453 const GCNTargetMachine &TM =
2454 static_cast<const GCNTargetMachine &>(getTargetMachine());
2455 return TM.isNoopAddrSpaceCast(DL, SrcAS, DestAS);
2456}
2457
2458TargetLoweringBase::LegalizeTypeAction
2459SITargetLowering::getPreferredVectorAction(MVT VT) const {
2460 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2461 VT.getScalarType().bitsLE(VT: MVT::i16))
2462 return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
2463 return TargetLoweringBase::getPreferredVectorAction(VT);
2464}
2465
2466bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2467 Type *Ty) const {
2468 // FIXME: Could be smarter if called for vector constants.
2469 return true;
2470}
2471
2472TargetLowering::ExtractSubvectorCost
2473SITargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2474 unsigned Index) const {
2475 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2476 return ExtractSubvectorCost::Expensive;
2477
2478 // TODO: Add more cases that are cheap.
2479 if (Index == 0)
2480 return ExtractSubvectorCost::Free;
2481 return ExtractSubvectorCost::Expensive;
2482}
2483
2484bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
2485 // TODO: This should be more aggressive, particular for 16-bit element
2486 // vectors. However there are some mixed improvements and regressions.
2487 EVT EltTy = VT.getVectorElementType();
2488 unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
2489 return EltTy.getSizeInBits() % MinAlign == 0;
2490}
2491
2492bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
2493 if (Subtarget->has16BitInsts() && VT == MVT::i16) {
2494 switch (Op) {
2495 case ISD::LOAD:
2496 case ISD::STORE:
2497 return true;
2498 default:
2499 return false;
2500 }
2501 }
2502
2503 // SimplifySetCC uses this function to determine whether or not it should
2504 // create setcc with i1 operands. We don't have instructions for i1 setcc.
2505 if (VT == MVT::i1 && Op == ISD::SETCC)
2506 return false;
2507
2508 if (VT.getScalarType() == MVT::i64 &&
2509 (Op == ISD::SINT_TO_FP || Op == ISD::UINT_TO_FP))
2510 return false;
2511
2512 return TargetLowering::isTypeDesirableForOp(Op, VT);
2513}
2514
2515bool SITargetLowering::isTypeDesirableForOp(SDNode *N, EVT VT) const {
2516 // Do not convert uniform loads to 16-bit.
2517 // Uniform 16-bit loads are legalized to i16 = trunc (zextload i16->i32)
2518 // to match subword load patterns. Allowing conversion back to a 16-bit
2519 // load would create an infinite loop.
2520 if (Subtarget->hasScalarSubwordLoads() && N->getOpcode() == ISD::LOAD &&
2521 !VT.isVector() && VT.getSizeInBits() == 16) {
2522 auto *Load = dyn_cast<LoadSDNode>(Val: N);
2523 if (Load && isUniformLoad(Load)) {
2524 return false;
2525 }
2526 }
2527
2528 return isTypeDesirableForOp(Op: N->getOpcode(), VT);
2529}
2530
2531bool SITargetLowering::isUniformLoad(const LoadSDNode *Load) const {
2532 const MachineMemOperand *MMO = Load->getMemOperand();
2533
2534 // FIXME: We ought to able able to take the direct isDivergent result. We
2535 // cannot rely on the MMO for a uniformity check, and should stop using
2536 // it. This is a hack for 2 ways that the IR divergence analysis is superior
2537 // to the DAG divergence: Recognizing shift-of-workitem-id as always
2538 // uniform, and isSingleLaneExecution. These should be handled in the DAG
2539 // version, and then this can be dropped.
2540 if (Load->isDivergent() && !AMDGPU::isUniformMMO(MMO))
2541 return false;
2542
2543 return MMO->getSize().hasValue() &&
2544 Load->getAlign() >=
2545 Align(std::min(a: MMO->getSize().getValue().getKnownMinValue(),
2546 b: uint64_t(4))) &&
2547 (MMO->isInvariant() ||
2548 (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2549 Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
2550 (Load->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2551 Load->isSimple() && isMemOpHasNoClobberedMemOperand(N: Load)));
2552}
2553
2554MachinePointerInfo
2555SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
2556 // This isn't really a constant pool but close enough.
2557 MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
2558 PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
2559 return PtrInfo;
2560}
2561
2562SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
2563 const SDLoc &SL,
2564 SDValue Chain,
2565 uint64_t Offset) const {
2566 const DataLayout &DL = DAG.getDataLayout();
2567 MachineFunction &MF = DAG.getMachineFunction();
2568 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2569 MVT PtrVT = getPointerTy(DL, AS: AMDGPUAS::CONSTANT_ADDRESS);
2570
2571 auto [InputPtrReg, RC, ArgTy] =
2572 Info->getPreloadedValue(Value: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
2573
2574 // We may not have the kernarg segment argument if we have no kernel
2575 // arguments.
2576 if (!InputPtrReg)
2577 return DAG.getConstant(Val: Offset, DL: SL, VT: PtrVT);
2578
2579 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
2580 SDValue BasePtr = DAG.getCopyFromReg(
2581 Chain, dl: SL, Reg: MRI.getLiveInVirtReg(PReg: InputPtrReg->getRegister()), VT: PtrVT);
2582
2583 return DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
2584}
2585
2586SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
2587 const SDLoc &SL) const {
2588 uint64_t Offset =
2589 getImplicitParameterOffset(MF: DAG.getMachineFunction(), Param: FIRST_IMPLICIT);
2590 return lowerKernArgParameterPtr(DAG, SL, Chain: DAG.getEntryNode(), Offset);
2591}
2592
2593SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
2594 const SDLoc &SL) const {
2595
2596 Function &F = DAG.getMachineFunction().getFunction();
2597 std::optional<uint32_t> KnownSize =
2598 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
2599 if (KnownSize.has_value())
2600 return DAG.getConstant(Val: *KnownSize, DL: SL, VT: MVT::i32);
2601 return SDValue();
2602}
2603
2604SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
2605 const SDLoc &SL, SDValue Val,
2606 bool Signed,
2607 const ISD::InputArg *Arg) const {
2608 // First, if it is a widened vector, narrow it.
2609 if (VT.isVector() &&
2610 VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
2611 EVT NarrowedVT =
2612 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(),
2613 NumElements: VT.getVectorNumElements());
2614 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: NarrowedVT, N1: Val,
2615 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
2616 }
2617
2618 // Then convert the vector elements or scalar value.
2619 if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && VT.bitsLT(VT: MemVT)) {
2620 unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
2621 Val = DAG.getNode(Opcode: Opc, DL: SL, VT: MemVT, N1: Val, N2: DAG.getValueType(VT));
2622 }
2623
2624 if (MemVT.isFloatingPoint()) {
2625 if (VT.isFloatingPoint()) {
2626 Val = getFPExtOrFPRound(DAG, Op: Val, DL: SL, VT);
2627 } else {
2628 assert(!MemVT.isVector());
2629 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
2630 SDValue Cast = DAG.getBitcast(VT: IntVT, V: Val);
2631 Val = DAG.getAnyExtOrTrunc(Op: Cast, DL: SL, VT);
2632 }
2633 } else if (Signed)
2634 Val = DAG.getSExtOrTrunc(Op: Val, DL: SL, VT);
2635 else
2636 Val = DAG.getZExtOrTrunc(Op: Val, DL: SL, VT);
2637
2638 return Val;
2639}
2640
2641SDValue SITargetLowering::lowerKernargMemParameter(
2642 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2643 uint64_t Offset, Align Alignment, bool Signed,
2644 const ISD::InputArg *Arg) const {
2645
2646 MachinePointerInfo PtrInfo =
2647 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
2648
2649 // Try to avoid using an extload by loading earlier than the argument address,
2650 // and extracting the relevant bits. The load should hopefully be merged with
2651 // the previous argument.
2652 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
2653 // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
2654 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
2655 int64_t OffsetDiff = Offset - AlignDownOffset;
2656
2657 EVT IntVT = MemVT.changeTypeToInteger();
2658
2659 // TODO: If we passed in the base kernel offset we could have a better
2660 // alignment than 4, but we don't really need it.
2661 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset: AlignDownOffset);
2662 SDValue Load = DAG.getLoad(VT: MVT::i32, dl: SL, Chain, Ptr,
2663 PtrInfo: PtrInfo.getWithOffset(O: AlignDownOffset), Alignment: Align(4),
2664 MMOFlags: MachineMemOperand::MODereferenceable |
2665 MachineMemOperand::MOInvariant);
2666
2667 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL: SL, VT: MVT::i32);
2668 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Load, N2: ShiftAmt);
2669
2670 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: IntVT, Operand: Extract);
2671 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MemVT, Operand: ArgVal);
2672 ArgVal = convertArgType(DAG, VT, MemVT, SL, Val: ArgVal, Signed, Arg);
2673
2674 return DAG.getMergeValues(Ops: {ArgVal, Load.getValue(R: 1)}, dl: SL);
2675 }
2676
2677 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2678 SDValue Load = DAG.getLoad(
2679 VT: MemVT, dl: SL, Chain, Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
2680 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2681
2682 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Val: Load, Signed, Arg);
2683 return DAG.getMergeValues(Ops: {Val, Load.getValue(R: 1)}, dl: SL);
2684}
2685
2686/// Coerce an argument which was passed in a different ABI type to the original
2687/// expected value type.
2688SDValue SITargetLowering::convertABITypeToValueType(SelectionDAG &DAG,
2689 SDValue Val,
2690 CCValAssign &VA,
2691 const SDLoc &SL) const {
2692 EVT ValVT = VA.getValVT();
2693
2694 // If this is an 8 or 16-bit value, it is really passed promoted
2695 // to 32 bits. Insert an assert[sz]ext to capture this, then
2696 // truncate to the right size.
2697 switch (VA.getLocInfo()) {
2698 case CCValAssign::Full:
2699 return Val;
2700 case CCValAssign::BCvt:
2701 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ValVT, Operand: Val);
2702 case CCValAssign::SExt:
2703 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: SL, VT: VA.getLocVT(), N1: Val,
2704 N2: DAG.getValueType(ValVT));
2705 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2706 case CCValAssign::ZExt:
2707 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: VA.getLocVT(), N1: Val,
2708 N2: DAG.getValueType(ValVT));
2709 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2710 case CCValAssign::AExt:
2711 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2712 default:
2713 llvm_unreachable("Unknown loc info!");
2714 }
2715}
2716
2717SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG,
2718 CCValAssign &VA, const SDLoc &SL,
2719 SDValue Chain,
2720 const ISD::InputArg &Arg) const {
2721 MachineFunction &MF = DAG.getMachineFunction();
2722 MachineFrameInfo &MFI = MF.getFrameInfo();
2723
2724 if (Arg.Flags.isByVal()) {
2725 unsigned Size = Arg.Flags.getByValSize();
2726 int FrameIdx = MFI.CreateFixedObject(Size, SPOffset: VA.getLocMemOffset(), IsImmutable: false);
2727 return DAG.getFrameIndex(FI: FrameIdx, VT: MVT::i32);
2728 }
2729
2730 unsigned ArgOffset = VA.getLocMemOffset();
2731 unsigned ArgSize = VA.getValVT().getStoreSize();
2732
2733 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: true);
2734
2735 // Create load nodes to retrieve arguments from the stack.
2736 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
2737
2738 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
2739 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
2740 MVT MemVT = VA.getValVT();
2741
2742 switch (VA.getLocInfo()) {
2743 default:
2744 break;
2745 case CCValAssign::BCvt:
2746 MemVT = VA.getLocVT();
2747 break;
2748 case CCValAssign::SExt:
2749 ExtType = ISD::SEXTLOAD;
2750 break;
2751 case CCValAssign::ZExt:
2752 ExtType = ISD::ZEXTLOAD;
2753 break;
2754 case CCValAssign::AExt:
2755 ExtType = ISD::EXTLOAD;
2756 break;
2757 }
2758
2759 SDValue ArgValue = DAG.getExtLoad(
2760 ExtType, dl: SL, VT: VA.getLocVT(), Chain, Ptr: FIN,
2761 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT);
2762
2763 SDValue ConvertedVal = convertABITypeToValueType(DAG, Val: ArgValue, VA, SL);
2764 if (ConvertedVal == ArgValue)
2765 return ConvertedVal;
2766
2767 return DAG.getMergeValues(Ops: {ConvertedVal, ArgValue.getValue(R: 1)}, dl: SL);
2768}
2769
2770SDValue SITargetLowering::lowerWorkGroupId(
2771 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2772 AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
2773 AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
2774 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
2775 if (!Subtarget->hasClusters())
2776 return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2777
2778 // Clusters are supported. Return the global position in the grid. If clusters
2779 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
2780
2781 // WorkGroupIdXYZ = ClusterId == 0 ?
2782 // ClusterIdXYZ :
2783 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
2784 SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2785 SDLoc SL(ClusterIdXYZ);
2786 SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
2787 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT);
2788 SDValue ClusterSizeXYZ = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterMaxIdXYZ, N2: One);
2789 SDValue ClusterWorkGroupIdXYZ =
2790 getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
2791 SDValue GlobalIdXYZ =
2792 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterWorkGroupIdXYZ,
2793 N2: DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: ClusterIdXYZ, N2: ClusterSizeXYZ));
2794
2795 switch (MFI.getClusterDims().getKind()) {
2796 case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
2797 case AMDGPU::ClusterDimsAttr::Kind::VariableDims:
2798 return GlobalIdXYZ;
2799 case AMDGPU::ClusterDimsAttr::Kind::NoCluster:
2800 return ClusterIdXYZ;
2801 case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
2802 using namespace AMDGPU::Hwreg;
2803 SDValue ClusterIdField =
2804 DAG.getTargetConstant(Val: HwregEncoding::encode(Values: ID_IB_STS2, Values: 6, Values: 4), DL: SL, VT);
2805 SDNode *GetReg =
2806 DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT, Op1: ClusterIdField);
2807 SDValue ClusterId(GetReg, 0);
2808 SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT);
2809 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SL, VT, N1: ClusterId, N2: Zero, N3: ClusterIdXYZ,
2810 N4: GlobalIdXYZ, N5: DAG.getCondCode(Cond: ISD::SETEQ));
2811 }
2812 }
2813
2814 llvm_unreachable("nothing should reach here");
2815}
2816
2817SDValue SITargetLowering::getPreloadedValue(
2818 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2819 AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
2820 const ArgDescriptor *Reg = nullptr;
2821 const TargetRegisterClass *RC = nullptr;
2822 LLT Ty;
2823
2824 CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
2825 const ArgDescriptor WorkGroupIDX =
2826 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP9);
2827 // If GridZ is not programmed in an entry function then the hardware will set
2828 // it to all zeros, so there is no need to mask the GridY value in the low
2829 // order bits.
2830 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
2831 Reg: AMDGPU::TTMP7,
2832 Mask: AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
2833 const ArgDescriptor WorkGroupIDZ =
2834 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP7, Mask: 0xFFFF0000u);
2835 const ArgDescriptor ClusterWorkGroupIDX =
2836 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000000Fu);
2837 const ArgDescriptor ClusterWorkGroupIDY =
2838 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000000F0u);
2839 const ArgDescriptor ClusterWorkGroupIDZ =
2840 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00000F00u);
2841 const ArgDescriptor ClusterWorkGroupMaxIDX =
2842 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000F000u);
2843 const ArgDescriptor ClusterWorkGroupMaxIDY =
2844 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000F0000u);
2845 const ArgDescriptor ClusterWorkGroupMaxIDZ =
2846 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00F00000u);
2847 const ArgDescriptor ClusterWorkGroupMaxFlatID =
2848 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0F000000u);
2849
2850 auto LoadConstant = [&](unsigned N) {
2851 return DAG.getConstant(Val: N, DL: SDLoc(), VT);
2852 };
2853
2854 if (Subtarget->hasArchitectedSGPRs() &&
2855 (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
2856 AMDGPU::ClusterDimsAttr ClusterDims = MFI.getClusterDims();
2857 bool HasFixedDims = ClusterDims.isFixedDims();
2858
2859 switch (PVID) {
2860 case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
2861 Reg = &WorkGroupIDX;
2862 RC = &AMDGPU::SReg_32RegClass;
2863 Ty = LLT::scalar(SizeInBits: 32);
2864 break;
2865 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
2866 Reg = &WorkGroupIDY;
2867 RC = &AMDGPU::SReg_32RegClass;
2868 Ty = LLT::scalar(SizeInBits: 32);
2869 break;
2870 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
2871 Reg = &WorkGroupIDZ;
2872 RC = &AMDGPU::SReg_32RegClass;
2873 Ty = LLT::scalar(SizeInBits: 32);
2874 break;
2875 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
2876 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
2877 return LoadConstant(0);
2878 Reg = &ClusterWorkGroupIDX;
2879 RC = &AMDGPU::SReg_32RegClass;
2880 Ty = LLT::scalar(SizeInBits: 32);
2881 break;
2882 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
2883 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
2884 return LoadConstant(0);
2885 Reg = &ClusterWorkGroupIDY;
2886 RC = &AMDGPU::SReg_32RegClass;
2887 Ty = LLT::scalar(SizeInBits: 32);
2888 break;
2889 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
2890 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
2891 return LoadConstant(0);
2892 Reg = &ClusterWorkGroupIDZ;
2893 RC = &AMDGPU::SReg_32RegClass;
2894 Ty = LLT::scalar(SizeInBits: 32);
2895 break;
2896 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
2897 if (HasFixedDims)
2898 return LoadConstant(ClusterDims.getDims()[0] - 1);
2899 Reg = &ClusterWorkGroupMaxIDX;
2900 RC = &AMDGPU::SReg_32RegClass;
2901 Ty = LLT::scalar(SizeInBits: 32);
2902 break;
2903 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
2904 if (HasFixedDims)
2905 return LoadConstant(ClusterDims.getDims()[1] - 1);
2906 Reg = &ClusterWorkGroupMaxIDY;
2907 RC = &AMDGPU::SReg_32RegClass;
2908 Ty = LLT::scalar(SizeInBits: 32);
2909 break;
2910 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
2911 if (HasFixedDims)
2912 return LoadConstant(ClusterDims.getDims()[2] - 1);
2913 Reg = &ClusterWorkGroupMaxIDZ;
2914 RC = &AMDGPU::SReg_32RegClass;
2915 Ty = LLT::scalar(SizeInBits: 32);
2916 break;
2917 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
2918 Reg = &ClusterWorkGroupMaxFlatID;
2919 RC = &AMDGPU::SReg_32RegClass;
2920 Ty = LLT::scalar(SizeInBits: 32);
2921 break;
2922 default:
2923 break;
2924 }
2925 }
2926
2927 if (!Reg)
2928 std::tie(args&: Reg, args&: RC, args&: Ty) = MFI.getPreloadedValue(Value: PVID);
2929 if (!Reg) {
2930 if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
2931 // It's possible for a kernarg intrinsic call to appear in a kernel with
2932 // no allocated segment, in which case we do not add the user sgpr
2933 // argument, so just return null.
2934 return DAG.getConstant(Val: 0, DL: SDLoc(), VT);
2935 }
2936
2937 // It's undefined behavior if a function marked with the amdgpu-no-*
2938 // attributes uses the corresponding intrinsic.
2939 return DAG.getPOISON(VT);
2940 }
2941
2942 return loadInputValue(DAG, RC, VT, SL: SDLoc(DAG.getEntryNode()), Arg: *Reg);
2943}
2944
2945static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
2946 CallingConv::ID CallConv,
2947 ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
2948 FunctionType *FType,
2949 SIMachineFunctionInfo *Info) {
2950 for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
2951 const ISD::InputArg *Arg = &Ins[I];
2952
2953 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2954 "vector type argument should have been split");
2955
2956 // First check if it's a PS input addr.
2957 if (CallConv == CallingConv::AMDGPU_PS && !Arg->Flags.isInReg() &&
2958 PSInputNum <= 15) {
2959 bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(Index: PSInputNum);
2960
2961 // Inconveniently only the first part of the split is marked as isSplit,
2962 // so skip to the end. We only want to increment PSInputNum once for the
2963 // entire split argument.
2964 if (Arg->Flags.isSplit()) {
2965 while (!Arg->Flags.isSplitEnd()) {
2966 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2967 "unexpected vector split in ps argument type");
2968 if (!SkipArg)
2969 Splits.push_back(Elt: *Arg);
2970 Arg = &Ins[++I];
2971 }
2972 }
2973
2974 if (SkipArg) {
2975 // We can safely skip PS inputs.
2976 Skipped.set(Arg->getOrigArgIndex());
2977 ++PSInputNum;
2978 continue;
2979 }
2980
2981 Info->markPSInputAllocated(Index: PSInputNum);
2982 if (Arg->Used)
2983 Info->markPSInputEnabled(Index: PSInputNum);
2984
2985 ++PSInputNum;
2986 }
2987
2988 Splits.push_back(Elt: *Arg);
2989 }
2990}
2991
2992// Allocate special inputs passed in VGPRs.
2993void SITargetLowering::allocateSpecialEntryInputVGPRs(
2994 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2995 SIMachineFunctionInfo &Info) const {
2996 const LLT I32 = LLT::integer(SizeInBits: 32);
2997 MachineRegisterInfo &MRI = MF.getRegInfo();
2998
2999 if (Info.hasWorkItemIDX()) {
3000 Register Reg = AMDGPU::VGPR0;
3001 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
3002
3003 CCInfo.AllocateReg(Reg);
3004 unsigned Mask =
3005 (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
3006 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3007 }
3008
3009 if (Info.hasWorkItemIDY()) {
3010 assert(Info.hasWorkItemIDX());
3011 if (Subtarget->hasPackedTID()) {
3012 Info.setWorkItemIDY(
3013 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 10));
3014 } else {
3015 unsigned Reg = AMDGPU::VGPR1;
3016 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
3017
3018 CCInfo.AllocateReg(Reg);
3019 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
3020 }
3021 }
3022
3023 if (Info.hasWorkItemIDZ()) {
3024 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
3025 if (Subtarget->hasPackedTID()) {
3026 Info.setWorkItemIDZ(
3027 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 20));
3028 } else {
3029 unsigned Reg = AMDGPU::VGPR2;
3030 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
3031
3032 CCInfo.AllocateReg(Reg);
3033 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
3034 }
3035 }
3036}
3037
3038static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
3039 const TargetRegisterClass *RC,
3040 unsigned NumArgRegs) {
3041 ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
3042 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgSGPRs);
3043 if (RegIdx == ArgSGPRs.size())
3044 report_fatal_error(reason: "ran out of SGPRs for arguments");
3045
3046 unsigned Reg = ArgSGPRs[RegIdx];
3047 Reg = CCInfo.AllocateReg(Reg);
3048 assert(Reg != AMDGPU::NoRegister);
3049
3050 MachineFunction &MF = CCInfo.getMachineFunction();
3051 MF.addLiveIn(PReg: Reg, RC);
3052 return ArgDescriptor::createRegister(Reg);
3053}
3054
3055// If this has a fixed position, we still should allocate the register in the
3056// CCInfo state. Technically we could get away with this for values passed
3057// outside of the normal argument range.
3058static void allocateFixedSGPRInputImpl(CCState &CCInfo,
3059 const TargetRegisterClass *RC,
3060 MCRegister Reg) {
3061 Reg = CCInfo.AllocateReg(Reg);
3062 assert(Reg != AMDGPU::NoRegister);
3063 MachineFunction &MF = CCInfo.getMachineFunction();
3064 MF.addLiveIn(PReg: Reg, RC);
3065}
3066
3067static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
3068 if (Arg) {
3069 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass,
3070 Reg: Arg.getRegister());
3071 } else
3072 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass, NumArgRegs: 32);
3073}
3074
3075static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
3076 if (Arg) {
3077 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass,
3078 Reg: Arg.getRegister());
3079 } else
3080 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass, NumArgRegs: 16);
3081}
3082
3083/// Allocate implicit function VGPR arguments in fixed registers.
3084void SITargetLowering::allocateSpecialInputVGPRsFixed(
3085 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3086 SIMachineFunctionInfo &Info) const {
3087 Register Reg = CCInfo.AllocateReg(Reg: AMDGPU::VGPR31);
3088 if (!Reg)
3089 report_fatal_error(reason: "failed to allocate VGPR for implicit arguments");
3090
3091 const unsigned Mask = 0x3ff;
3092 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3093 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask: Mask << 10));
3094 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask: Mask << 20));
3095}
3096
3097void SITargetLowering::allocateSpecialInputSGPRs(
3098 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3099 SIMachineFunctionInfo &Info) const {
3100 auto &ArgInfo = Info.getArgInfo();
3101 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3102
3103 // TODO: Unify handling with private memory pointers.
3104 if (UserSGPRInfo.hasDispatchPtr())
3105 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchPtr);
3106
3107 if (UserSGPRInfo.hasQueuePtr())
3108 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.QueuePtr);
3109
3110 // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
3111 // constant offset from the kernarg segment.
3112 if (Info.hasImplicitArgPtr())
3113 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.ImplicitArgPtr);
3114
3115 if (UserSGPRInfo.hasDispatchID())
3116 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchID);
3117
3118 // flat_scratch_init is not applicable for non-kernel functions.
3119
3120 if (Info.hasWorkGroupIDX())
3121 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDX);
3122
3123 if (Info.hasWorkGroupIDY())
3124 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDY);
3125
3126 if (Info.hasWorkGroupIDZ())
3127 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDZ);
3128
3129 if (Info.hasLDSKernelId())
3130 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.LDSKernelId);
3131}
3132
3133// Allocate special inputs passed in user SGPRs.
3134void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
3135 MachineFunction &MF,
3136 const SIRegisterInfo &TRI,
3137 SIMachineFunctionInfo &Info) const {
3138 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3139 if (UserSGPRInfo.hasImplicitBufferPtr()) {
3140 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
3141 MF.addLiveIn(PReg: ImplicitBufferPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3142 CCInfo.AllocateReg(Reg: ImplicitBufferPtrReg);
3143 }
3144
3145 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
3146 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
3147 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
3148 MF.addLiveIn(PReg: PrivateSegmentBufferReg, RC: &AMDGPU::SGPR_128RegClass);
3149 CCInfo.AllocateReg(Reg: PrivateSegmentBufferReg);
3150 }
3151
3152 if (UserSGPRInfo.hasDispatchPtr()) {
3153 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
3154 MF.addLiveIn(PReg: DispatchPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3155 CCInfo.AllocateReg(Reg: DispatchPtrReg);
3156 }
3157
3158 if (UserSGPRInfo.hasQueuePtr()) {
3159 Register QueuePtrReg = Info.addQueuePtr(TRI);
3160 MF.addLiveIn(PReg: QueuePtrReg, RC: &AMDGPU::SGPR_64RegClass);
3161 CCInfo.AllocateReg(Reg: QueuePtrReg);
3162 }
3163
3164 if (UserSGPRInfo.hasKernargSegmentPtr()) {
3165 MachineRegisterInfo &MRI = MF.getRegInfo();
3166 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
3167 CCInfo.AllocateReg(Reg: InputPtrReg);
3168
3169 Register VReg = MF.addLiveIn(PReg: InputPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3170 MRI.setType(VReg, Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
3171 }
3172
3173 if (UserSGPRInfo.hasDispatchID()) {
3174 Register DispatchIDReg = Info.addDispatchID(TRI);
3175 MF.addLiveIn(PReg: DispatchIDReg, RC: &AMDGPU::SGPR_64RegClass);
3176 CCInfo.AllocateReg(Reg: DispatchIDReg);
3177 }
3178
3179 if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
3180 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
3181 MF.addLiveIn(PReg: FlatScratchInitReg, RC: &AMDGPU::SGPR_64RegClass);
3182 CCInfo.AllocateReg(Reg: FlatScratchInitReg);
3183 }
3184
3185 if (UserSGPRInfo.hasPrivateSegmentSize()) {
3186 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
3187 MF.addLiveIn(PReg: PrivateSegmentSizeReg, RC: &AMDGPU::SGPR_32RegClass);
3188 CCInfo.AllocateReg(Reg: PrivateSegmentSizeReg);
3189 }
3190
3191 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
3192 // these from the dispatch pointer.
3193}
3194
3195// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
3196// sequential starting from the first argument.
3197void SITargetLowering::allocatePreloadKernArgSGPRs(
3198 CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
3199 const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
3200 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
3201 Function &F = MF.getFunction();
3202 unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
3203 GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
3204 bool InPreloadSequence = true;
3205 unsigned InIdx = 0;
3206 bool AlignedForImplictArgs = false;
3207 unsigned ImplicitArgOffset = 0;
3208 for (auto &Arg : F.args()) {
3209 if (!InPreloadSequence || !Arg.hasInRegAttr())
3210 break;
3211
3212 unsigned ArgIdx = Arg.getArgNo();
3213 // Don't preload non-original args or parts not in the current preload
3214 // sequence.
3215 if (InIdx < Ins.size() &&
3216 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
3217 break;
3218
3219 for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
3220 Ins[InIdx].getOrigArgIndex() == ArgIdx;
3221 InIdx++) {
3222 assert(ArgLocs[ArgIdx].isMemLoc());
3223 auto &ArgLoc = ArgLocs[InIdx];
3224 const Align KernelArgBaseAlign = Align(16);
3225 unsigned ArgOffset = ArgLoc.getLocMemOffset();
3226 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset: ArgOffset);
3227 unsigned NumAllocSGPRs =
3228 alignTo(Value: ArgLoc.getLocVT().getFixedSizeInBits(), Align: 32) / 32;
3229
3230 // Fix alignment for hidden arguments.
3231 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument")) {
3232 if (!AlignedForImplictArgs) {
3233 ImplicitArgOffset =
3234 alignTo(Size: LastExplicitArgOffset,
3235 A: Subtarget->getAlignmentForImplicitArgPtr()) -
3236 LastExplicitArgOffset;
3237 AlignedForImplictArgs = true;
3238 }
3239 ArgOffset += ImplicitArgOffset;
3240 }
3241
3242 // Arg is preloaded into the previous SGPR.
3243 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
3244 assert(InIdx >= 1 && "No previous SGPR");
3245 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
3246 Elt: Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
3247 continue;
3248 }
3249
3250 unsigned Padding = ArgOffset - LastExplicitArgOffset;
3251 unsigned PaddingSGPRs = alignTo(Value: Padding, Align: 4) / 4;
3252 // Check for free user SGPRs for preloading.
3253 if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
3254 InPreloadSequence = false;
3255 break;
3256 }
3257
3258 // Preload this argument.
3259 const TargetRegisterClass *RC =
3260 TRI.getSGPRClassForBitWidth(BitWidth: NumAllocSGPRs * 32);
3261 SmallVectorImpl<MCRegister> *PreloadRegs =
3262 Info.addPreloadedKernArg(TRI, RC, AllocSizeDWord: NumAllocSGPRs, KernArgIdx: InIdx, PaddingSGPRs);
3263
3264 if (PreloadRegs->size() > 1)
3265 RC = &AMDGPU::SGPR_32RegClass;
3266 for (auto &Reg : *PreloadRegs) {
3267 assert(Reg);
3268 MF.addLiveIn(PReg: Reg, RC);
3269 CCInfo.AllocateReg(Reg);
3270 }
3271
3272 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
3273 }
3274 }
3275}
3276
3277void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
3278 const SIRegisterInfo &TRI,
3279 SIMachineFunctionInfo &Info) const {
3280 // Always allocate this last since it is a synthetic preload.
3281 if (Info.hasLDSKernelId()) {
3282 Register Reg = Info.addLDSKernelId();
3283 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3284 CCInfo.AllocateReg(Reg);
3285 }
3286}
3287
3288// Allocate special input registers that are initialized per-wave.
3289void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF,
3290 SIMachineFunctionInfo &Info,
3291 CallingConv::ID CallConv,
3292 bool IsShader) const {
3293 bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
3294 if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
3295 // Note: user SGPRs are handled by the front-end for graphics shaders
3296 // Pad up the used user SGPRs with dead inputs.
3297
3298 // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
3299 // before enabling architected SGPRs for workgroup IDs.
3300 assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
3301
3302 unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
3303 // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
3304 // rely on it to reach 16 since if we end up having no stack usage, it will
3305 // not really be added.
3306 unsigned NumRequiredSystemSGPRs =
3307 Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
3308 Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
3309 for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
3310 Register Reg = Info.addReservedUserSGPR();
3311 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3312 CCInfo.AllocateReg(Reg);
3313 }
3314 }
3315
3316 if (!HasArchitectedSGPRs) {
3317 if (Info.hasWorkGroupIDX()) {
3318 Register Reg = Info.addWorkGroupIDX();
3319 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3320 CCInfo.AllocateReg(Reg);
3321 }
3322
3323 if (Info.hasWorkGroupIDY()) {
3324 Register Reg = Info.addWorkGroupIDY();
3325 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3326 CCInfo.AllocateReg(Reg);
3327 }
3328
3329 if (Info.hasWorkGroupIDZ()) {
3330 Register Reg = Info.addWorkGroupIDZ();
3331 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3332 CCInfo.AllocateReg(Reg);
3333 }
3334 }
3335
3336 if (Info.hasWorkGroupInfo()) {
3337 Register Reg = Info.addWorkGroupInfo();
3338 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3339 CCInfo.AllocateReg(Reg);
3340 }
3341
3342 if (Info.hasPrivateSegmentWaveByteOffset()) {
3343 // Scratch wave offset passed in system SGPR.
3344 unsigned PrivateSegmentWaveByteOffsetReg;
3345
3346 if (IsShader) {
3347 PrivateSegmentWaveByteOffsetReg =
3348 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
3349
3350 // This is true if the scratch wave byte offset doesn't have a fixed
3351 // location.
3352 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
3353 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
3354 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
3355 }
3356 } else
3357 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
3358
3359 MF.addLiveIn(PReg: PrivateSegmentWaveByteOffsetReg, RC: &AMDGPU::SGPR_32RegClass);
3360 CCInfo.AllocateReg(Reg: PrivateSegmentWaveByteOffsetReg);
3361 }
3362
3363 assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
3364 Info.getNumPreloadedSGPRs() >= 16);
3365}
3366
3367static void reservePrivateMemoryRegs(const TargetMachine &TM,
3368 MachineFunction &MF,
3369 const SIRegisterInfo &TRI,
3370 SIMachineFunctionInfo &Info) {
3371 // Now that we've figured out where the scratch register inputs are, see if
3372 // should reserve the arguments and use them directly.
3373 MachineFrameInfo &MFI = MF.getFrameInfo();
3374 bool HasStackObjects = MFI.hasStackObjects();
3375 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3376
3377 // Record that we know we have non-spill stack objects so we don't need to
3378 // check all stack objects later.
3379 if (HasStackObjects)
3380 Info.setHasNonSpillStackObjects(true);
3381
3382 // Everything live out of a block is spilled with fast regalloc, so it's
3383 // almost certain that spilling will be required.
3384 if (TM.getOptLevel() == CodeGenOptLevel::None)
3385 HasStackObjects = true;
3386
3387 // For now assume stack access is needed in any callee functions, so we need
3388 // the scratch registers to pass in.
3389 bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
3390
3391 if (!ST.hasFlatScratchEnabled()) {
3392 if (RequiresStackAccess && ST.isAmdHsaOrMesa(F: MF.getFunction())) {
3393 // If we have stack objects, we unquestionably need the private buffer
3394 // resource. For the Code Object V2 ABI, this will be the first 4 user
3395 // SGPR inputs. We can reserve those and use them directly.
3396
3397 Register PrivateSegmentBufferReg =
3398 Info.getPreloadedReg(Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
3399 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
3400 } else {
3401 unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
3402 // We tentatively reserve the last registers (skipping the last registers
3403 // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
3404 // we'll replace these with the ones immediately after those which were
3405 // really allocated. In the prologue copies will be inserted from the
3406 // argument to these reserved registers.
3407
3408 // Without HSA, relocations are used for the scratch pointer and the
3409 // buffer resource setup is always inserted in the prologue. Scratch wave
3410 // offset is still in an input SGPR.
3411 Info.setScratchRSrcReg(ReservedBufferReg);
3412 }
3413 }
3414
3415 MachineRegisterInfo &MRI = MF.getRegInfo();
3416
3417 // For entry functions we have to set up the stack pointer if we use it,
3418 // whereas non-entry functions get this "for free". This means there is no
3419 // intrinsic advantage to using S32 over S34 in cases where we do not have
3420 // calls but do need a frame pointer (i.e. if we are requested to have one
3421 // because frame pointer elimination is disabled). To keep things simple we
3422 // only ever use S32 as the call ABI stack pointer, and so using it does not
3423 // imply we need a separate frame pointer.
3424 //
3425 // Try to use s32 as the SP, but move it if it would interfere with input
3426 // arguments. This won't work with calls though.
3427 //
3428 // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
3429 // registers.
3430 if (!MRI.isLiveIn(Reg: AMDGPU::SGPR32)) {
3431 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
3432 } else {
3433 assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
3434
3435 if (MFI.hasCalls())
3436 report_fatal_error(reason: "call in graphics shader with too many input SGPRs");
3437
3438 for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
3439 if (!MRI.isLiveIn(Reg)) {
3440 Info.setStackPtrOffsetReg(Reg);
3441 break;
3442 }
3443 }
3444
3445 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
3446 report_fatal_error(reason: "failed to find register for SP");
3447 }
3448
3449 // hasFP should be accurate for entry functions even before the frame is
3450 // finalized, because it does not rely on the known stack size, only
3451 // properties like whether variable sized objects are present.
3452 if (ST.getFrameLowering()->hasFP(MF)) {
3453 Info.setFrameOffsetReg(AMDGPU::SGPR33);
3454 }
3455}
3456
3457bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
3458 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
3459 return !Info->isEntryFunction();
3460}
3461
3462void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {}
3463
3464void SITargetLowering::insertCopiesSplitCSR(
3465 MachineBasicBlock *Entry,
3466 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
3467 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3468
3469 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
3470 if (!IStart)
3471 return;
3472
3473 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3474 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
3475 MachineBasicBlock::iterator MBBI = Entry->begin();
3476 for (const MCPhysReg *I = IStart; *I; ++I) {
3477 const TargetRegisterClass *RC = nullptr;
3478 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3479 RC = &AMDGPU::SGPR_64RegClass;
3480 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3481 RC = &AMDGPU::SGPR_32RegClass;
3482 else
3483 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3484
3485 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
3486 // Create copy from CSR to a virtual register.
3487 Entry->addLiveIn(PhysReg: *I);
3488 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
3489 .addReg(RegNo: *I);
3490
3491 // Insert the copy-back instructions right before the terminator.
3492 for (auto *Exit : Exits)
3493 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
3494 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
3495 .addReg(RegNo: NewVR);
3496 }
3497}
3498
3499SDValue SITargetLowering::LowerFormalArguments(
3500 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3501 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3502 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3503 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3504
3505 MachineFunction &MF = DAG.getMachineFunction();
3506 const Function &Fn = MF.getFunction();
3507 FunctionType *FType = MF.getFunction().getFunctionType();
3508 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3509 bool IsError = false;
3510
3511 if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CC: CallConv)) {
3512 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3513 Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()));
3514 IsError = true;
3515 }
3516
3517 SmallVector<ISD::InputArg, 16> Splits;
3518 SmallVector<CCValAssign, 16> ArgLocs;
3519 BitVector Skipped(Fn.arg_size());
3520 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3521 *DAG.getContext());
3522
3523 bool IsGraphics = AMDGPU::isGraphics(CC: CallConv);
3524 bool IsKernel = AMDGPU::isKernel(CC: CallConv);
3525 bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC: CallConv);
3526
3527 if (IsGraphics) {
3528 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
3529 assert(!UserSGPRInfo.hasDispatchPtr() &&
3530 !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
3531 !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
3532 !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
3533 (void)UserSGPRInfo;
3534 if (!Subtarget->hasFlatScratchEnabled())
3535 assert(!UserSGPRInfo.hasFlatScratchInit());
3536 if ((CallConv != CallingConv::AMDGPU_CS &&
3537 CallConv != CallingConv::AMDGPU_Gfx &&
3538 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) ||
3539 !Subtarget->hasArchitectedSGPRs())
3540 assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
3541 !Info->hasWorkGroupIDZ());
3542 }
3543
3544 bool IsWholeWaveFunc = Info->isWholeWaveFunction();
3545
3546 if (CallConv == CallingConv::AMDGPU_PS) {
3547 processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
3548
3549 // At least one interpolation mode must be enabled or else the GPU will
3550 // hang.
3551 //
3552 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
3553 // set PSInputAddr, the user wants to enable some bits after the compilation
3554 // based on run-time states. Since we can't know what the final PSInputEna
3555 // will look like, so we shouldn't do anything here and the user should take
3556 // responsibility for the correct programming.
3557 //
3558 // Otherwise, the following restrictions apply:
3559 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
3560 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
3561 // enabled too.
3562 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
3563 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(Index: 11))) {
3564 CCInfo.AllocateReg(Reg: AMDGPU::VGPR0);
3565 CCInfo.AllocateReg(Reg: AMDGPU::VGPR1);
3566 Info->markPSInputAllocated(Index: 0);
3567 Info->markPSInputEnabled(Index: 0);
3568 }
3569 if (Subtarget->isAmdPalOS()) {
3570 // For isAmdPalOS, the user does not enable some bits after compilation
3571 // based on run-time states; the register values being generated here are
3572 // the final ones set in hardware. Therefore we need to apply the
3573 // workaround to PSInputAddr and PSInputEnable together. (The case where
3574 // a bit is set in PSInputAddr but not PSInputEnable is where the
3575 // frontend set up an input arg for a particular interpolation mode, but
3576 // nothing uses that input arg. Really we should have an earlier pass
3577 // that removes such an arg.)
3578 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
3579 if ((PsInputBits & 0x7F) == 0 ||
3580 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
3581 Info->markPSInputEnabled(Index: llvm::countr_zero(Val: Info->getPSInputAddr()));
3582 }
3583 } else if (IsKernel) {
3584 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
3585 } else {
3586 Splits.append(in_start: IsWholeWaveFunc ? std::next(x: Ins.begin()) : Ins.begin(),
3587 in_end: Ins.end());
3588 }
3589
3590 if (IsKernel)
3591 analyzeFormalArgumentsCompute(State&: CCInfo, Ins);
3592
3593 if (IsEntryFunc) {
3594 allocateSpecialEntryInputVGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3595 allocateHSAUserSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3596 if (IsKernel && Subtarget->hasKernargPreload())
3597 allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, TRI: *TRI, Info&: *Info);
3598
3599 allocateLDSKernelId(CCInfo, MF, TRI: *TRI, Info&: *Info);
3600 } else if (!IsGraphics) {
3601 // For the fixed ABI, pass workitem IDs in the last argument register.
3602 allocateSpecialInputVGPRsFixed(CCInfo, MF, TRI: *TRI, Info&: *Info);
3603
3604 // FIXME: Sink this into allocateSpecialInputSGPRs
3605 if (!Subtarget->hasFlatScratchEnabled())
3606 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
3607
3608 allocateSpecialInputSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3609 }
3610
3611 if (!IsKernel) {
3612 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg: isVarArg);
3613 CCInfo.AnalyzeFormalArguments(Ins: Splits, Fn: AssignFn);
3614
3615 // This assumes the registers are allocated by CCInfo in ascending order
3616 // with no gaps.
3617 Info->setNumWaveDispatchSGPRs(
3618 CCInfo.getFirstUnallocated(Regs: AMDGPU::SGPR_32RegClass.getRegisters()));
3619 Info->setNumWaveDispatchVGPRs(
3620 CCInfo.getFirstUnallocated(Regs: AMDGPU::VGPR_32RegClass.getRegisters()));
3621 } else if (Info->getNumKernargPreloadedSGPRs()) {
3622 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
3623 }
3624
3625 SmallVector<SDValue, 16> Chains;
3626
3627 if (IsWholeWaveFunc) {
3628 SDValue Setup = DAG.getNode(Opcode: AMDGPUISD::WHOLE_WAVE_SETUP, DL,
3629 ResultTys: {MVT::i1, MVT::Other}, Ops: Chain);
3630 InVals.push_back(Elt: Setup.getValue(R: 0));
3631 Chains.push_back(Elt: Setup.getValue(R: 1));
3632 }
3633
3634 // FIXME: This is the minimum kernel argument alignment. We should improve
3635 // this to the maximum alignment of the arguments.
3636 //
3637 // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
3638 // kern arg offset.
3639 const Align KernelArgBaseAlign = Align(16);
3640
3641 for (unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.size(), ArgIdx = 0; i != e;
3642 ++i) {
3643 const ISD::InputArg &Arg = Ins[i];
3644 if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) {
3645 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
3646 continue;
3647 }
3648
3649 CCValAssign &VA = ArgLocs[ArgIdx++];
3650 MVT VT = VA.getLocVT();
3651
3652 if (IsEntryFunc && VA.isMemLoc()) {
3653 VT = Ins[i].VT;
3654 EVT MemVT = VA.getLocVT();
3655
3656 const uint64_t Offset = VA.getLocMemOffset();
3657 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset);
3658
3659 if (Arg.Flags.isByRef()) {
3660 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain, Offset);
3661
3662 const GCNTargetMachine &TM =
3663 static_cast<const GCNTargetMachine &>(getTargetMachine());
3664 if (!TM.isNoopAddrSpaceCast(DL: DAG.getDataLayout(),
3665 SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3666 DestAS: Arg.Flags.getPointerAddrSpace())) {
3667 Ptr = DAG.getAddrSpaceCast(dl: DL, VT, Ptr, SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3668 DestAS: Arg.Flags.getPointerAddrSpace());
3669 }
3670
3671 InVals.push_back(Elt: Ptr);
3672 continue;
3673 }
3674
3675 SDValue NewArg;
3676 if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(Val: i)) {
3677 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
3678 // In this case the argument is packed into the previous preload SGPR.
3679 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
3680 int64_t OffsetDiff = Offset - AlignDownOffset;
3681 EVT IntVT = MemVT.changeTypeToInteger();
3682
3683 const SIMachineFunctionInfo *Info =
3684 MF.getInfo<SIMachineFunctionInfo>();
3685 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3686 Register Reg =
3687 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs[0];
3688
3689 assert(Reg);
3690 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3691 SDValue Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3692
3693 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL, VT: MVT::i32);
3694 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Copy, N2: ShiftAmt);
3695
3696 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Extract);
3697 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MemVT, Operand: ArgVal);
3698 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: ArgVal,
3699 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3700
3701 NewArg = DAG.getMergeValues(Ops: {NewArg, Copy.getValue(R: 1)}, dl: DL);
3702 } else {
3703 const SIMachineFunctionInfo *Info =
3704 MF.getInfo<SIMachineFunctionInfo>();
3705 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3706 const SmallVectorImpl<MCRegister> &PreloadRegs =
3707 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs;
3708
3709 SDValue Copy;
3710 if (PreloadRegs.size() == 1) {
3711 Register VReg = MRI.getLiveInVirtReg(PReg: PreloadRegs[0]);
3712 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VReg);
3713 NewArg = DAG.getCopyFromReg(
3714 Chain, dl: DL, Reg: VReg,
3715 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
3716 BitWidth: TRI->getRegSizeInBits(RC: *RC)));
3717
3718 } else {
3719 // If the kernarg alignment does not match the alignment of the SGPR
3720 // tuple RC that can accommodate this argument, it will be built up
3721 // via copies from from the individual SGPRs that the argument was
3722 // preloaded to.
3723 SmallVector<SDValue, 4> Elts;
3724 for (auto Reg : PreloadRegs) {
3725 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3726 Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3727 Elts.push_back(Elt: Copy);
3728 }
3729 NewArg =
3730 DAG.getBuildVector(VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
3731 NumElements: PreloadRegs.size()),
3732 DL, Ops: Elts);
3733 }
3734
3735 // If the argument was preloaded to multiple consecutive 32-bit
3736 // registers because of misalignment between addressable SGPR tuples
3737 // and the argument size, we can still assume that because of kernarg
3738 // segment alignment restrictions that NewArg's size is the same as
3739 // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
3740 // truncate since we cannot preload to less than a single SGPR and the
3741 // MemVT may be smaller.
3742 EVT MemVTInt =
3743 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
3744 if (MemVT.bitsLT(VT: NewArg.getSimpleValueType()))
3745 NewArg = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVTInt, Operand: NewArg);
3746
3747 NewArg = DAG.getBitcast(VT: MemVT, V: NewArg);
3748 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: NewArg,
3749 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3750 NewArg = DAG.getMergeValues(Ops: {NewArg, Chain}, dl: DL);
3751 }
3752 } else {
3753 // Hidden arguments that are in the kernel signature must be preloaded
3754 // to user SGPRs. Print a diagnostic error if a hidden argument is in
3755 // the argument list and is not preloaded.
3756 if (Arg.isOrigArg()) {
3757 Argument *OrigArg = Fn.getArg(i: Arg.getOrigArgIndex());
3758 if (OrigArg->hasAttribute(Kind: "amdgpu-hidden-argument")) {
3759 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3760 *OrigArg->getParent(),
3761 "hidden argument in kernel signature was not preloaded",
3762 DL.getDebugLoc()));
3763 }
3764 }
3765
3766 NewArg =
3767 lowerKernargMemParameter(DAG, VT, MemVT, SL: DL, Chain, Offset,
3768 Alignment, Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3769 }
3770 Chains.push_back(Elt: NewArg.getValue(R: 1));
3771
3772 auto *ParamTy =
3773 dyn_cast<PointerType>(Val: FType->getParamType(i: Ins[i].getOrigArgIndex()));
3774 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
3775 ParamTy &&
3776 (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
3777 ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
3778 // On SI local pointers are just offsets into LDS, so they are always
3779 // less than 16-bits. On CI and newer they could potentially be
3780 // real pointers, so we can't guarantee their size.
3781 NewArg = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: NewArg.getValueType(), N1: NewArg,
3782 N2: DAG.getValueType(MVT::i16));
3783 }
3784
3785 InVals.push_back(Elt: NewArg);
3786 continue;
3787 }
3788 if (!IsEntryFunc && VA.isMemLoc()) {
3789 SDValue Val = lowerStackParameter(DAG, VA, SL: DL, Chain, Arg);
3790 InVals.push_back(Elt: Val);
3791 if (!Arg.Flags.isByVal())
3792 Chains.push_back(Elt: Val.getValue(R: 1));
3793 continue;
3794 }
3795
3796 assert(VA.isRegLoc() && "Parameter must be in a register!");
3797
3798 Register Reg = VA.getLocReg();
3799 const TargetRegisterClass *RC = nullptr;
3800 if (AMDGPU::VGPR_32RegClass.contains(Reg))
3801 RC = &AMDGPU::VGPR_32RegClass;
3802 else if (AMDGPU::SGPR_32RegClass.contains(Reg))
3803 RC = &AMDGPU::SGPR_32RegClass;
3804 else
3805 llvm_unreachable("Unexpected register class in LowerFormalArguments!");
3806
3807 Reg = MF.addLiveIn(PReg: Reg, RC);
3808 SDValue Val = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT);
3809 if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
3810 // FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
3811 // they will read physical regs before any side effect instructions.
3812 SDValue ReadFirstLane =
3813 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3814 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
3815 N1: ReadFirstLane, N2: Val);
3816 }
3817
3818 if (Arg.Flags.isSRet()) {
3819 // The return object should be reasonably addressable.
3820 Val = annotateStackObjectPointer(Ptr: Val, DAG, DL,
3821 Alignment: Arg.Flags.getNonZeroMemAlign());
3822 }
3823
3824 Val = convertABITypeToValueType(DAG, Val, VA, SL: DL);
3825 InVals.push_back(Elt: Val);
3826 }
3827
3828 // Start adding system SGPRs.
3829 if (IsEntryFunc)
3830 allocateSystemSGPRs(CCInfo, MF, Info&: *Info, CallConv, IsShader: IsGraphics);
3831
3832 unsigned StackArgSize = CCInfo.getStackSize();
3833 Info->setBytesInStackArgArea(StackArgSize);
3834
3835 return Chains.empty() ? Chain
3836 : DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3837}
3838
3839// TODO: If return values can't fit in registers, we should return as many as
3840// possible in registers before passing on stack.
3841bool SITargetLowering::CanLowerReturn(
3842 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
3843 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
3844 const Type *RetTy) const {
3845 // Replacing returns with sret/stack usage doesn't make sense for shaders.
3846 // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
3847 // for shaders. Vector types should be explicitly handled by CC.
3848 if (AMDGPU::isEntryFunctionCC(CC: CallConv))
3849 return true;
3850
3851 SmallVector<CCValAssign, 16> RVLocs;
3852 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3853 if (!CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg)))
3854 return false;
3855
3856 // We must use the stack if return would require unavailable registers.
3857 unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
3858 unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
3859 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3860 if (CCInfo.isAllocated(Reg: AMDGPU::VGPR_32RegClass.getRegister(i)))
3861 return false;
3862
3863 return true;
3864}
3865
3866SDValue
3867SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3868 bool isVarArg,
3869 const SmallVectorImpl<ISD::OutputArg> &Outs,
3870 const SmallVectorImpl<SDValue> &OutVals,
3871 const SDLoc &DL, SelectionDAG &DAG) const {
3872 MachineFunction &MF = DAG.getMachineFunction();
3873 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3874 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3875
3876 if (AMDGPU::isKernel(CC: CallConv)) {
3877 return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
3878 OutVals, DL, DAG);
3879 }
3880
3881 bool IsShader = AMDGPU::isShader(CC: CallConv);
3882
3883 Info->setIfReturnsVoid(Outs.empty());
3884 bool IsWaveEnd = Info->returnsVoid() && IsShader;
3885
3886 // CCValAssign - represent the assignment of the return value to a location.
3887 SmallVector<CCValAssign, 48> RVLocs;
3888
3889 // CCState - Info about the registers and stack slots.
3890 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3891 *DAG.getContext());
3892
3893 // Analyze outgoing return values.
3894 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg: isVarArg));
3895
3896 SDValue Glue;
3897 SmallVector<SDValue, 48> RetOps;
3898 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
3899
3900 SDValue ReadFirstLane =
3901 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3902 // Copy the result values into the output registers.
3903 for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
3904 ++I, ++RealRVLocIdx) {
3905 CCValAssign &VA = RVLocs[I];
3906 assert(VA.isRegLoc() && "Can only return in registers!");
3907 // TODO: Partially return in registers if return values don't fit.
3908 SDValue Arg = OutVals[RealRVLocIdx];
3909
3910 // Copied from other backends.
3911 switch (VA.getLocInfo()) {
3912 case CCValAssign::Full:
3913 break;
3914 case CCValAssign::BCvt:
3915 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
3916 break;
3917 case CCValAssign::SExt:
3918 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3919 break;
3920 case CCValAssign::ZExt:
3921 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3922 break;
3923 case CCValAssign::AExt:
3924 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3925 break;
3926 default:
3927 llvm_unreachable("Unknown loc info!");
3928 }
3929 if (TRI->isSGPRPhysReg(Reg: VA.getLocReg()))
3930 Arg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Arg.getValueType(),
3931 N1: ReadFirstLane, N2: Arg);
3932 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Arg, Glue);
3933 Glue = Chain.getValue(R: 1);
3934 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3935 }
3936
3937 // FIXME: Does sret work properly?
3938 if (!Info->isEntryFunction()) {
3939 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3940 const MCPhysReg *I =
3941 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3942 if (I) {
3943 for (; *I; ++I) {
3944 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3945 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i64));
3946 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3947 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3948 else
3949 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3950 }
3951 }
3952 }
3953
3954 // Update chain and glue.
3955 RetOps[0] = Chain;
3956 if (Glue.getNode())
3957 RetOps.push_back(Elt: Glue);
3958
3959 unsigned Opc = AMDGPUISD::ENDPGM;
3960 if (!IsWaveEnd)
3961 Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
3962 : IsShader ? AMDGPUISD::RETURN_TO_EPILOG
3963 : AMDGPUISD::RET_GLUE;
3964 return DAG.getNode(Opcode: Opc, DL, VT: MVT::Other, Ops: RetOps);
3965}
3966
3967SDValue SITargetLowering::LowerCallResult(
3968 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
3969 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3970 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
3971 SDValue ThisVal) const {
3972 CCAssignFn *RetCC = CCAssignFnForReturn(CC: CallConv, IsVarArg);
3973
3974 // Assign locations to each value returned by this call.
3975 SmallVector<CCValAssign, 16> RVLocs;
3976 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
3977 *DAG.getContext());
3978 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC);
3979
3980 // Copy all of the result registers out of their specified physreg.
3981 for (CCValAssign VA : RVLocs) {
3982 SDValue Val;
3983
3984 if (VA.isRegLoc()) {
3985 Val =
3986 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
3987 Chain = Val.getValue(R: 1);
3988 InGlue = Val.getValue(R: 2);
3989 } else if (VA.isMemLoc()) {
3990 report_fatal_error(reason: "TODO: return values in memory");
3991 } else
3992 llvm_unreachable("unknown argument location type");
3993
3994 switch (VA.getLocInfo()) {
3995 case CCValAssign::Full:
3996 break;
3997 case CCValAssign::BCvt:
3998 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
3999 break;
4000 case CCValAssign::ZExt:
4001 Val = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Val,
4002 N2: DAG.getValueType(VA.getValVT()));
4003 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4004 break;
4005 case CCValAssign::SExt:
4006 Val = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Val,
4007 N2: DAG.getValueType(VA.getValVT()));
4008 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4009 break;
4010 case CCValAssign::AExt:
4011 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4012 break;
4013 default:
4014 llvm_unreachable("Unknown loc info!");
4015 }
4016
4017 InVals.push_back(Elt: Val);
4018 }
4019
4020 return Chain;
4021}
4022
4023// Add code to pass special inputs required depending on used features separate
4024// from the explicit user arguments present in the IR.
4025void SITargetLowering::passSpecialInputs(
4026 CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info,
4027 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
4028 SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const {
4029 // If we don't have a call site, this was a call inserted by
4030 // legalization. These can never use special inputs.
4031 if (!CLI.CB)
4032 return;
4033
4034 SelectionDAG &DAG = CLI.DAG;
4035 const SDLoc &DL = CLI.DL;
4036 const Function &F = DAG.getMachineFunction().getFunction();
4037
4038 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
4039 const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
4040
4041 const AMDGPUFunctionArgInfo &CalleeArgInfo =
4042 AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
4043
4044 // TODO: Unify with private memory register handling. This is complicated by
4045 // the fact that at least in kernels, the input argument is not necessarily
4046 // in the same location as the input.
4047 // clang-format off
4048 static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
4049 std::array<StringLiteral, 2>> ImplicitAttrs[] = {
4050 {AMDGPUFunctionArgInfo::DISPATCH_PTR, {"amdgpu-no-dispatch-ptr", ""}},
4051 {AMDGPUFunctionArgInfo::QUEUE_PTR, {"amdgpu-no-queue-ptr", ""}},
4052 {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, {"amdgpu-no-implicitarg-ptr", ""}},
4053 {AMDGPUFunctionArgInfo::DISPATCH_ID, {"amdgpu-no-dispatch-id", ""}},
4054 {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"}},
4055 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"}},
4056 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"}},
4057 {AMDGPUFunctionArgInfo::LDS_KERNEL_ID, {"amdgpu-no-lds-kernel-id", ""}},
4058 };
4059 // clang-format on
4060
4061 for (auto [InputID, Attrs] : ImplicitAttrs) {
4062 // If the callee does not use the attribute value, skip copying the value.
4063 if (all_of(Range&: Attrs, P: [&](StringRef Attr) {
4064 return Attr.empty() || CLI.CB->hasFnAttr(Kind: Attr);
4065 }))
4066 continue;
4067
4068 const auto [OutgoingArg, ArgRC, ArgTy] =
4069 CalleeArgInfo.getPreloadedValue(Value: InputID);
4070 if (!OutgoingArg)
4071 continue;
4072
4073 const auto [IncomingArg, IncomingArgRC, Ty] =
4074 CallerArgInfo.getPreloadedValue(Value: InputID);
4075 assert(IncomingArgRC == ArgRC);
4076
4077 // All special arguments are ints for now.
4078 EVT ArgVT = TRI->getSpillSize(RC: *ArgRC) == 8 ? MVT::i64 : MVT::i32;
4079 SDValue InputReg;
4080
4081 if (IncomingArg) {
4082 InputReg = loadInputValue(DAG, RC: ArgRC, VT: ArgVT, SL: DL, Arg: *IncomingArg);
4083 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
4084 // The implicit arg ptr is special because it doesn't have a corresponding
4085 // input for kernels, and is computed from the kernarg segment pointer.
4086 InputReg = getImplicitArgPtr(DAG, SL: DL);
4087 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
4088 std::optional<uint32_t> Id =
4089 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
4090 if (Id.has_value()) {
4091 InputReg = DAG.getConstant(Val: *Id, DL, VT: ArgVT);
4092 } else {
4093 InputReg = DAG.getPOISON(VT: ArgVT);
4094 }
4095 } else {
4096 // We may have proven the input wasn't needed, although the ABI is
4097 // requiring it. We just need to allocate the register appropriately.
4098 InputReg = DAG.getPOISON(VT: ArgVT);
4099 }
4100
4101 if (OutgoingArg->isRegister()) {
4102 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4103 if (!CCInfo.AllocateReg(Reg: OutgoingArg->getRegister()))
4104 report_fatal_error(reason: "failed to allocate implicit input argument");
4105 } else {
4106 unsigned SpecialArgOffset =
4107 CCInfo.AllocateStack(Size: ArgVT.getStoreSize(), Alignment: Align(4));
4108 SDValue ArgStore =
4109 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4110 MemOpChains.push_back(Elt: ArgStore);
4111 }
4112 }
4113
4114 // Pack workitem IDs into a single register or pass it as is if already
4115 // packed.
4116
4117 auto [OutgoingArg, ArgRC, Ty] =
4118 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
4119 if (!OutgoingArg)
4120 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4121 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
4122 if (!OutgoingArg)
4123 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4124 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
4125 if (!OutgoingArg)
4126 return;
4127
4128 const ArgDescriptor *IncomingArgX = std::get<0>(
4129 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X));
4130 const ArgDescriptor *IncomingArgY = std::get<0>(
4131 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
4132 const ArgDescriptor *IncomingArgZ = std::get<0>(
4133 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
4134
4135 SDValue InputReg;
4136 SDLoc SL;
4137
4138 const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-x");
4139 const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-y");
4140 const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-z");
4141
4142 // If incoming ids are not packed we need to pack them.
4143 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
4144 NeedWorkItemIDX) {
4145 if (Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 0) != 0) {
4146 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgX);
4147 } else {
4148 InputReg = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
4149 }
4150 }
4151
4152 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
4153 NeedWorkItemIDY && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 1) != 0) {
4154 SDValue Y = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgY);
4155 Y = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Y,
4156 N2: DAG.getShiftAmountConstant(Val: 10, VT: MVT::i32, DL: SL));
4157 InputReg = InputReg.getNode()
4158 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Y)
4159 : Y;
4160 }
4161
4162 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
4163 NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 2) != 0) {
4164 SDValue Z = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgZ);
4165 Z = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Z,
4166 N2: DAG.getShiftAmountConstant(Val: 20, VT: MVT::i32, DL: SL));
4167 InputReg = InputReg.getNode()
4168 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Z)
4169 : Z;
4170 }
4171
4172 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
4173 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
4174 // We're in a situation where the outgoing function requires the workitem
4175 // ID, but the calling function does not have it (e.g a graphics function
4176 // calling a C calling convention function). This is illegal, but we need
4177 // to produce something.
4178 InputReg = DAG.getPOISON(VT: MVT::i32);
4179 } else {
4180 // Workitem ids are already packed, any of present incoming arguments
4181 // will carry all required fields.
4182 ArgDescriptor IncomingArg =
4183 ArgDescriptor::createArg(Arg: IncomingArgX ? *IncomingArgX
4184 : IncomingArgY ? *IncomingArgY
4185 : *IncomingArgZ,
4186 Mask: ~0u);
4187 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: IncomingArg);
4188 }
4189 }
4190
4191 if (OutgoingArg->isRegister()) {
4192 if (InputReg)
4193 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4194
4195 CCInfo.AllocateReg(Reg: OutgoingArg->getRegister());
4196 } else {
4197 unsigned SpecialArgOffset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
4198 if (InputReg) {
4199 SDValue ArgStore =
4200 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4201 MemOpChains.push_back(Elt: ArgStore);
4202 }
4203 }
4204}
4205
4206bool SITargetLowering::isEligibleForTailCallOptimization(
4207 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
4208 const SmallVectorImpl<ISD::OutputArg> &Outs,
4209 const SmallVectorImpl<SDValue> &OutVals,
4210 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4211 if (AMDGPU::isChainCC(CC: CalleeCC))
4212 return true;
4213
4214 if (!AMDGPU::mayTailCallThisCC(CC: CalleeCC))
4215 return false;
4216
4217 // For a divergent call target, we need to do a waterfall loop over the
4218 // possible callees which precludes us from using a simple jump.
4219 if (Callee->isDivergent())
4220 return false;
4221
4222 MachineFunction &MF = DAG.getMachineFunction();
4223 const Function &CallerF = MF.getFunction();
4224 CallingConv::ID CallerCC = CallerF.getCallingConv();
4225 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
4226 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
4227
4228 // Kernels aren't callable, and don't have a live in return address so it
4229 // doesn't make sense to do a tail call with entry functions.
4230 if (!CallerPreserved)
4231 return false;
4232
4233 bool CCMatch = CallerCC == CalleeCC;
4234
4235 if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
4236 if (AMDGPU::canGuaranteeTCO(CC: CalleeCC) && CCMatch)
4237 return true;
4238 return false;
4239 }
4240
4241 // TODO: Can we handle var args?
4242 if (IsVarArg)
4243 return false;
4244
4245 for (const Argument &Arg : CallerF.args()) {
4246 if (Arg.hasByValAttr())
4247 return false;
4248 }
4249
4250 LLVMContext &Ctx = *DAG.getContext();
4251
4252 // Check that the call results are passed in the same way.
4253 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C&: Ctx, Ins,
4254 CalleeFn: CCAssignFnForCall(CC: CalleeCC, IsVarArg),
4255 CallerFn: CCAssignFnForCall(CC: CallerCC, IsVarArg)))
4256 return false;
4257
4258 // The callee has to preserve all registers the caller needs to preserve.
4259 if (!CCMatch) {
4260 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
4261 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
4262 return false;
4263 }
4264
4265 // Nothing more to check if the callee is taking no arguments.
4266 if (Outs.empty())
4267 return true;
4268
4269 SmallVector<CCValAssign, 16> ArgLocs;
4270 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
4271
4272 // FIXME: We are not allocating special input registers, so we will be
4273 // deciding based on incorrect register assignments.
4274 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CalleeCC, IsVarArg));
4275
4276 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
4277 // If the stack arguments for this call do not fit into our own save area then
4278 // the call cannot be made tail.
4279 // TODO: Is this really necessary?
4280 if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
4281 return false;
4282
4283 for (const auto &[CCVA, ArgVal] : zip_equal(t&: ArgLocs, u: OutVals)) {
4284 // FIXME: What about inreg arguments that end up passed in memory?
4285 if (!CCVA.isRegLoc())
4286 continue;
4287
4288 // If we are passing an argument in an SGPR, and the value is divergent,
4289 // this call requires a waterfall loop.
4290 if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(Reg: CCVA.getLocReg())) {
4291 LLVM_DEBUG(
4292 dbgs() << "Cannot tail call due to divergent outgoing argument in "
4293 << printReg(CCVA.getLocReg(), TRI) << '\n');
4294 return false;
4295 }
4296 }
4297
4298 const MachineRegisterInfo &MRI = MF.getRegInfo();
4299 return parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals);
4300}
4301
4302bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
4303 if (!CI->isTailCall())
4304 return false;
4305
4306 const Function *ParentFn = CI->getFunction();
4307 if (AMDGPU::isEntryFunctionCC(CC: ParentFn->getCallingConv()))
4308 return false;
4309 return true;
4310}
4311
4312namespace {
4313// Chain calls have special arguments that we need to handle. These are
4314// tagging along at the end of the arguments list(s), after the SGPR and VGPR
4315// arguments (index 0 and 1 respectively).
4316enum ChainCallArgIdx {
4317 Exec = 2,
4318 Flags,
4319 NumVGPRs,
4320 FallbackExec,
4321 FallbackCallee
4322};
4323} // anonymous namespace
4324
4325// The wave scratch offset register is used as the global base pointer.
4326SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
4327 SmallVectorImpl<SDValue> &InVals) const {
4328 CallingConv::ID CallConv = CLI.CallConv;
4329 bool IsChainCallConv = AMDGPU::isChainCC(CC: CallConv);
4330
4331 SelectionDAG &DAG = CLI.DAG;
4332
4333 const SDLoc &DL = CLI.DL;
4334 SDValue Chain = CLI.Chain;
4335 SDValue Callee = CLI.Callee;
4336
4337 llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
4338 bool UsesDynamicVGPRs = false;
4339 if (IsChainCallConv) {
4340 // The last arguments should be the value that we need to put in EXEC,
4341 // followed by the flags and any other arguments with special meanings.
4342 // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so
4343 // we don't treat them like the "real" arguments.
4344 auto RequestedExecIt =
4345 llvm::find_if(Range&: CLI.Outs, P: [](const ISD::OutputArg &Arg) {
4346 return Arg.OrigArgIndex == 2;
4347 });
4348 assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC");
4349
4350 size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin();
4351 CLI.OutVals.erase(CS: CLI.OutVals.begin() + SpecialArgsBeginIdx,
4352 CE: CLI.OutVals.end());
4353 CLI.Outs.erase(CS: RequestedExecIt, CE: CLI.Outs.end());
4354
4355 assert(CLI.Outs.back().OrigArgIndex < 2 &&
4356 "Haven't popped all the special args");
4357
4358 TargetLowering::ArgListEntry RequestedExecArg =
4359 CLI.Args[ChainCallArgIdx::Exec];
4360 if (!RequestedExecArg.Ty->isIntegerTy(BitWidth: Subtarget->getWavefrontSize()))
4361 return lowerUnhandledCall(CLI, InVals, Reason: "Invalid value for EXEC");
4362
4363 // Convert constants into TargetConstants, so they become immediate operands
4364 // instead of being selected into S_MOV.
4365 auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) {
4366 if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Val&: Arg.Node)) {
4367 ChainCallSpecialArgs.push_back(Elt: DAG.getTargetConstant(
4368 Val: ArgNode->getAPIntValue(), DL, VT: ArgNode->getValueType(ResNo: 0)));
4369 } else
4370 ChainCallSpecialArgs.push_back(Elt: Arg.Node);
4371 };
4372
4373 PushNodeOrTargetConstant(RequestedExecArg);
4374
4375 // Process any other special arguments depending on the value of the flags.
4376 TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags];
4377
4378 const APInt &FlagsValue = cast<ConstantSDNode>(Val&: Flags.Node)->getAPIntValue();
4379 if (FlagsValue.isZero()) {
4380 if (CLI.Args.size() > ChainCallArgIdx::Flags + 1)
4381 return lowerUnhandledCall(CLI, InVals,
4382 Reason: "no additional args allowed if flags == 0");
4383 } else if (FlagsValue.isOneBitSet(BitNo: 0)) {
4384 if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
4385 return lowerUnhandledCall(CLI, InVals, Reason: "expected 3 additional args");
4386 }
4387
4388 if (!Subtarget->isWave32()) {
4389 return lowerUnhandledCall(
4390 CLI, InVals, Reason: "dynamic VGPR mode is only supported for wave32");
4391 }
4392
4393 UsesDynamicVGPRs = true;
4394 std::for_each(first: CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
4395 last: CLI.Args.end(), f: PushNodeOrTargetConstant);
4396 }
4397 }
4398
4399 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
4400 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
4401 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
4402 bool &IsTailCall = CLI.IsTailCall;
4403 bool IsVarArg = CLI.IsVarArg;
4404 bool IsSibCall = false;
4405 MachineFunction &MF = DAG.getMachineFunction();
4406
4407 if (Callee.isUndef() || isNullConstant(V: Callee)) {
4408 if (!CLI.IsTailCall) {
4409 for (ISD::InputArg &Arg : CLI.Ins)
4410 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
4411 }
4412
4413 return Chain;
4414 }
4415
4416 if (IsVarArg) {
4417 return lowerUnhandledCall(CLI, InVals,
4418 Reason: "unsupported call to variadic function ");
4419 }
4420
4421 if (!CLI.CB)
4422 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported libcall legalization");
4423
4424 if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
4425 return lowerUnhandledCall(CLI, InVals,
4426 Reason: "unsupported required tail call to function ");
4427 }
4428
4429 if (IsTailCall) {
4430 IsTailCall = isEligibleForTailCallOptimization(Callee, CalleeCC: CallConv, IsVarArg,
4431 Outs, OutVals, Ins, DAG);
4432 if (!IsTailCall &&
4433 ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
4434 report_fatal_error(reason: "failed to perform tail call elimination on a call "
4435 "site marked musttail or on llvm.amdgcn.cs.chain");
4436 }
4437
4438 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4439
4440 // A sibling call is one where we're under the usual C ABI and not planning
4441 // to change that but can still do a tail call:
4442 if (!TailCallOpt && IsTailCall)
4443 IsSibCall = true;
4444
4445 if (IsTailCall)
4446 ++NumTailCalls;
4447 }
4448
4449 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4450 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4451 SmallVector<SDValue, 8> MemOpChains;
4452
4453 // Analyze operands of the call, assigning locations to each operand.
4454 SmallVector<CCValAssign, 16> ArgLocs;
4455 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
4456 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg);
4457
4458 if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CC: CallConv) &&
4459 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {
4460 // With a fixed ABI, allocate fixed registers before user arguments.
4461 passSpecialInputs(CLI, CCInfo, Info: *Info, RegsToPass, MemOpChains, Chain);
4462 }
4463
4464 // Mark the scratch resource descriptor as allocated so the CC analysis
4465 // does not assign user arguments to these registers, matching the callee.
4466 if (!Subtarget->hasFlatScratchEnabled())
4467 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
4468
4469 CCInfo.AnalyzeCallOperands(Outs, Fn: AssignFn);
4470
4471 // Get a count of how many bytes are to be pushed on the stack.
4472 unsigned NumBytes = CCInfo.getStackSize();
4473
4474 if (IsSibCall) {
4475 // Since we're not changing the ABI to make this a tail call, the memory
4476 // operands are already available in the caller's incoming argument space.
4477 NumBytes = 0;
4478 }
4479
4480 // FPDiff is the byte offset of the call's argument area from the callee's.
4481 // Stores to callee stack arguments will be placed in FixedStackSlots offset
4482 // by this amount for a tail call. In a sibling call it must be 0 because the
4483 // caller will deallocate the entire stack and the callee still expects its
4484 // arguments to begin at SP+0. Completely unused for non-tail calls.
4485 int32_t FPDiff = 0;
4486 MachineFrameInfo &MFI = MF.getFrameInfo();
4487 auto *TRI = Subtarget->getRegisterInfo();
4488
4489 // Adjust the stack pointer for the new arguments...
4490 // These operations are automatically eliminated by the prolog/epilog pass
4491 if (!IsSibCall)
4492 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL);
4493
4494 if (!IsSibCall || IsChainCallConv) {
4495 if (!Subtarget->hasFlatScratchEnabled()) {
4496 SmallVector<SDValue, 4> CopyFromChains;
4497
4498 // In the HSA case, this should be an identity copy.
4499 SDValue ScratchRSrcReg =
4500 DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
4501 RegsToPass.emplace_back(Args: IsChainCallConv
4502 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
4503 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3,
4504 Args&: ScratchRSrcReg);
4505 CopyFromChains.push_back(Elt: ScratchRSrcReg.getValue(R: 1));
4506 Chain = DAG.getTokenFactor(DL, Vals&: CopyFromChains);
4507 }
4508 }
4509
4510 const unsigned NumSpecialInputs = RegsToPass.size();
4511
4512 MVT PtrVT = MVT::i32;
4513
4514 // Walk the register/memloc assignments, inserting copies/loads.
4515 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4516 CCValAssign &VA = ArgLocs[i];
4517 SDValue Arg = OutVals[i];
4518
4519 // Promote the value if needed.
4520 switch (VA.getLocInfo()) {
4521 case CCValAssign::Full:
4522 break;
4523 case CCValAssign::BCvt:
4524 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
4525 break;
4526 case CCValAssign::ZExt:
4527 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4528 break;
4529 case CCValAssign::SExt:
4530 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4531 break;
4532 case CCValAssign::AExt:
4533 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4534 break;
4535 case CCValAssign::FPExt:
4536 Arg = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4537 break;
4538 default:
4539 llvm_unreachable("Unknown loc info!");
4540 }
4541
4542 if (VA.isRegLoc()) {
4543 RegsToPass.push_back(Elt: std::pair(VA.getLocReg(), Arg));
4544 } else {
4545 assert(VA.isMemLoc());
4546
4547 SDValue DstAddr;
4548 MachinePointerInfo DstInfo;
4549
4550 unsigned LocMemOffset = VA.getLocMemOffset();
4551 int32_t Offset = LocMemOffset;
4552
4553 SDValue PtrOff = DAG.getConstant(Val: Offset, DL, VT: PtrVT);
4554 MaybeAlign Alignment;
4555
4556 if (IsTailCall) {
4557 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4558 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
4559 : VA.getValVT().getStoreSize();
4560
4561 // FIXME: We can have better than the minimum byval required alignment.
4562 Alignment =
4563 Flags.isByVal()
4564 ? Flags.getNonZeroByValAlign()
4565 : commonAlignment(A: Subtarget->getStackAlignment(), Offset);
4566
4567 Offset = Offset + FPDiff;
4568 int FI = MFI.CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
4569
4570 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
4571 DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
4572
4573 // Make sure any stack arguments overlapping with where we're storing
4574 // are loaded before this eventual operation. Otherwise they'll be
4575 // clobbered.
4576
4577 // FIXME: Why is this really necessary? This seems to just result in a
4578 // lot of code to copy the stack and write them back to the same
4579 // locations, which are supposed to be immutable?
4580 Chain = addTokenForArgument(Chain, DAG, MFI, ClobberedFI: FI);
4581 } else {
4582 // Stores to the argument stack area are relative to the stack pointer.
4583 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getStackPtrOffsetReg(),
4584 VT: MVT::i32);
4585 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SP, N2: PtrOff);
4586 DstInfo = MachinePointerInfo::getStack(MF, Offset: LocMemOffset);
4587 Alignment =
4588 commonAlignment(A: Subtarget->getStackAlignment(), Offset: LocMemOffset);
4589 }
4590
4591 if (Outs[i].Flags.isByVal()) {
4592 SDValue SizeNode =
4593 DAG.getConstant(Val: Outs[i].Flags.getByValSize(), DL, VT: MVT::i32);
4594 SDValue Cpy =
4595 DAG.getMemcpy(Chain, dl: DL, Dst: DstAddr, Src: Arg, Size: SizeNode,
4596 DstAlign: Outs[i].Flags.getNonZeroByValAlign(),
4597 SrcAlign: Outs[i].Flags.getNonZeroByValAlign(),
4598 /*isVol = */ false, /*AlwaysInline = */ true,
4599 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: DstInfo,
4600 SrcPtrInfo: MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
4601
4602 MemOpChains.push_back(Elt: Cpy);
4603 } else {
4604 SDValue Store =
4605 DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo, Alignment);
4606 MemOpChains.push_back(Elt: Store);
4607 }
4608 }
4609 }
4610
4611 if (!MemOpChains.empty())
4612 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
4613
4614 SDValue ReadFirstLaneID =
4615 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4616
4617 SDValue TokenGlue;
4618 if (CLI.ConvergenceControlToken) {
4619 TokenGlue = DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL, VT: MVT::Glue,
4620 Operand: CLI.ConvergenceControlToken);
4621 }
4622
4623 // Build a sequence of copy-to-reg nodes chained together with token chain
4624 // and flag operands which copy the outgoing args into the appropriate regs.
4625 SDValue InGlue;
4626
4627 unsigned ArgIdx = 0;
4628 for (auto [Reg, Val] : RegsToPass) {
4629 if (ArgIdx++ >= NumSpecialInputs &&
4630 (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
4631 // For chain calls, the inreg arguments are required to be
4632 // uniform. Speculatively Insert a readfirstlane in case we cannot prove
4633 // they are uniform.
4634 //
4635 // For other calls, if an inreg arguments is known to be uniform,
4636 // speculatively insert a readfirstlane in case it is in a VGPR.
4637 //
4638 // FIXME: We need to execute this in a waterfall loop if it is a divergent
4639 // value, so let that continue to produce invalid code.
4640
4641 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
4642 if (TokenGlue)
4643 ReadfirstlaneArgs.push_back(Elt: TokenGlue);
4644 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
4645 Ops: ReadfirstlaneArgs);
4646 }
4647
4648 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: Val, Glue: InGlue);
4649 InGlue = Chain.getValue(R: 1);
4650 }
4651
4652 // We don't usually want to end the call-sequence here because we would tidy
4653 // the frame up *after* the call, however in the ABI-changing tail-call case
4654 // we've carefully laid out the parameters so that when sp is reset they'll be
4655 // in the correct location.
4656 if (IsTailCall && !IsSibCall) {
4657 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL);
4658 InGlue = Chain.getValue(R: 1);
4659 }
4660
4661 std::vector<SDValue> Ops({Chain});
4662
4663 // Add a redundant copy of the callee global which will not be legalized, as
4664 // we need direct access to the callee later.
4665 if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
4666 const GlobalValue *GV = GSD->getGlobal();
4667 Ops.push_back(x: Callee);
4668 Ops.push_back(x: DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64));
4669 } else {
4670 if (IsTailCall) {
4671 // isEligibleForTailCallOptimization considered whether the call target is
4672 // divergent, but we may still end up with a uniform value in a VGPR.
4673 // Insert a readfirstlane just in case.
4674 SDValue ReadFirstLaneID =
4675 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4676
4677 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
4678 if (TokenGlue)
4679 ReadfirstlaneArgs.push_back(Elt: TokenGlue); // Wire up convergence token.
4680 Callee = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Callee.getValueType(),
4681 Ops: ReadfirstlaneArgs);
4682 }
4683
4684 Ops.push_back(x: Callee);
4685 Ops.push_back(x: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
4686 }
4687
4688 if (IsTailCall) {
4689 // Each tail call may have to adjust the stack by a different amount, so
4690 // this information must travel along with the operation for eventual
4691 // consumption by emitEpilogue.
4692 Ops.push_back(x: DAG.getTargetConstant(Val: FPDiff, DL, VT: MVT::i32));
4693 }
4694
4695 if (IsChainCallConv)
4696 llvm::append_range(C&: Ops, R&: ChainCallSpecialArgs);
4697
4698 // Add argument registers to the end of the list so that they are known live
4699 // into the call.
4700 for (auto &[Reg, Val] : RegsToPass)
4701 Ops.push_back(x: DAG.getRegister(Reg, VT: Val.getValueType()));
4702
4703 // Add a register mask operand representing the call-preserved registers.
4704 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
4705 assert(Mask && "Missing call preserved mask for calling convention");
4706 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
4707
4708 if (SDValue Token = CLI.ConvergenceControlToken) {
4709 SmallVector<SDValue, 2> GlueOps;
4710 GlueOps.push_back(Elt: Token);
4711 if (InGlue)
4712 GlueOps.push_back(Elt: InGlue);
4713
4714 InGlue = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::CONVERGENCECTRL_GLUE, dl: DL,
4715 VT: MVT::Glue, Ops: GlueOps),
4716 0);
4717 }
4718
4719 if (InGlue)
4720 Ops.push_back(x: InGlue);
4721
4722 // If we're doing a tall call, use a TC_RETURN here rather than an
4723 // actual call instruction.
4724 if (IsTailCall) {
4725 MFI.setHasTailCall();
4726 unsigned OPC = AMDGPUISD::TC_RETURN;
4727 switch (CallConv) {
4728 case CallingConv::AMDGPU_Gfx:
4729 OPC = AMDGPUISD::TC_RETURN_GFX;
4730 break;
4731 case CallingConv::AMDGPU_CS_Chain:
4732 case CallingConv::AMDGPU_CS_ChainPreserve:
4733 OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
4734 : AMDGPUISD::TC_RETURN_CHAIN;
4735 break;
4736 }
4737
4738 // If the caller is a whole wave function, we need to use a special opcode
4739 // so we can patch up EXEC.
4740 if (Info->isWholeWaveFunction())
4741 OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
4742
4743 SDValue Ret = DAG.getNode(Opcode: OPC, DL, VT: MVT::Other, Ops);
4744 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
4745 return Ret;
4746 }
4747
4748 // Returns a chain and a flag for retval copy to use.
4749 SDValue Call = DAG.getNode(Opcode: AMDGPUISD::CALL, DL, ResultTys: {MVT::Other, MVT::Glue}, Ops);
4750 DAG.addNoMergeSiteInfo(Node: Call.getNode(), NoMerge: CLI.NoMerge);
4751 Chain = Call.getValue(R: 0);
4752 InGlue = Call.getValue(R: 1);
4753
4754 uint64_t CalleePopBytes = NumBytes;
4755 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: CalleePopBytes, Glue: InGlue, DL);
4756 if (!Ins.empty())
4757 InGlue = Chain.getValue(R: 1);
4758
4759 // Handle result values, copying them out of physregs into vregs that we
4760 // return.
4761 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
4762 InVals, /*IsThisReturn=*/false, ThisVal: SDValue());
4763}
4764
4765// This is similar to the default implementation in ExpandDYNAMIC_STACKALLOC,
4766// except for:
4767// 1. Stack growth direction(default: downwards, AMDGPU: upwards), and
4768// 2. Scale size where, scale = wave-reduction(alloca-size) * wave-size
4769SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
4770 SelectionDAG &DAG) const {
4771 const MachineFunction &MF = DAG.getMachineFunction();
4772 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4773
4774 SDLoc dl(Op);
4775 EVT VT = Op.getValueType();
4776 SDValue Chain = Op.getOperand(i: 0);
4777 Register SPReg = Info->getStackPtrOffsetReg();
4778
4779 // Chain the dynamic stack allocation so that it doesn't modify the stack
4780 // pointer when other instructions are using the stack.
4781 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
4782
4783 SDValue Size = Op.getOperand(i: 1);
4784 SDValue BaseAddr = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
4785 Align Alignment = cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getAlignValue();
4786
4787 const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
4788 assert(TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
4789 "Stack grows upwards for AMDGPU");
4790
4791 Chain = BaseAddr.getValue(R: 1);
4792 // When using flat-scratch, the stack offset is unscaled.
4793 const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
4794 const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
4795
4796 Align StackAlign = TFL->getStackAlign();
4797 if (Alignment > StackAlign) {
4798 uint64_t ScaledAlignment = Alignment.value()
4799 << (HasFlatScratch ? 0 : WavefrontSizeLog2);
4800 uint64_t StackAlignMask = ScaledAlignment - 1;
4801 SDValue TmpAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr,
4802 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT));
4803 BaseAddr = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TmpAddr,
4804 N2: DAG.getSignedConstant(Val: -ScaledAlignment, DL: dl, VT));
4805 }
4806
4807 assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
4808 SDValue NewSP;
4809 if (isa<ConstantSDNode>(Val: Size)) {
4810 // Increase the stack pointer by the size of the alloca.
4811 // If not using flat-scratch, we have to scale the size by the wave-size.
4812 SDValue ScaledSize =
4813 HasFlatScratch
4814 ? Size
4815 : DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4816 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4817 NewSP = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value
4818 } else {
4819 // For dynamic sized alloca, perform wave-wide reduction to get max of
4820 // alloca size(divergent), and then scale it (when not using flat-scratch)
4821 // by wave-size.
4822 SDValue WaveReduction =
4823 DAG.getTargetConstant(Val: Intrinsic::amdgcn_wave_reduce_umax, DL: dl, VT: MVT::i32);
4824 Size = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: WaveReduction,
4825 N2: Size, N3: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
4826 SDValue ScaledSize = Size;
4827 if (!HasFlatScratch) {
4828 ScaledSize =
4829 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4830 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4831 }
4832 NewSP =
4833 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value in vgpr.
4834 SDValue ReadFirstLaneID =
4835 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: dl, VT: MVT::i32);
4836 NewSP = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: ReadFirstLaneID,
4837 N2: NewSP);
4838 }
4839
4840 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: NewSP); // Output chain
4841 SDValue CallSeqEnd = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
4842
4843 return DAG.getMergeValues(Ops: {BaseAddr, CallSeqEnd}, dl);
4844}
4845
4846SDValue SITargetLowering::LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const {
4847 if (Op.getValueType() != MVT::i32)
4848 return Op; // Defer to cannot select error.
4849
4850 Register SP = getStackPointerRegisterToSaveRestore();
4851 SDLoc SL(Op);
4852
4853 SDValue CopyFromSP = DAG.getCopyFromReg(Chain: Op->getOperand(Num: 0), dl: SL, Reg: SP, VT: MVT::i32);
4854
4855 // Convert from wave uniform to swizzled vector address. This should protect
4856 // from any edge cases where the stacksave result isn't directly used with
4857 // stackrestore.
4858 SDValue VectorAddress =
4859 DAG.getNode(Opcode: AMDGPUISD::WAVE_ADDRESS, DL: SL, VT: MVT::i32, Operand: CopyFromSP);
4860 return DAG.getMergeValues(Ops: {VectorAddress, CopyFromSP.getValue(R: 1)}, dl: SL);
4861}
4862
4863SDValue SITargetLowering::lowerGET_ROUNDING(SDValue Op,
4864 SelectionDAG &DAG) const {
4865 SDLoc SL(Op);
4866 assert(Op.getValueType() == MVT::i32);
4867
4868 uint32_t BothRoundHwReg =
4869 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4870 SDValue GetRoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4871
4872 SDValue IntrinID =
4873 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
4874 SDValue GetReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList: Op->getVTList(),
4875 N1: Op.getOperand(i: 0), N2: IntrinID, N3: GetRoundBothImm);
4876
4877 // There are two rounding modes, one for f32 and one for f64/f16. We only
4878 // report in the standard value range if both are the same.
4879 //
4880 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
4881 // ties away from zero is not supported, and the other values are rotated by
4882 // 1.
4883 //
4884 // If the two rounding modes are not the same, report a target defined value.
4885
4886 // Mode register rounding mode fields:
4887 //
4888 // [1:0] Single-precision round mode.
4889 // [3:2] Double/Half-precision round mode.
4890 //
4891 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
4892 //
4893 // Hardware Spec
4894 // Toward-0 3 0
4895 // Nearest Even 0 1
4896 // +Inf 1 2
4897 // -Inf 2 3
4898 // NearestAway0 N/A 4
4899 //
4900 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
4901 // table we can index by the raw hardware mode.
4902 //
4903 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
4904
4905 SDValue BitTable =
4906 DAG.getConstant(Val: AMDGPU::FltRoundConversionTable, DL: SL, VT: MVT::i64);
4907
4908 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4909 SDValue RoundModeTimesNumBits =
4910 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: GetReg, N2: Two);
4911
4912 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
4913 // knew only one mode was demanded.
4914 SDValue TableValue =
4915 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4916 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4917
4918 SDValue EntryMask = DAG.getConstant(Val: 0xf, DL: SL, VT: MVT::i32);
4919 SDValue TableEntry =
4920 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TruncTable, N2: EntryMask);
4921
4922 // There's a gap in the 4-bit encoded table and actual enum values, so offset
4923 // if it's an extended value.
4924 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4925 SDValue IsStandardValue =
4926 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: TableEntry, RHS: Four, Cond: ISD::SETULT);
4927 SDValue EnumOffset = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: TableEntry, N2: Four);
4928 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: IsStandardValue,
4929 N2: TableEntry, N3: EnumOffset);
4930
4931 return DAG.getMergeValues(Ops: {Result, GetReg.getValue(R: 1)}, dl: SL);
4932}
4933
4934SDValue SITargetLowering::lowerSET_ROUNDING(SDValue Op,
4935 SelectionDAG &DAG) const {
4936 SDLoc SL(Op);
4937
4938 SDValue NewMode = Op.getOperand(i: 1);
4939 assert(NewMode.getValueType() == MVT::i32);
4940
4941 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
4942 // hardware MODE.fp_round values.
4943 if (auto *ConstMode = dyn_cast<ConstantSDNode>(Val&: NewMode)) {
4944 uint32_t ClampedVal = std::min(
4945 a: static_cast<uint32_t>(ConstMode->getZExtValue()),
4946 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
4947 NewMode = DAG.getConstant(
4948 Val: AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal), DL: SL, VT: MVT::i32);
4949 } else {
4950 // If we know the input can only be one of the supported standard modes in
4951 // the range 0-3, we can use a simplified mapping to hardware values.
4952 KnownBits KB = DAG.computeKnownBits(Op: NewMode);
4953 const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
4954 // The supported standard values are 0-3. The extended values start at 8. We
4955 // need to offset by 4 if the value is in the extended range.
4956
4957 if (UseReducedTable) {
4958 // Truncate to the low 32-bits.
4959 SDValue BitTable = DAG.getConstant(
4960 Val: AMDGPU::FltRoundToHWConversionTable & 0xffff, DL: SL, VT: MVT::i32);
4961
4962 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4963 SDValue RoundModeTimesNumBits =
4964 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewMode, N2: Two);
4965
4966 NewMode =
4967 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: BitTable, N2: RoundModeTimesNumBits);
4968
4969 // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
4970 // the table extracted bits into inline immediates.
4971 } else {
4972 // table_index = umin(value, value - 4)
4973 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
4974 SDValue BitTable =
4975 DAG.getConstant(Val: AMDGPU::FltRoundToHWConversionTable, DL: SL, VT: MVT::i64);
4976
4977 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4978 SDValue OffsetEnum = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewMode, N2: Four);
4979 SDValue IndexVal =
4980 DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewMode, N2: OffsetEnum);
4981
4982 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4983 SDValue RoundModeTimesNumBits =
4984 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: IndexVal, N2: Two);
4985
4986 SDValue TableValue =
4987 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4988 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4989
4990 // No need to mask out the high bits since the setreg will ignore them
4991 // anyway.
4992 NewMode = TruncTable;
4993 }
4994
4995 // Insert a readfirstlane in case the value is a VGPR. We could do this
4996 // earlier and keep more operations scalar, but that interferes with
4997 // combining the source.
4998 SDValue ReadFirstLaneID =
4999 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
5000 NewMode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5001 N1: ReadFirstLaneID, N2: NewMode);
5002 }
5003
5004 // N.B. The setreg will be later folded into s_round_mode on supported
5005 // targets.
5006 SDValue IntrinID =
5007 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
5008 uint32_t BothRoundHwReg =
5009 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
5010 SDValue RoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
5011
5012 SDValue SetReg =
5013 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VTList: Op->getVTList(), N1: Op.getOperand(i: 0),
5014 N2: IntrinID, N3: RoundBothImm, N4: NewMode);
5015
5016 return SetReg;
5017}
5018
5019SDValue SITargetLowering::lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const {
5020 if (Op->isDivergent() &&
5021 (!Subtarget->hasVmemPrefInsts() || !Op.getConstantOperandVal(i: 4)))
5022 // Cannot do I$ prefetch with divergent pointer.
5023 return SDValue();
5024
5025 switch (cast<MemSDNode>(Val&: Op)->getAddressSpace()) {
5026 case AMDGPUAS::FLAT_ADDRESS:
5027 case AMDGPUAS::GLOBAL_ADDRESS:
5028 case AMDGPUAS::CONSTANT_ADDRESS:
5029 break;
5030 case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
5031 if (Subtarget->hasSafeSmemPrefetch())
5032 break;
5033 [[fallthrough]];
5034 default:
5035 return SDValue();
5036 }
5037
5038 // I$ prefetch
5039 if (!Subtarget->hasSafeSmemPrefetch() && !Op.getConstantOperandVal(i: 4))
5040 return SDValue();
5041
5042 return Op;
5043}
5044
5045// Work around DAG legality rules only based on the result type.
5046SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
5047 bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
5048 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
5049 EVT SrcVT = Src.getValueType();
5050
5051 if (SrcVT.getScalarType() != MVT::bf16)
5052 return Op;
5053
5054 SDLoc SL(Op);
5055 SDValue BitCast =
5056 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcVT.changeTypeToInteger(), Operand: Src);
5057
5058 EVT DstVT = Op.getValueType();
5059 if (IsStrict)
5060 llvm_unreachable("Need STRICT_BF16_TO_FP");
5061
5062 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SL, VT: DstVT, Operand: BitCast);
5063}
5064
5065SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5066 SDLoc SL(Op);
5067 if (Op.getValueType() != MVT::i64)
5068 return Op;
5069
5070 uint32_t ModeHwReg =
5071 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5072 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5073 uint32_t TrapHwReg =
5074 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5075 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5076
5077 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
5078 SDValue IntrinID =
5079 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
5080 SDValue GetModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5081 N1: Op.getOperand(i: 0), N2: IntrinID, N3: ModeHwRegImm);
5082 SDValue GetTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5083 N1: Op.getOperand(i: 0), N2: IntrinID, N3: TrapHwRegImm);
5084 SDValue TokenReg =
5085 DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: GetModeReg.getValue(R: 1),
5086 N2: GetTrapReg.getValue(R: 1));
5087
5088 SDValue CvtPtr =
5089 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: GetModeReg, N2: GetTrapReg);
5090 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
5091
5092 return DAG.getMergeValues(Ops: {Result, TokenReg}, dl: SL);
5093}
5094
5095SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5096 SDLoc SL(Op);
5097 if (Op.getOperand(i: 1).getValueType() != MVT::i64)
5098 return Op;
5099
5100 SDValue Input = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
5101 SDValue NewModeReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5102 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
5103 SDValue NewTrapReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5104 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
5105
5106 SDValue ReadFirstLaneID =
5107 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
5108 NewModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5109 N1: ReadFirstLaneID, N2: NewModeReg);
5110 NewTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5111 N1: ReadFirstLaneID, N2: NewTrapReg);
5112
5113 unsigned ModeHwReg =
5114 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5115 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5116 unsigned TrapHwReg =
5117 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5118 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5119
5120 SDValue IntrinID =
5121 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
5122 SDValue SetModeReg =
5123 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5124 N2: IntrinID, N3: ModeHwRegImm, N4: NewModeReg);
5125 SDValue SetTrapReg =
5126 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5127 N2: IntrinID, N3: TrapHwRegImm, N4: NewTrapReg);
5128 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: SetTrapReg, N2: SetModeReg);
5129}
5130
5131Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT,
5132 const MachineFunction &MF) const {
5133 Register Reg =
5134 StringSwitch<Register>(RegName)
5135 .Case(S: "m0", Value: AMDGPU::M0)
5136 .Case(S: "exec", Value: AMDGPU::EXEC)
5137 .Case(S: "exec_lo", Value: AMDGPU::EXEC_LO)
5138 .Case(S: "exec_hi", Value: AMDGPU::EXEC_HI)
5139 .Case(S: "flat_scratch", Value: AMDGPU::FLAT_SCR)
5140 .Case(S: "flat_scratch_lo", Value: AMDGPU::FLAT_SCR_LO)
5141 .Case(S: "flat_scratch_hi", Value: AMDGPU::FLAT_SCR_HI)
5142 .Case(S: "src_flat_scratch_base", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE)
5143 .Case(S: "src_flat_scratch_base_lo", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
5144 .Case(S: "src_flat_scratch_base_hi", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
5145 .Default(Value: Register());
5146 if (!Reg)
5147 return Reg;
5148
5149 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
5150 if (!Subtarget->hasFlatScrRegister() &&
5151 TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::FLAT_SCR))
5152 return Register();
5153
5154 if (!Subtarget->hasGloballyAddressableScratch() &&
5155 TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::SRC_FLAT_SCRATCH_BASE))
5156 return Register();
5157
5158 switch (Reg) {
5159 case AMDGPU::M0:
5160 case AMDGPU::EXEC_LO:
5161 case AMDGPU::EXEC_HI:
5162 case AMDGPU::FLAT_SCR_LO:
5163 case AMDGPU::FLAT_SCR_HI:
5164 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
5165 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
5166 if (VT.getSizeInBits() == 32)
5167 return Reg;
5168 break;
5169 case AMDGPU::EXEC:
5170 case AMDGPU::FLAT_SCR:
5171 case AMDGPU::SRC_FLAT_SCRATCH_BASE:
5172 if (VT.getSizeInBits() == 64)
5173 return Reg;
5174 break;
5175 default:
5176 llvm_unreachable("missing register type checking");
5177 }
5178
5179 report_fatal_error(
5180 reason: Twine("invalid type for register \"" + StringRef(RegName) + "\"."));
5181}
5182
5183// If kill is not the last instruction, split the block so kill is always a
5184// proper terminator.
5185MachineBasicBlock *
5186SITargetLowering::splitKillBlock(MachineInstr &MI,
5187 MachineBasicBlock *BB) const {
5188 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, /*UpdateLiveIns=*/true);
5189 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5190 MI.setDesc(TII->getKillTerminatorFromPseudo(Opcode: MI.getOpcode()));
5191 return SplitBB;
5192}
5193
5194// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
5195// \p MI will be the only instruction in the loop body block. Otherwise, it will
5196// be the first instruction in the remainder block.
5197//
5198/// \returns { LoopBody, Remainder }
5199static std::pair<MachineBasicBlock *, MachineBasicBlock *>
5200splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
5201 MachineFunction *MF = MBB.getParent();
5202 MachineBasicBlock::iterator I(&MI);
5203
5204 // To insert the loop we need to split the block. Move everything after this
5205 // point to a new block, and insert a new empty block between the two.
5206 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
5207 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
5208 MachineFunction::iterator MBBI(MBB);
5209 ++MBBI;
5210
5211 MF->insert(MBBI, MBB: LoopBB);
5212 MF->insert(MBBI, MBB: RemainderBB);
5213
5214 LoopBB->addSuccessor(Succ: LoopBB);
5215 LoopBB->addSuccessor(Succ: RemainderBB);
5216
5217 // Move the rest of the block into a new block.
5218 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
5219
5220 if (InstInLoop) {
5221 auto Next = std::next(x: I);
5222
5223 // Move instruction to loop body.
5224 LoopBB->splice(Where: LoopBB->begin(), Other: &MBB, From: I, To: Next);
5225
5226 // Move the rest of the block.
5227 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Next, To: MBB.end());
5228 } else {
5229 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: I, To: MBB.end());
5230 }
5231
5232 MBB.addSuccessor(Succ: LoopBB);
5233
5234 return std::pair(LoopBB, RemainderBB);
5235}
5236
5237/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
5238void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
5239 MachineBasicBlock *MBB = MI.getParent();
5240 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5241 auto I = MI.getIterator();
5242 auto E = std::next(x: I);
5243
5244 // clang-format off
5245 BuildMI(BB&: *MBB, I: E, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_WAITCNT))
5246 .addImm(Val: 0);
5247 // clang-format on
5248
5249 MIBundleBuilder Bundler(*MBB, I, E);
5250 finalizeBundle(MBB&: *MBB, FirstMI: Bundler.begin());
5251}
5252
5253MachineBasicBlock *
5254SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
5255 MachineBasicBlock *BB) const {
5256 const DebugLoc &DL = MI.getDebugLoc();
5257
5258 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5259
5260 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5261
5262 // Apparently kill flags are only valid if the def is in the same block?
5263 if (MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0))
5264 Src->setIsKill(false);
5265
5266 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB&: *BB, InstInLoop: true);
5267
5268 MachineBasicBlock::iterator I = LoopBB->end();
5269
5270 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
5271 Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: AMDGPU::Hwreg::OFFSET_MEM_VIOL, Values: 1);
5272
5273 // Clear TRAP_STS.MEM_VIOL
5274 BuildMI(BB&: *LoopBB, I: LoopBB->begin(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
5275 .addImm(Val: 0)
5276 .addImm(Val: EncodedReg);
5277
5278 bundleInstWithWaitcnt(MI);
5279
5280 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5281
5282 // Load and check TRAP_STS.MEM_VIOL
5283 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: Reg)
5284 .addImm(Val: EncodedReg);
5285
5286 // clang-format off
5287 // FIXME: Do we need to use an isel pseudo that may clobber scc?
5288 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
5289 .addReg(RegNo: Reg)
5290 .addImm(Val: 0);
5291 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
5292 .addMBB(MBB: LoopBB);
5293 // clang-format on
5294
5295 return RemainderBB;
5296}
5297
5298// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
5299// wavefront. If the value is uniform and just happens to be in a VGPR, this
5300// will only do one iteration. In the worst case, this will loop 64 times.
5301//
5302// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
5303static MachineBasicBlock::iterator
5304emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
5305 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
5306 const DebugLoc &DL, const MachineOperand &Idx,
5307 unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
5308 unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
5309 Register &SGPRIdxReg) {
5310
5311 MachineFunction *MF = OrigBB.getParent();
5312 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5313 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5314 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5315 MachineBasicBlock::iterator I = LoopBB.begin();
5316
5317 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5318 Register PhiExec = MRI.createVirtualRegister(RegClass: BoolRC);
5319 Register NewExec = MRI.createVirtualRegister(RegClass: BoolRC);
5320 Register CurrentIdxReg =
5321 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5322 Register CondReg = MRI.createVirtualRegister(RegClass: BoolRC);
5323
5324 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiReg)
5325 .addReg(RegNo: InitReg)
5326 .addMBB(MBB: &OrigBB)
5327 .addReg(RegNo: ResultReg)
5328 .addMBB(MBB: &LoopBB);
5329
5330 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
5331 .addReg(RegNo: InitSaveExecReg)
5332 .addMBB(MBB: &OrigBB)
5333 .addReg(RegNo: NewExec)
5334 .addMBB(MBB: &LoopBB);
5335
5336 // Read the next variant <- also loop target.
5337 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurrentIdxReg)
5338 .addReg(RegNo: Idx.getReg(), Flags: getUndefRegState(B: Idx.isUndef()));
5339
5340 // Compare the just read M0 value to all possible Idx values.
5341 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CondReg)
5342 .addReg(RegNo: CurrentIdxReg)
5343 .addReg(RegNo: Idx.getReg(), Flags: {}, SubReg: Idx.getSubReg());
5344
5345 // Update EXEC, save the original EXEC value to VCC.
5346 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.AndSaveExecOpc), DestReg: NewExec)
5347 .addReg(RegNo: CondReg)
5348 .setOperandDead(3); // Dead scc
5349
5350 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
5351
5352 if (UseGPRIdxMode) {
5353 if (Offset == 0) {
5354 SGPRIdxReg = CurrentIdxReg;
5355 } else {
5356 SGPRIdxReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
5357 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: SGPRIdxReg)
5358 .addReg(RegNo: CurrentIdxReg)
5359 .addImm(Val: Offset)
5360 .setOperandDead(3); // Dead scc
5361 }
5362 } else {
5363 // Move index from VCC into M0
5364 if (Offset == 0) {
5365 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5366 .addReg(RegNo: CurrentIdxReg);
5367 } else {
5368 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5369 .addReg(RegNo: CurrentIdxReg)
5370 .addImm(Val: Offset)
5371 .setOperandDead(3); // Dead scc
5372 }
5373 }
5374
5375 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
5376 MachineInstr *InsertPt =
5377 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
5378 .addReg(RegNo: LMC.ExecReg)
5379 .addReg(RegNo: NewExec)
5380 .setOperandDead(3); // Dead scc
5381
5382 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
5383 // s_cbranch_scc0?
5384
5385 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
5386 // clang-format off
5387 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
5388 .addMBB(MBB: &LoopBB);
5389 // clang-format on
5390
5391 return InsertPt->getIterator();
5392}
5393
5394// This has slightly sub-optimal regalloc when the source vector is killed by
5395// the read. The register allocator does not understand that the kill is
5396// per-workitem, so is kept alive for the whole loop so we end up not re-using a
5397// subregister from it, using 1 more VGPR than necessary. This was saved when
5398// this was expanded after register allocation.
5399static MachineBasicBlock::iterator
5400loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
5401 unsigned InitResultReg, unsigned PhiReg, int Offset,
5402 bool UseGPRIdxMode, Register &SGPRIdxReg) {
5403 MachineFunction *MF = MBB.getParent();
5404 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5405 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5406 MachineRegisterInfo &MRI = MF->getRegInfo();
5407 const DebugLoc &DL = MI.getDebugLoc();
5408 MachineBasicBlock::iterator I(&MI);
5409
5410 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
5411 Register DstReg = MI.getOperand(i: 0).getReg();
5412 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5413 Register TmpExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5414 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5415
5416 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: TmpExec);
5417
5418 // Save the EXEC mask
5419 // clang-format off
5420 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExec)
5421 .addReg(RegNo: LMC.ExecReg);
5422 // clang-format on
5423
5424 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, InstInLoop: false);
5425
5426 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5427
5428 auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, OrigBB&: MBB, LoopBB&: *LoopBB, DL, Idx: *Idx,
5429 InitReg: InitResultReg, ResultReg: DstReg, PhiReg, InitSaveExecReg: TmpExec,
5430 Offset, UseGPRIdxMode, SGPRIdxReg);
5431
5432 MachineBasicBlock *LandingPad = MF->CreateMachineBasicBlock();
5433 MachineFunction::iterator MBBI(LoopBB);
5434 ++MBBI;
5435 MF->insert(MBBI, MBB: LandingPad);
5436 LoopBB->removeSuccessor(Succ: RemainderBB);
5437 LandingPad->addSuccessor(Succ: RemainderBB);
5438 LoopBB->addSuccessor(Succ: LandingPad);
5439 MachineBasicBlock::iterator First = LandingPad->begin();
5440 // clang-format off
5441 BuildMI(BB&: *LandingPad, I: First, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
5442 .addReg(RegNo: SaveExec);
5443 // clang-format on
5444
5445 return InsPt;
5446}
5447
5448// Returns subreg index, offset
5449static std::pair<unsigned, int>
5450computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
5451 const TargetRegisterClass *SuperRC, unsigned VecReg,
5452 int Offset) {
5453 int NumElts = TRI.getRegSizeInBits(RC: *SuperRC) / 32;
5454
5455 // Skip out of bounds offsets, or else we would end up using an undefined
5456 // register.
5457 if (Offset >= NumElts || Offset < 0)
5458 return std::pair(AMDGPU::sub0, Offset);
5459
5460 return std::pair(SIRegisterInfo::getSubRegFromChannel(Channel: Offset), 0);
5461}
5462
5463static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
5464 MachineRegisterInfo &MRI, MachineInstr &MI,
5465 int Offset) {
5466 MachineBasicBlock *MBB = MI.getParent();
5467 const DebugLoc &DL = MI.getDebugLoc();
5468 MachineBasicBlock::iterator I(&MI);
5469
5470 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5471
5472 assert(Idx->getReg() != AMDGPU::NoRegister);
5473
5474 if (Offset == 0) {
5475 // clang-format off
5476 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5477 .add(MO: *Idx);
5478 // clang-format on
5479 } else {
5480 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5481 .add(MO: *Idx)
5482 .addImm(Val: Offset)
5483 .setOperandDead(3); // Dead scc
5484 }
5485}
5486
5487static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
5488 MachineRegisterInfo &MRI, MachineInstr &MI,
5489 int Offset) {
5490 MachineBasicBlock *MBB = MI.getParent();
5491 const DebugLoc &DL = MI.getDebugLoc();
5492 MachineBasicBlock::iterator I(&MI);
5493
5494 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5495
5496 if (Offset == 0)
5497 return Idx->getReg();
5498
5499 Register Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5500 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: Tmp)
5501 .add(MO: *Idx)
5502 .addImm(Val: Offset)
5503 .setOperandDead(3); // Dead scc
5504 return Tmp;
5505}
5506
5507static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
5508 MachineBasicBlock &MBB,
5509 const GCNSubtarget &ST) {
5510 const SIInstrInfo *TII = ST.getInstrInfo();
5511 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5512 MachineFunction *MF = MBB.getParent();
5513 MachineRegisterInfo &MRI = MF->getRegInfo();
5514
5515 Register Dst = MI.getOperand(i: 0).getReg();
5516 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5517 Register SrcReg = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src)->getReg();
5518 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5519
5520 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcReg);
5521 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5522
5523 unsigned SubReg;
5524 std::tie(args&: SubReg, args&: Offset) =
5525 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcReg, Offset);
5526
5527 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5528
5529 // Check for a SGPR index.
5530 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5531 MachineBasicBlock::iterator I(&MI);
5532 const DebugLoc &DL = MI.getDebugLoc();
5533
5534 if (UseGPRIdxMode) {
5535 // TODO: Look at the uses to avoid the copy. This may require rescheduling
5536 // to avoid interfering with other uses, so probably requires a new
5537 // optimization pass.
5538 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5539
5540 const MCInstrDesc &GPRIDXDesc =
5541 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5542 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5543 .addReg(RegNo: SrcReg)
5544 .addReg(RegNo: Idx)
5545 .addImm(Val: SubReg);
5546 } else {
5547 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5548
5549 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5550 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5551 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5552 }
5553
5554 MI.eraseFromParent();
5555
5556 return &MBB;
5557 }
5558
5559 // Control flow needs to be inserted if indexing with a VGPR.
5560 const DebugLoc &DL = MI.getDebugLoc();
5561 MachineBasicBlock::iterator I(&MI);
5562
5563 Register PhiReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5564 Register InitReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5565
5566 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: InitReg);
5567
5568 Register SGPRIdxReg;
5569 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: InitReg, PhiReg, Offset,
5570 UseGPRIdxMode, SGPRIdxReg);
5571
5572 MachineBasicBlock *LoopBB = InsPt->getParent();
5573
5574 if (UseGPRIdxMode) {
5575 const MCInstrDesc &GPRIDXDesc =
5576 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5577
5578 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5579 .addReg(RegNo: SrcReg)
5580 .addReg(RegNo: SGPRIdxReg)
5581 .addImm(Val: SubReg);
5582 } else {
5583 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5584 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5585 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5586 }
5587
5588 MI.eraseFromParent();
5589
5590 return LoopBB;
5591}
5592
5593static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
5594 MachineBasicBlock &MBB,
5595 const GCNSubtarget &ST) {
5596 const SIInstrInfo *TII = ST.getInstrInfo();
5597 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5598 MachineFunction *MF = MBB.getParent();
5599 MachineRegisterInfo &MRI = MF->getRegInfo();
5600
5601 Register Dst = MI.getOperand(i: 0).getReg();
5602 const MachineOperand *SrcVec = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src);
5603 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5604 const MachineOperand *Val = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::val);
5605 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5606 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcVec->getReg());
5607 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5608
5609 // This can be an immediate, but will be folded later.
5610 assert(Val->getReg());
5611
5612 unsigned SubReg;
5613 std::tie(args&: SubReg, args&: Offset) =
5614 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcVec->getReg(), Offset);
5615 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5616
5617 if (Idx->getReg() == AMDGPU::NoRegister) {
5618 MachineBasicBlock::iterator I(&MI);
5619 const DebugLoc &DL = MI.getDebugLoc();
5620
5621 assert(Offset == 0);
5622
5623 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dst)
5624 .add(MO: *SrcVec)
5625 .add(MO: *Val)
5626 .addImm(Val: SubReg);
5627
5628 MI.eraseFromParent();
5629 return &MBB;
5630 }
5631
5632 // Check for a SGPR index.
5633 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5634 MachineBasicBlock::iterator I(&MI);
5635 const DebugLoc &DL = MI.getDebugLoc();
5636
5637 if (UseGPRIdxMode) {
5638 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5639
5640 const MCInstrDesc &GPRIDXDesc =
5641 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5642 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5643 .addReg(RegNo: SrcVec->getReg())
5644 .add(MO: *Val)
5645 .addReg(RegNo: Idx)
5646 .addImm(Val: SubReg);
5647 } else {
5648 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5649
5650 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5651 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5652 BuildMI(BB&: MBB, I, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5653 .addReg(RegNo: SrcVec->getReg())
5654 .add(MO: *Val)
5655 .addImm(Val: SubReg);
5656 }
5657 MI.eraseFromParent();
5658 return &MBB;
5659 }
5660
5661 // Control flow needs to be inserted if indexing with a VGPR.
5662 if (Val->isReg())
5663 MRI.clearKillFlags(Reg: Val->getReg());
5664
5665 const DebugLoc &DL = MI.getDebugLoc();
5666
5667 Register PhiReg = MRI.createVirtualRegister(RegClass: VecRC);
5668
5669 Register SGPRIdxReg;
5670 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: SrcVec->getReg(), PhiReg, Offset,
5671 UseGPRIdxMode, SGPRIdxReg);
5672 MachineBasicBlock *LoopBB = InsPt->getParent();
5673
5674 if (UseGPRIdxMode) {
5675 const MCInstrDesc &GPRIDXDesc =
5676 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5677
5678 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5679 .addReg(RegNo: PhiReg)
5680 .add(MO: *Val)
5681 .addReg(RegNo: SGPRIdxReg)
5682 .addImm(Val: SubReg);
5683 } else {
5684 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5685 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5686 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5687 .addReg(RegNo: PhiReg)
5688 .add(MO: *Val)
5689 .addImm(Val: SubReg);
5690 }
5691
5692 MI.eraseFromParent();
5693 return LoopBB;
5694}
5695
5696static MachineBasicBlock *expand64BitScalarArithmetic(MachineInstr &MI,
5697 MachineBasicBlock *BB) {
5698 // For targets older than GFX12, we emit a sequence of 32-bit operations.
5699 // For GFX12, we emit s_add_u64 and s_sub_u64.
5700 MachineFunction *MF = BB->getParent();
5701 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
5702 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5703 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5704 const DebugLoc &DL = MI.getDebugLoc();
5705 MachineOperand &Dest = MI.getOperand(i: 0);
5706 MachineOperand &Src0 = MI.getOperand(i: 1);
5707 MachineOperand &Src1 = MI.getOperand(i: 2);
5708 bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5709 if (ST.hasScalarAddSub64()) {
5710 // FIXME: If scc is used, this deletes the def
5711 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5712 // clang-format off
5713 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg())
5714 .add(MO: Src0)
5715 .add(MO: Src1);
5716 // clang-format on
5717 } else {
5718 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5719 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5720
5721 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5722 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5723
5724 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5725 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5726 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5727 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5728
5729 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5730 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5731 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5732 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5733
5734 const MachineOperand &ImpDefSCC = MI.getOperand(i: 3);
5735 assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef());
5736
5737 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5738 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5739 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0).add(MO: Src0Sub0).add(MO: Src1Sub0);
5740 auto Hi = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1)
5741 .add(MO: Src0Sub1)
5742 .add(MO: Src1Sub1);
5743 if (ImpDefSCC.isDead())
5744 Hi.setOperandDead(3);
5745 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
5746 .addReg(RegNo: DestSub0)
5747 .addImm(Val: AMDGPU::sub0)
5748 .addReg(RegNo: DestSub1)
5749 .addImm(Val: AMDGPU::sub1);
5750 }
5751 MI.eraseFromParent();
5752 return BB;
5753}
5754
5755static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
5756 MachineFunction *MF = BB->getParent();
5757 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5758 const SIInstrInfo *TII = ST.getInstrInfo();
5759 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5760 MachineRegisterInfo &MRI = MF->getRegInfo();
5761 const DebugLoc &DL = MI.getDebugLoc();
5762 Register Dst = MI.getOperand(i: 0).getReg();
5763 const MachineOperand &Src0 = MI.getOperand(i: 1);
5764 const MachineOperand &Src1 = MI.getOperand(i: 2);
5765 Register SrcCond = MI.getOperand(i: 3).getReg();
5766
5767 Register DstLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5768 Register DstHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5769 const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
5770 Register SrcCondCopy = MRI.createVirtualRegister(RegClass: CondRC);
5771
5772 int Src0Idx =
5773 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
5774 int Src1Idx =
5775 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
5776 const TargetRegisterClass *Src0RC =
5777 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src0Idx));
5778 const TargetRegisterClass *Src1RC =
5779 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src1Idx));
5780
5781 const TargetRegisterClass *Src0SubRC =
5782 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5783 const TargetRegisterClass *Src1SubRC =
5784 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5785
5786 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5787 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
5788 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5789 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
5790
5791 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5792 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
5793 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5794 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
5795
5796 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SrcCondCopy).addReg(RegNo: SrcCond);
5797 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstLo)
5798 .addImm(Val: 0)
5799 .add(MO: Src0Sub0)
5800 .addImm(Val: 0)
5801 .add(MO: Src1Sub0)
5802 .addReg(RegNo: SrcCondCopy);
5803
5804 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstHi)
5805 .addImm(Val: 0)
5806 .add(MO: Src0Sub1)
5807 .addImm(Val: 0)
5808 .add(MO: Src1Sub1)
5809 .addReg(RegNo: SrcCondCopy);
5810
5811 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
5812 .addReg(RegNo: DstLo)
5813 .addImm(Val: AMDGPU::sub0)
5814 .addReg(RegNo: DstHi)
5815 .addImm(Val: AMDGPU::sub1);
5816 MI.eraseFromParent();
5817}
5818
5819static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
5820 switch (Opc) {
5821 case AMDGPU::S_MIN_U32:
5822 return std::numeric_limits<uint32_t>::max();
5823 case AMDGPU::S_MIN_I32:
5824 return std::numeric_limits<int32_t>::max();
5825 case AMDGPU::S_MAX_U32:
5826 return std::numeric_limits<uint32_t>::min();
5827 case AMDGPU::S_MAX_I32:
5828 return std::numeric_limits<int32_t>::min();
5829 case AMDGPU::V_ADD_F32_e64: // -0.0
5830 return 0x80000000;
5831 case AMDGPU::V_SUB_F32_e64: // +0.0
5832 return 0x0;
5833 case AMDGPU::S_ADD_I32:
5834 case AMDGPU::S_SUB_I32:
5835 case AMDGPU::S_OR_B32:
5836 case AMDGPU::S_XOR_B32:
5837 return std::numeric_limits<uint32_t>::min();
5838 case AMDGPU::S_AND_B32:
5839 return std::numeric_limits<uint32_t>::max();
5840 case AMDGPU::V_MIN_F32_e64:
5841 case AMDGPU::V_MAX_F32_e64:
5842 return 0x7fc00000; // qNAN
5843 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5844 return std::numeric_limits<uint64_t>::max();
5845 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5846 return std::numeric_limits<int64_t>::max();
5847 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5848 return std::numeric_limits<uint64_t>::min();
5849 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5850 return std::numeric_limits<int64_t>::min();
5851 case AMDGPU::V_MIN_F64_e64:
5852 case AMDGPU::V_MAX_F64_e64:
5853 case AMDGPU::V_MIN_NUM_F64_e64:
5854 case AMDGPU::V_MAX_NUM_F64_e64:
5855 return 0x7FF8000000000000; // qNAN
5856 case AMDGPU::S_ADD_U64_PSEUDO:
5857 case AMDGPU::S_SUB_U64_PSEUDO:
5858 case AMDGPU::S_OR_B64:
5859 case AMDGPU::S_XOR_B64:
5860 return std::numeric_limits<uint64_t>::min();
5861 case AMDGPU::S_AND_B64:
5862 return std::numeric_limits<uint64_t>::max();
5863 case AMDGPU::V_ADD_F64_e64:
5864 case AMDGPU::V_ADD_F64_pseudo_e64:
5865 return 0x8000000000000000; // -0.0
5866 default:
5867 llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
5868 }
5869}
5870
5871static bool is32bitWaveReduceOperation(unsigned Opc) {
5872 return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
5873 Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
5874 Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
5875 Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
5876 Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
5877 Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
5878 Opc == AMDGPU::V_SUB_F32_e64;
5879}
5880
5881static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
5882 return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
5883 Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
5884 Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
5885 Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
5886 Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
5887}
5888
5889static std::tuple<unsigned, unsigned>
5890getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
5891 unsigned DPPOpc;
5892 switch (Opc) {
5893 case AMDGPU::S_MIN_U32:
5894 DPPOpc = AMDGPU::V_MIN_U32_dpp;
5895 break;
5896 case AMDGPU::S_MIN_I32:
5897 DPPOpc = AMDGPU::V_MIN_I32_dpp;
5898 break;
5899 case AMDGPU::S_MAX_U32:
5900 DPPOpc = AMDGPU::V_MAX_U32_dpp;
5901 break;
5902 case AMDGPU::S_MAX_I32:
5903 DPPOpc = AMDGPU::V_MAX_I32_dpp;
5904 break;
5905 case AMDGPU::S_ADD_I32:
5906 case AMDGPU::S_SUB_I32:
5907 DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
5908 : AMDGPU::V_ADD_CO_U32_dpp;
5909 break;
5910 case AMDGPU::S_AND_B32:
5911 DPPOpc = AMDGPU::V_AND_B32_dpp;
5912 break;
5913 case AMDGPU::S_OR_B32:
5914 DPPOpc = AMDGPU::V_OR_B32_dpp;
5915 break;
5916 case AMDGPU::S_XOR_B32:
5917 DPPOpc = AMDGPU::V_XOR_B32_dpp;
5918 break;
5919 case AMDGPU::V_ADD_F32_e64:
5920 case AMDGPU::V_SUB_F32_e64:
5921 DPPOpc = AMDGPU::V_ADD_F32_dpp;
5922 break;
5923 case AMDGPU::V_MIN_F32_e64:
5924 DPPOpc = AMDGPU::V_MIN_F32_dpp;
5925 break;
5926 case AMDGPU::V_MAX_F32_e64:
5927 DPPOpc = AMDGPU::V_MAX_F32_dpp;
5928 break;
5929 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5930 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5931 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5932 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5933 case AMDGPU::S_ADD_U64_PSEUDO:
5934 case AMDGPU::S_SUB_U64_PSEUDO:
5935 case AMDGPU::S_AND_B64:
5936 case AMDGPU::S_OR_B64:
5937 case AMDGPU::S_XOR_B64:
5938 case AMDGPU::V_MIN_NUM_F64_e64:
5939 case AMDGPU::V_MIN_F64_e64:
5940 case AMDGPU::V_MAX_NUM_F64_e64:
5941 case AMDGPU::V_MAX_F64_e64:
5942 case AMDGPU::V_ADD_F64_pseudo_e64:
5943 case AMDGPU::V_ADD_F64_e64:
5944 DPPOpc = AMDGPU::V_MOV_B64_DPP_PSEUDO;
5945 break;
5946 default:
5947 llvm_unreachable("unhandled lane op");
5948 }
5949 unsigned ClampOpc = Opc;
5950 if (!ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
5951 if (Opc == AMDGPU::S_SUB_I32)
5952 ClampOpc = AMDGPU::S_ADD_I32;
5953 if (Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO)
5954 ClampOpc = AMDGPU::V_ADD_CO_U32_e64;
5955 else if (Opc == AMDGPU::S_AND_B64)
5956 ClampOpc = AMDGPU::V_AND_B32_e64;
5957 else if (Opc == AMDGPU::S_OR_B64)
5958 ClampOpc = AMDGPU::V_OR_B32_e64;
5959 else if (Opc == AMDGPU::S_XOR_B64)
5960 ClampOpc = AMDGPU::V_XOR_B32_e64;
5961 else
5962 ClampOpc = ST.getInstrInfo()->getVALUOp(Opc: ClampOpc);
5963 }
5964 return {DPPOpc, ClampOpc};
5965}
5966
5967static std::pair<Register, Register>
5968ExtractSubRegs(MachineInstr &MI, MachineOperand &Op,
5969 const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
5970 MachineRegisterInfo &MRI) {
5971 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5972 const SIInstrInfo *TII = ST.getInstrInfo();
5973 const TargetRegisterClass *SrcSubRC =
5974 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
5975 Register Op1L =
5976 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
5977 Register Op1H =
5978 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
5979 return {Op1L, Op1H};
5980}
5981
5982static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
5983 MachineBasicBlock &BB,
5984 const GCNSubtarget &ST,
5985 unsigned Opc) {
5986 MachineRegisterInfo &MRI = BB.getParent()->getRegInfo();
5987 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5988 const DebugLoc &DL = MI.getDebugLoc();
5989 const SIInstrInfo *TII = ST.getInstrInfo();
5990
5991 // Reduction operations depend on whether the input operand is SGPR or VGPR.
5992 Register SrcReg = MI.getOperand(i: 1).getReg();
5993 bool isSGPR = TRI->isSGPRClass(RC: MRI.getRegClass(Reg: SrcReg));
5994 Register DstReg = MI.getOperand(i: 0).getReg();
5995 unsigned Stratergy = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
5996 enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
5997 MachineBasicBlock *RetBB = nullptr;
5998 unsigned MIOpc = MI.getOpcode();
5999 auto BuildRegSequence = [&](MachineBasicBlock &BB,
6000 MachineBasicBlock::iterator MI, Register Dst,
6001 Register Src0, Register Src1) {
6002 auto RegSequence =
6003 BuildMI(BB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
6004 .addReg(RegNo: Src0)
6005 .addImm(Val: AMDGPU::sub0)
6006 .addReg(RegNo: Src1)
6007 .addImm(Val: AMDGPU::sub1);
6008 return RegSequence;
6009 };
6010 if (isSGPR) {
6011 switch (Opc) {
6012 case AMDGPU::S_MIN_U32:
6013 case AMDGPU::S_MIN_I32:
6014 case AMDGPU::V_MIN_F32_e64:
6015 case AMDGPU::S_MAX_U32:
6016 case AMDGPU::S_MAX_I32:
6017 case AMDGPU::V_MAX_F32_e64:
6018 case AMDGPU::S_AND_B32:
6019 case AMDGPU::S_OR_B32: {
6020 // Idempotent operations.
6021 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstReg).addReg(RegNo: SrcReg);
6022 RetBB = &BB;
6023 break;
6024 }
6025 case AMDGPU::V_CMP_LT_U64_e64: // umin
6026 case AMDGPU::V_CMP_LT_I64_e64: // min
6027 case AMDGPU::V_CMP_GT_U64_e64: // umax
6028 case AMDGPU::V_CMP_GT_I64_e64: // max
6029 case AMDGPU::V_MIN_F64_e64:
6030 case AMDGPU::V_MIN_NUM_F64_e64:
6031 case AMDGPU::V_MAX_F64_e64:
6032 case AMDGPU::V_MAX_NUM_F64_e64:
6033 case AMDGPU::S_AND_B64:
6034 case AMDGPU::S_OR_B64: {
6035 // Idempotent operations.
6036 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg).addReg(RegNo: SrcReg);
6037 RetBB = &BB;
6038 break;
6039 }
6040 case AMDGPU::S_XOR_B32:
6041 case AMDGPU::S_XOR_B64:
6042 case AMDGPU::S_ADD_I32:
6043 case AMDGPU::S_ADD_U64_PSEUDO:
6044 case AMDGPU::V_ADD_F32_e64:
6045 case AMDGPU::V_ADD_F64_e64:
6046 case AMDGPU::V_ADD_F64_pseudo_e64:
6047 case AMDGPU::S_SUB_I32:
6048 case AMDGPU::S_SUB_U64_PSEUDO:
6049 case AMDGPU::V_SUB_F32_e64: {
6050 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6051 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
6052 Register ExecMask = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6053 Register NumActiveLanes =
6054 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6055
6056 bool IsWave32 = ST.isWave32();
6057 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6058 MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6059 unsigned BitCountOpc =
6060 IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
6061
6062 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MovOpc), DestReg: ExecMask).addReg(RegNo: ExecReg);
6063
6064 auto NewAccumulator =
6065 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BitCountOpc), DestReg: NumActiveLanes)
6066 .addReg(RegNo: ExecMask)
6067 .setOperandDead(2); // Dead scc
6068
6069 switch (Opc) {
6070 case AMDGPU::S_XOR_B32:
6071 case AMDGPU::S_XOR_B64: {
6072 // Performing an XOR operation on a uniform value
6073 // depends on the parity of the number of active lanes.
6074 // For even parity, the result will be 0, for odd
6075 // parity the result will be the same as the input value.
6076 Register ParityRegister =
6077 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6078 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_AND_B32), DestReg: ParityRegister)
6079 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6080 .addImm(Val: 1)
6081 .setOperandDead(3); // Dead scc
6082 // Check if Src is a known identity constant.
6083 MachineInstr *SrcDef = MRI.getVRegDef(Reg: SrcReg);
6084 if (SrcDef && SrcDef->isMoveImmediate()) {
6085 int64_t Imm = SrcDef->getOperand(i: 1).getImm();
6086 if (Imm == 1) { // 1 * parity(exec) = parity(exec)
6087 if (Opc == AMDGPU::S_XOR_B32) {
6088 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstReg)
6089 .addReg(RegNo: ParityRegister);
6090 } else {
6091 Register DstHi =
6092 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6093 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi).addImm(Val: 0);
6094 BuildRegSequence(BB, MI, DstReg, ParityRegister, DstHi);
6095 }
6096 break;
6097 }
6098 }
6099 if (Opc == AMDGPU::S_XOR_B32) {
6100 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6101 .addReg(RegNo: SrcReg)
6102 .addReg(RegNo: ParityRegister);
6103 } else {
6104 Register DestSub0 =
6105 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6106 Register DestSub1 =
6107 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6108 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6109 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6110 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6111 .addReg(RegNo: Op1L)
6112 .addReg(RegNo: ParityRegister);
6113 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub1)
6114 .addReg(RegNo: Op1H)
6115 .addReg(RegNo: ParityRegister);
6116 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6117 }
6118 break;
6119 }
6120 case AMDGPU::S_SUB_I32: {
6121 Register NegatedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6122 // Take the negation of the source operand.
6123 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedVal)
6124 .addImm(Val: 0)
6125 .addReg(RegNo: SrcReg)
6126 .setOperandDead(3); // Dead scc
6127 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6128 .addReg(RegNo: NegatedVal)
6129 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6130 break;
6131 }
6132 case AMDGPU::S_ADD_I32: {
6133 // Check if Src is a known identity constant.
6134 MachineInstr *SrcDef = MRI.getVRegDef(Reg: SrcReg);
6135 if (SrcDef && SrcDef->isMoveImmediate()) {
6136 int64_t Imm = SrcDef->getOperand(i: 1).getImm();
6137 if (Imm == 1) { // 1 * bitcount(exec) = bitcount(exec)
6138 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
6139 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6140 break;
6141 }
6142 }
6143 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6144 .addReg(RegNo: SrcReg)
6145 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6146 break;
6147 }
6148 case AMDGPU::S_ADD_U64_PSEUDO:
6149 case AMDGPU::S_SUB_U64_PSEUDO: {
6150 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6151 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6152 Register Op1H_Op0L_Reg =
6153 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6154 Register Op1L_Op0H_Reg =
6155 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6156 Register CarryReg =
6157 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6158 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6159 Register NegatedValLo =
6160 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6161 Register NegatedValHi =
6162 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6163 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6164 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6165 // Check if Src is a known identity constant.
6166 MachineInstr *SrcDef = MRI.getVRegDef(Reg: SrcReg);
6167 if (SrcDef && SrcDef->isMoveImmediate()) {
6168 int64_t Imm = SrcDef->getOperand(i: 1).getImm();
6169 if (Imm == 1 && Opc == AMDGPU::S_ADD_U64_PSEUDO) {
6170 // 1 * bitcount(exec) = bitcount(exec)
6171 Register DstHi =
6172 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6173 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi).addImm(Val: 0);
6174 BuildRegSequence(BB, MI, DstReg,
6175 NewAccumulator->getOperand(i: 0).getReg(), DstHi);
6176 break;
6177 }
6178 }
6179 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6180 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedValLo)
6181 .addImm(Val: 0)
6182 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6183 .setOperandDead(3); // Dead scc
6184 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ASHR_I32), DestReg: NegatedValHi)
6185 .addReg(RegNo: NegatedValLo)
6186 .addImm(Val: 31)
6187 .setOperandDead(3); // Dead scc
6188 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1L_Op0H_Reg)
6189 .addReg(RegNo: Op1L)
6190 .addReg(RegNo: NegatedValHi);
6191 }
6192 Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
6193 ? NegatedValLo
6194 : NewAccumulator->getOperand(i: 0).getReg();
6195 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6196 .addReg(RegNo: Op1L)
6197 .addReg(RegNo: LowOpcode);
6198 if (ST.hasScalarMulHiInsts()) {
6199 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_HI_U32), DestReg: CarryReg)
6200 .addReg(RegNo: Op1L)
6201 .addReg(RegNo: LowOpcode);
6202 } else {
6203 Register VCarryReg =
6204 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6205 Register LowOpVGPR =
6206 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6207 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LowOpVGPR)
6208 .addReg(RegNo: LowOpcode);
6209 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: VCarryReg)
6210 .addReg(RegNo: Op1L)
6211 .addReg(RegNo: LowOpVGPR);
6212 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CarryReg)
6213 .addReg(RegNo: VCarryReg);
6214 }
6215 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1H_Op0L_Reg)
6216 .addReg(RegNo: Op1H)
6217 .addReg(RegNo: LowOpcode);
6218
6219 Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
6220 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: HiVal)
6221 .addReg(RegNo: CarryReg)
6222 .addReg(RegNo: Op1H_Op0L_Reg)
6223 .setOperandDead(3); // Dead scc
6224
6225 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6226 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: DestSub1)
6227 .addReg(RegNo: HiVal)
6228 .addReg(RegNo: Op1L_Op0H_Reg)
6229 .setOperandDead(3); // Dead scc
6230 }
6231 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6232 break;
6233 }
6234 case AMDGPU::V_ADD_F32_e64:
6235 case AMDGPU::V_ADD_F64_e64:
6236 case AMDGPU::V_ADD_F64_pseudo_e64:
6237 case AMDGPU::V_SUB_F32_e64: {
6238 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6239 const TargetRegisterClass *VregRC = TII->getRegClass(MCID: TII->get(Opcode: Opc), OpNum: 0);
6240 Register ActiveLanesVreg = MRI.createVirtualRegister(RegClass: VregRC);
6241 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6242 // Get number of active lanes as a float val.
6243 BuildMI(BB, I&: MI, MIMD: DL,
6244 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
6245 : AMDGPU::V_CVT_F64_I32_e64),
6246 DestReg: ActiveLanesVreg)
6247 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6248 .addImm(Val: 0) // clamp
6249 .addImm(Val: 0); // output-modifier
6250
6251 // Take negation of input for SUB reduction
6252 unsigned srcMod = (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6253 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
6254 ? SISrcMods::NEG
6255 : SISrcMods::NONE;
6256 unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
6257 : ST.getGeneration() >= AMDGPUSubtarget::GFX12
6258 ? AMDGPU::V_MUL_F64_pseudo_e64
6259 : AMDGPU::V_MUL_F64_e64;
6260 auto DestVregInst = BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MulOpc),
6261 DestReg: DstVreg)
6262 .addImm(Val: srcMod) // src0 modifier
6263 .addReg(RegNo: SrcReg)
6264 .addImm(Val: SISrcMods::NONE) // src1 modifier
6265 .addReg(RegNo: ActiveLanesVreg)
6266 .addImm(Val: SISrcMods::NONE) // clamp
6267 .addImm(Val: SISrcMods::NONE); // output-mod
6268 if (is32BitOpc) {
6269 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
6270 .addReg(RegNo: DstVreg);
6271 } else {
6272 Register LaneValueLoReg =
6273 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6274 Register LaneValueHiReg =
6275 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6276 auto [Op1L, Op1H] =
6277 ExtractSubRegs(MI, Op&: DestVregInst->getOperand(i: 0), SrcRC: VregRC, ST, MRI);
6278 // lane value input should be in an sgpr
6279 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6280 DestReg: LaneValueLoReg)
6281 .addReg(RegNo: Op1L);
6282 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6283 DestReg: LaneValueHiReg)
6284 .addReg(RegNo: Op1H);
6285 NewAccumulator =
6286 BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
6287 }
6288 }
6289 }
6290 RetBB = &BB;
6291 }
6292 }
6293 } else {
6294 MachineBasicBlock::iterator I = BB.end();
6295 Register SrcReg = MI.getOperand(i: 1).getReg();
6296 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6297 bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
6298 bool NeedsMovDPP = !is32BitOpc;
6299 // Create virtual registers required for lowering.
6300 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6301 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
6302 const TargetRegisterClass *SrcRegClass = MRI.getRegClass(Reg: SrcReg);
6303 bool IsWave32 = ST.isWave32();
6304 unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6305 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6306 if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
6307 !ST.hasDPP()) { // If target doesn't support DPP operations, default to
6308 // iterative stratergy
6309
6310 // To reduce the VGPR using iterative approach, we need to iterate
6311 // over all the active lanes. Lowering consists of ComputeLoop,
6312 // which iterate over only active lanes. We use copy of EXEC register
6313 // as induction variable and every active lane modifies it using bitset0
6314 // so that we will get the next active lane for next iteration.
6315
6316 // Create Control flow for loop
6317 // Split MI's Machine Basic block into For loop
6318 auto [ComputeLoop, ComputeEnd] = splitBlockForLoop(MI, MBB&: BB, InstInLoop: true);
6319
6320 Register LoopIterator = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6321 Register IdentityValReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6322 Register AccumulatorReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6323 Register ActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6324 Register NewActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6325 Register FF1Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6326 Register LaneValueReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6327
6328 // Create initial values of induction variable from Exec, Accumulator and
6329 // insert branch instr to newly created ComputeBlock
6330 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: MovOpcForExec), DestReg: LoopIterator).addReg(RegNo: ExecReg);
6331 uint64_t IdentityValue =
6332 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6333 ? 0x0 // +0.0 for double sub reduction
6334 : getIdentityValueForWaveReduction(Opc);
6335 BuildMI(BB, I, MIMD: DL,
6336 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6337 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6338 DestReg: IdentityValReg)
6339 .addImm(Val: IdentityValue);
6340 // clang-format off
6341 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_BRANCH))
6342 .addMBB(MBB: ComputeLoop);
6343 // clang-format on
6344
6345 // Start constructing ComputeLoop
6346 I = ComputeLoop->begin();
6347 auto Accumulator =
6348 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: AccumulatorReg)
6349 .addReg(RegNo: IdentityValReg)
6350 .addMBB(MBB: &BB);
6351 auto ActiveBits =
6352 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: ActiveBitsReg)
6353 .addReg(RegNo: LoopIterator)
6354 .addMBB(MBB: &BB);
6355
6356 I = ComputeLoop->end();
6357 MachineInstr *NewAccumulator;
6358 // Perform the computations
6359 unsigned SFFOpc =
6360 IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
6361 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: SFFOpc), DestReg: FF1Reg)
6362 .addReg(RegNo: ActiveBitsReg);
6363 if (is32BitOpc) {
6364 Register OpDstReg = DstReg;
6365 bool hasSrc0Modifier = AMDGPU::getNamedOperandIdx(
6366 Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers) != -1;
6367 bool hasSrc1Modifier = AMDGPU::getNamedOperandIdx(
6368 Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers) != -1;
6369 bool hasClamp =
6370 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::clamp) != -1;
6371 bool hasOpSel =
6372 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel) != -1;
6373 bool hasOMod =
6374 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::omod) != -1;
6375 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6376 DestReg: LaneValueReg)
6377 .addReg(RegNo: SrcReg)
6378 .addReg(RegNo: FF1Reg);
6379 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
6380 // Get the Lane Value in VGPR to avoid the Constant Bus Restriction
6381 Register LaneValVgpr = MRI.createVirtualRegister(RegClass: SrcRegClass);
6382 Register VgprResultReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6383 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LaneValVgpr)
6384 .addReg(RegNo: LaneValueReg);
6385 OpDstReg = VgprResultReg;
6386 LaneValueReg = LaneValVgpr;
6387 }
6388 auto OpInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: OpDstReg);
6389 if (hasSrc0Modifier)
6390 OpInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6391 OpInstr.addReg(RegNo: AccumulatorReg); // src0
6392 if (hasSrc1Modifier)
6393 OpInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6394 OpInstr.addReg(RegNo: LaneValueReg); // src1
6395 if (hasClamp)
6396 OpInstr.addImm(Val: 0); // clamp
6397 if (hasOpSel)
6398 OpInstr.addImm(Val: 0); // opsel
6399 if (hasOMod)
6400 OpInstr.addImm(Val: 0); // omod
6401 if (TII->isSALU(Opcode: Opc))
6402 OpInstr.setOperandDead(3); // Dead scc
6403 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
6404 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6405 DestReg: DstReg)
6406 .addReg(RegNo: OpDstReg);
6407 }
6408 } else {
6409 Register LaneValueLoReg =
6410 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6411 Register LaneValueHiReg =
6412 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6413 Register LaneValReg =
6414 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6415 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6416 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6417 // lane value input should be in an sgpr
6418 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6419 DestReg: LaneValueLoReg)
6420 .addReg(RegNo: Op1L)
6421 .addReg(RegNo: FF1Reg);
6422 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6423 DestReg: LaneValueHiReg)
6424 .addReg(RegNo: Op1H)
6425 .addReg(RegNo: FF1Reg);
6426 auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
6427 LaneValueLoReg, LaneValueHiReg);
6428 switch (Opc) {
6429 case AMDGPU::S_OR_B64:
6430 case AMDGPU::S_AND_B64:
6431 case AMDGPU::S_XOR_B64: {
6432 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6433 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6434 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6435 .setOperandDead(3); // Dead scc
6436 break;
6437 }
6438 case AMDGPU::V_CMP_GT_I64_e64:
6439 case AMDGPU::V_CMP_GT_U64_e64:
6440 case AMDGPU::V_CMP_LT_I64_e64:
6441 case AMDGPU::V_CMP_LT_U64_e64: {
6442 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6443 Register ComparisonResultReg =
6444 MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6445 int SrcIdx =
6446 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6447 const TargetRegisterClass *VregClass =
6448 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6449 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregClass);
6450 auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
6451 MI, Op&: Accumulator->getOperand(i: 0), SrcRC: VregClass, ST, MRI);
6452 BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
6453 SrcReg0Sub1);
6454 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: LaneMaskReg)
6455 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6456 .addReg(RegNo: AccumulatorVReg);
6457
6458 unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6459 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AndOpc), DestReg: ComparisonResultReg)
6460 .addReg(RegNo: LaneMaskReg)
6461 .addReg(RegNo: ActiveBitsReg);
6462
6463 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6464 MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
6465 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6466 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6467 break;
6468 }
6469 case AMDGPU::V_MIN_F64_e64:
6470 case AMDGPU::V_MIN_NUM_F64_e64:
6471 case AMDGPU::V_MAX_F64_e64:
6472 case AMDGPU::V_MAX_NUM_F64_e64:
6473 case AMDGPU::V_ADD_F64_e64:
6474 case AMDGPU::V_ADD_F64_pseudo_e64: {
6475 int SrcIdx =
6476 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6477 const TargetRegisterClass *VregRC =
6478 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6479 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregRC);
6480 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6481 Register LaneValLo =
6482 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6483 Register LaneValHi =
6484 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6485 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AccumulatorVReg)
6486 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6487 unsigned Modifier =
6488 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6489 ? SISrcMods::NEG
6490 : SISrcMods::NONE;
6491 auto DstVregInst =
6492 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstVreg)
6493 .addImm(Val: Modifier) // src0 modifiers
6494 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6495 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6496 .addReg(RegNo: AccumulatorVReg)
6497 .addImm(Val: SISrcMods::NONE) // clamp
6498 .addImm(Val: SISrcMods::NONE); // omod
6499 auto ReadLaneLo =
6500 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6501 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValLo);
6502 auto ReadLaneHi =
6503 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6504 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValHi);
6505 MachineBasicBlock::iterator Iters = *ReadLaneLo;
6506 auto [Op1L, Op1H] = ExtractSubRegs(MI&: *Iters, Op&: DstVregInst->getOperand(i: 0),
6507 SrcRC: VregRC, ST, MRI);
6508 ReadLaneLo.addReg(RegNo: Op1L);
6509 ReadLaneHi.addReg(RegNo: Op1H);
6510 NewAccumulator =
6511 BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
6512 break;
6513 }
6514 case AMDGPU::S_ADD_U64_PSEUDO:
6515 case AMDGPU::S_SUB_U64_PSEUDO: {
6516 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6517 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6518 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6519 .setOperandDead(3); // Dead scc
6520 ComputeLoop =
6521 expand64BitScalarArithmetic(MI&: *NewAccumulator, BB: ComputeLoop);
6522 break;
6523 }
6524 }
6525 }
6526 // Manipulate the iterator to get the next active lane
6527 unsigned BITSETOpc =
6528 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
6529 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: BITSETOpc), DestReg: NewActiveBitsReg)
6530 .addReg(RegNo: FF1Reg)
6531 .addReg(RegNo: ActiveBitsReg);
6532
6533 // Add phi nodes
6534 Accumulator.addReg(RegNo: DstReg).addMBB(MBB: ComputeLoop);
6535 ActiveBits.addReg(RegNo: NewActiveBitsReg).addMBB(MBB: ComputeLoop);
6536
6537 // Creating branching
6538 MachineInstrBuilder SetSCCInstr;
6539 if (!ST.hasScalarCompareEq64()) {
6540 // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
6541 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6542 unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
6543 SetSCCInstr =
6544 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc), DestReg: LaneMaskReg);
6545 } else {
6546 unsigned CMPOpc =
6547 IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
6548 SetSCCInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc));
6549 }
6550 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6551 if (ST.hasScalarCompareEq64())
6552 SetSCCInstr.addImm(Val: 0);
6553 else
6554 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6555 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
6556 .addMBB(MBB: ComputeLoop);
6557
6558 RetBB = ComputeEnd;
6559 } else {
6560 assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
6561 MachineBasicBlock *CurrBB = &BB;
6562 Register SrcWithIdentity = MRI.createVirtualRegister(RegClass: SrcRegClass);
6563 Register IdentityVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6564 Register IdentitySGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6565 Register DPPRowShr1 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6566 Register DPPRowShr2 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6567 Register DPPRowShr4 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6568 Register DPPRowShr8 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6569 Register RowBcast15 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6570 Register ReducedValSGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6571 Register NegatedReducedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6572 Register RowBcast31 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6573 Register UndefExec = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6574 Register FinalDPPResult;
6575 MachineInstr *SrcWithIdentityInstr;
6576 MachineInstr *LastBcastInstr;
6577 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExec);
6578
6579 uint64_t IdentityValue = getIdentityValueForWaveReduction(Opc);
6580 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL,
6581 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6582 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6583 DestReg: IdentitySGPR)
6584 .addImm(Val: IdentityValue);
6585 auto IdentityCopyInstr =
6586 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: IdentityVGPR)
6587 .addReg(RegNo: IdentitySGPR);
6588 unsigned DPPOpc, ClampOpc;
6589 std::tie(args&: DPPOpc, args&: ClampOpc) = getDPPOpcForWaveReduction(Opc, ST);
6590 auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
6591 Register Src1) {
6592 return BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
6593 DestReg: Dst)
6594 .addImm(Val: 0) // src0 modifiers
6595 .addReg(RegNo: Src0) // src0
6596 .addImm(Val: 0) // src1 modifiers
6597 .addReg(RegNo: Src1) // identity value for inactive lanes
6598 .addReg(RegNo: UndefExec); // bool i1
6599 };
6600 auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
6601 unsigned DPPCtrl) {
6602 auto DPPInstr =
6603 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: DPPOpc), DestReg: Dst).addReg(RegNo: Src); // old
6604 if (isFPOp && !NeedsMovDPP)
6605 DPPInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6606 DPPInstr.addReg(RegNo: Src); // src0
6607 if (isFPOp && !NeedsMovDPP)
6608 DPPInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6609 if (!NeedsMovDPP)
6610 DPPInstr.addReg(RegNo: Src); // src1
6611 if (AMDGPU::getNamedOperandIdx(Opcode: DPPOpc, Name: AMDGPU::OpName::clamp) >= 0)
6612 DPPInstr.addImm(Val: 0); // clamp
6613 DPPInstr
6614 .addImm(Val: DPPCtrl) // dpp-ctrl
6615 .addImm(Val: 0xf) // row-mask
6616 .addImm(Val: 0xf) // bank-mask
6617 .addImm(Val: 0); // bound-control
6618 };
6619 auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1,
6620 bool isAddSub = false,
6621 bool needsCarryIn = false,
6622 Register CarryIn = Register()) {
6623 unsigned InstrOpc = ClampOpc;
6624 Register CarryOutReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6625 if (needsCarryIn)
6626 InstrOpc = AMDGPU::V_ADDC_U32_e64;
6627 auto ClampInstr = BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InstrOpc), DestReg: Dst);
6628 if (isFPOp)
6629 ClampInstr.addImm(Val: SISrcMods::NONE); // src0 mod
6630 if (isAddSub) {
6631 if (needsCarryIn)
6632 ClampInstr.addReg(RegNo: CarryOutReg,
6633 Flags: RegState::Define |
6634 RegState::Dead); // killed carry-out reg
6635 else
6636 ClampInstr.addReg(RegNo: CarryOutReg, Flags: RegState::Define); // carry-out reg
6637 }
6638 ClampInstr.addReg(RegNo: Src0); // src0
6639 if (isFPOp)
6640 ClampInstr.addImm(Val: SISrcMods::NONE); // src1 mod
6641 ClampInstr.addReg(RegNo: Src1); // src1
6642 if (needsCarryIn)
6643 ClampInstr.addReg(RegNo: CarryIn); // carry-in reg
6644 if (AMDGPU::getNamedOperandIdx(Opcode: InstrOpc, Name: AMDGPU::OpName::clamp) >= 0)
6645 ClampInstr.addImm(Val: 0); // clamp
6646 if (isFPOp)
6647 ClampInstr.addImm(Val: 0); // omod
6648 LastBcastInstr = ClampInstr;
6649 return CarryOutReg;
6650 };
6651 auto BuildPostDPPInstr = [&](Register Src0, Register Src1) {
6652 bool isAddSubOpc =
6653 Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO;
6654 bool isBitWiseOpc = Opc == AMDGPU::S_AND_B64 ||
6655 Opc == AMDGPU::S_OR_B64 || Opc == AMDGPU::S_XOR_B64;
6656 Register ReturnReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6657 if (isAddSubOpc || isBitWiseOpc) {
6658 Register ResLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6659 Register ResHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6660 MachineOperand Src0Operand =
6661 MachineOperand::CreateReg(Reg: Src0, /*isDef=*/false);
6662 MachineOperand Src1Operand =
6663 MachineOperand::CreateReg(Reg: Src1, /*isDef=*/false);
6664 auto [Src0Lo, Src0Hi] =
6665 ExtractSubRegs(MI, Op&: Src0Operand, SrcRC: SrcRegClass, ST, MRI);
6666 auto [Src1Lo, Src1Hi] =
6667 ExtractSubRegs(MI, Op&: Src1Operand, SrcRC: SrcRegClass, ST, MRI);
6668 Register CarryReg = BuildClampInstr(
6669 ResLo, Src0Lo, Src1Lo, isAddSubOpc, /*needsCarryIn*/ false);
6670 BuildClampInstr(ResHi, Src0Hi, Src1Hi, isAddSubOpc,
6671 /*needsCarryIn*/ isAddSubOpc, CarryReg);
6672 BuildRegSequence(*CurrBB, MI, ReturnReg, ResLo, ResHi);
6673 } else {
6674 if (isFPOp) {
6675 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: ReturnReg)
6676 .addImm(Val: SISrcMods::NONE) // src0 modifiers
6677 .addReg(RegNo: Src0)
6678 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6679 .addReg(RegNo: Src1)
6680 .addImm(Val: SISrcMods::NONE) // clamp
6681 .addImm(Val: SISrcMods::NONE); // omod
6682 } else {
6683 Register CmpMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6684 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: CmpMaskReg)
6685 .addReg(RegNo: Src0) // src0
6686 .addReg(RegNo: Src1); // src1
6687 LastBcastInstr =
6688 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO),
6689 DestReg: ReturnReg)
6690 .addReg(RegNo: Src1) // src0
6691 .addReg(RegNo: Src0) // src1
6692 .addReg(RegNo: CmpMaskReg); // src2
6693 expand64BitV_CNDMASK(MI&: *LastBcastInstr, BB: CurrBB);
6694 }
6695 }
6696 return ReturnReg;
6697 };
6698
6699 // Set inactive lanes to the identity value.
6700 if (is32BitOpc) {
6701 SrcWithIdentityInstr =
6702 BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
6703 } else {
6704 Register SrcWithIdentitylo =
6705 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6706 Register SrcWithIdentityhi =
6707 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6708 auto [Reg0Sub0, Reg0Sub1] = ExtractSubRegs(
6709 MI, Op&: IdentityCopyInstr->getOperand(i: 0), SrcRC: SrcRegClass, ST, MRI);
6710 auto [SrcReg0Sub0, SrcReg0Sub1] =
6711 ExtractSubRegs(MI, Op&: MI.getOperand(i: 1), SrcRC: SrcRegClass, ST, MRI);
6712 MachineInstr *SetInactiveLoInstr =
6713 BuildSetInactiveInstr(SrcWithIdentitylo, SrcReg0Sub0, Reg0Sub0);
6714 MachineInstr *SetInactiveHiInstr =
6715 BuildSetInactiveInstr(SrcWithIdentityhi, SrcReg0Sub1, Reg0Sub1);
6716 SrcWithIdentityInstr =
6717 BuildRegSequence(*CurrBB, MI, SrcWithIdentity,
6718 SetInactiveLoInstr->getOperand(i: 0).getReg(),
6719 SetInactiveHiInstr->getOperand(i: 0).getReg());
6720 }
6721 // DPP reduction
6722 Register SrcWithIdentityReg =
6723 SrcWithIdentityInstr->getOperand(i: 0).getReg();
6724 BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentityReg,
6725 AMDGPU::DPP::ROW_SHR_FIRST);
6726 if (NeedsMovDPP)
6727 DPPRowShr1 = BuildPostDPPInstr(SrcWithIdentityReg, DPPRowShr1);
6728
6729 BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
6730 (AMDGPU::DPP::ROW_SHR_FIRST + 1));
6731 if (NeedsMovDPP)
6732 DPPRowShr2 = BuildPostDPPInstr(DPPRowShr1, DPPRowShr2);
6733
6734 BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
6735 (AMDGPU::DPP::ROW_SHR_FIRST + 3));
6736 if (NeedsMovDPP)
6737 DPPRowShr4 = BuildPostDPPInstr(DPPRowShr2, DPPRowShr4);
6738
6739 BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
6740 (AMDGPU::DPP::ROW_SHR_FIRST + 7));
6741 if (NeedsMovDPP)
6742 DPPRowShr8 = BuildPostDPPInstr(DPPRowShr4, DPPRowShr8);
6743
6744 if (ST.hasDPPBroadcasts()) {
6745 BuildDPPMachineInstr(RowBcast15, DPPRowShr8, AMDGPU::DPP::BCAST15);
6746 if (NeedsMovDPP)
6747 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, RowBcast15);
6748 } else {
6749 // magic constant: 0x1E0
6750 // To Set BIT_MODE : bit 15 = 0
6751 // XOR mask : bit [14:10] = 0
6752 // OR mask : bit [9:5] = 15
6753 // AND mask : bit [4:0] = 0
6754 if (is32BitOpc) {
6755 Register SwizzledValue =
6756 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6757 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6758 DestReg: SwizzledValue)
6759 .addReg(RegNo: DPPRowShr8) // addr
6760 .addImm(Val: 0x1E0) // swizzle offset (i16)
6761 .addImm(Val: 0x0); // gds (i1)
6762 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
6763 } else {
6764 Register SwizzledValuelo =
6765 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6766 Register SwizzledValuehi =
6767 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6768 Register SwizzledValue64 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6769 MachineOperand DPPRowShr8Op =
6770 MachineOperand::CreateReg(Reg: DPPRowShr8, /*isDef=*/false);
6771 auto [Op1L, Op1H] =
6772 ExtractSubRegs(MI, Op&: DPPRowShr8Op, SrcRC: SrcRegClass, ST, MRI);
6773 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6774 DestReg: SwizzledValuelo)
6775 .addReg(RegNo: Op1L) // addr
6776 .addImm(Val: 0x1E0) // swizzle offset (i16)
6777 .addImm(Val: 0x0); // gds (i1)
6778 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6779 DestReg: SwizzledValuehi)
6780 .addReg(RegNo: Op1H) // addr
6781 .addImm(Val: 0x1E0) // swizzle offset (i16)
6782 .addImm(Val: 0x0); // gds (i1)
6783 BuildRegSequence(*CurrBB, MI, SwizzledValue64, SwizzledValuelo,
6784 SwizzledValuehi);
6785 if (NeedsMovDPP)
6786 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, SwizzledValue64);
6787 else
6788 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue64);
6789 }
6790 }
6791 FinalDPPResult = RowBcast15;
6792 if (!IsWave32) {
6793 if (ST.hasDPPBroadcasts()) {
6794 BuildDPPMachineInstr(RowBcast31, RowBcast15, AMDGPU::DPP::BCAST31);
6795 if (NeedsMovDPP)
6796 RowBcast31 = BuildPostDPPInstr(RowBcast15, RowBcast31);
6797 } else {
6798 Register ShiftedThreadID =
6799 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6800 Register PermuteByteOffset =
6801 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6802 Register PermutedValue = MRI.createVirtualRegister(RegClass: SrcRegClass);
6803 Register Lane32Offset =
6804 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6805 Register WordSizeConst =
6806 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6807 Register ThreadIDRegLo =
6808 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6809 Register ThreadIDReg =
6810 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6811 // Get the thread ID.
6812 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64),
6813 DestReg: ThreadIDRegLo)
6814 .addImm(Val: -1)
6815 .addImm(Val: 0);
6816 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_HI_U32_B32_e64),
6817 DestReg: ThreadIDReg)
6818 .addImm(Val: -1)
6819 .addReg(RegNo: ThreadIDRegLo);
6820 // shift each lane over by 32 positions, so value in 31st lane is
6821 // present in 63rd lane.
6822 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: Lane32Offset)
6823 .addImm(Val: 0x20);
6824 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U32_e64),
6825 DestReg: ShiftedThreadID)
6826 .addReg(RegNo: ThreadIDReg)
6827 .addReg(RegNo: Lane32Offset)
6828 .addImm(Val: 0); // clamp
6829 // multiply by reg size.
6830 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: WordSizeConst)
6831 .addImm(Val: 0x4);
6832 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_LO_U32_e64),
6833 DestReg: PermuteByteOffset)
6834 .addReg(RegNo: WordSizeConst)
6835 .addReg(RegNo: ShiftedThreadID);
6836 // Permute the lanes
6837 if (is32BitOpc) {
6838 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6839 DestReg: PermutedValue)
6840 .addReg(RegNo: PermuteByteOffset) // addr
6841 .addReg(RegNo: RowBcast15) // data
6842 .addImm(Val: 0); // offset
6843 } else {
6844 Register PermutedValuelo =
6845 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6846 Register PermutedValuehi =
6847 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6848 MachineOperand RowBcast15Op =
6849 MachineOperand::CreateReg(Reg: RowBcast15, /*isDef=*/false);
6850 auto [RowBcast15Lo, RowBcast15Hi] =
6851 ExtractSubRegs(MI, Op&: RowBcast15Op, SrcRC: SrcRegClass, ST, MRI);
6852 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6853 DestReg: PermutedValuelo)
6854 .addReg(RegNo: PermuteByteOffset) // addr
6855 .addReg(RegNo: RowBcast15Lo) // swizzle offset (i16)
6856 .addImm(Val: 0x0); // gds (i1)
6857 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6858 DestReg: PermutedValuehi)
6859 .addReg(RegNo: PermuteByteOffset) // addr
6860 .addReg(RegNo: RowBcast15Hi) // swizzle offset (i16)
6861 .addImm(Val: 0x0); // gds (i1)
6862 BuildRegSequence(*CurrBB, MI, PermutedValue, PermutedValuelo,
6863 PermutedValuehi);
6864 }
6865 if (NeedsMovDPP)
6866 RowBcast31 = BuildPostDPPInstr(RowBcast15, PermutedValue);
6867 else
6868 BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
6869 }
6870 FinalDPPResult = RowBcast31;
6871 }
6872 if (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6873 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64) {
6874 Register NegatedValVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6875 // Opc for f32 reduction is V_SUB_F32.
6876 // For f64, there is no equivalent V_SUB_F64 opcode, so use
6877 // V_ADD_F64/V_ADD_F64_pseudo, and negate the second operand.
6878 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc),
6879 DestReg: NegatedValVGPR)
6880 .addImm(Val: SISrcMods::NONE) // src0 mods
6881 .addReg(RegNo: IdentityVGPR) // src0
6882 .addImm(Val: is32BitOpc ? SISrcMods::NONE : SISrcMods::NEG) // src1 mods
6883 .addReg(RegNo: IsWave32 ? RowBcast15 : RowBcast31) // src1
6884 .addImm(Val: SISrcMods::NONE) // clamp
6885 .addImm(Val: SISrcMods::NONE); // omod
6886 FinalDPPResult = NegatedValVGPR;
6887 }
6888 // The final reduced value is in the last lane.
6889 if (is32BitOpc) {
6890 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6891 DestReg: ReducedValSGPR)
6892 .addReg(RegNo: FinalDPPResult)
6893 .addImm(Val: ST.getWavefrontSize() - 1);
6894 } else {
6895 Register LaneValueLoReg =
6896 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6897 Register LaneValueHiReg =
6898 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6899 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
6900 MachineOperand FinalDPPResultOperand =
6901 MachineOperand::CreateReg(Reg: FinalDPPResult, /*isDef=*/false);
6902 auto [Op1L, Op1H] =
6903 ExtractSubRegs(MI, Op&: FinalDPPResultOperand, SrcRC, ST, MRI);
6904 // lane value input should be in an sgpr
6905 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6906 DestReg: LaneValueLoReg)
6907 .addReg(RegNo: Op1L)
6908 .addImm(Val: ST.getWavefrontSize() - 1);
6909 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6910 DestReg: LaneValueHiReg)
6911 .addReg(RegNo: Op1H)
6912 .addImm(Val: ST.getWavefrontSize() - 1);
6913 BuildRegSequence(*CurrBB, MI, ReducedValSGPR, LaneValueLoReg,
6914 LaneValueHiReg);
6915 }
6916 if (Opc == AMDGPU::S_SUB_I32) {
6917 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedReducedVal)
6918 .addImm(Val: 0)
6919 .addReg(RegNo: ReducedValSGPR)
6920 .setOperandDead(3); // Dead scc
6921 } else if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6922 auto NegatedValInstr =
6923 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: NegatedReducedVal)
6924 .addImm(Val: 0)
6925 .addReg(RegNo: ReducedValSGPR)
6926 .setOperandDead(3); // Dead scc
6927 CurrBB = expand64BitScalarArithmetic(MI&: *NegatedValInstr, BB: CurrBB);
6928 }
6929 // Mark the final result as a whole-wave-mode calculation.
6930 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::STRICT_WWM), DestReg: DstReg)
6931 .addReg(RegNo: Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U64_PSEUDO
6932 ? NegatedReducedVal
6933 : ReducedValSGPR);
6934 RetBB = CurrBB;
6935 }
6936 }
6937 MI.eraseFromParent();
6938 return RetBB;
6939}
6940
6941MachineBasicBlock *
6942SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
6943 MachineBasicBlock *BB) const {
6944 MachineFunction *MF = BB->getParent();
6945 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
6946 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6947 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
6948 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
6949 MachineRegisterInfo &MRI = MF->getRegInfo();
6950 const DebugLoc &DL = MI.getDebugLoc();
6951
6952 switch (MI.getOpcode()) {
6953 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
6954 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_U32);
6955 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
6956 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_U64_e64);
6957 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
6958 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_I32);
6959 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
6960 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_I64_e64);
6961 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
6962 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MIN_F32_e64);
6963 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
6964 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6965 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6966 ? AMDGPU::V_MIN_NUM_F64_e64
6967 : AMDGPU::V_MIN_F64_e64);
6968 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
6969 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_U32);
6970 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
6971 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_U64_e64);
6972 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
6973 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_I32);
6974 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
6975 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_I64_e64);
6976 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
6977 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MAX_F32_e64);
6978 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
6979 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6980 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6981 ? AMDGPU::V_MAX_NUM_F64_e64
6982 : AMDGPU::V_MAX_F64_e64);
6983 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
6984 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_I32);
6985 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
6986 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_U64_PSEUDO);
6987 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
6988 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_ADD_F32_e64);
6989 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
6990 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6991 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6992 ? AMDGPU::V_ADD_F64_pseudo_e64
6993 : AMDGPU::V_ADD_F64_e64);
6994 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
6995 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_I32);
6996 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
6997 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_U64_PSEUDO);
6998 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
6999 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_SUB_F32_e64);
7000 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
7001 // There is no S/V_SUB_F64 opcode. Double type subtraction is expanded as
7002 // fadd + neg, by setting the NEG bit in the instruction.
7003 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
7004 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
7005 ? AMDGPU::V_ADD_F64_pseudo_e64
7006 : AMDGPU::V_ADD_F64_e64);
7007 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
7008 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B32);
7009 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
7010 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B64);
7011 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
7012 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B32);
7013 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
7014 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B64);
7015 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
7016 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B32);
7017 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
7018 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B64);
7019 case AMDGPU::S_UADDO_PSEUDO:
7020 case AMDGPU::S_USUBO_PSEUDO: {
7021 MachineOperand &Dest0 = MI.getOperand(i: 0);
7022 MachineOperand &Dest1 = MI.getOperand(i: 1);
7023 MachineOperand &Src0 = MI.getOperand(i: 2);
7024 MachineOperand &Src1 = MI.getOperand(i: 3);
7025
7026 unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
7027 ? AMDGPU::S_ADD_U32
7028 : AMDGPU::S_SUB_U32;
7029 // clang-format off
7030 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest0.getReg())
7031 .add(MO: Src0)
7032 .add(MO: Src1);
7033 // clang-format on
7034
7035 unsigned SelOpc =
7036 Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7037 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: Dest1.getReg()).addImm(Val: -1).addImm(Val: 0);
7038
7039 MI.eraseFromParent();
7040 return BB;
7041 }
7042 case AMDGPU::S_ADD_U64_PSEUDO:
7043 case AMDGPU::S_SUB_U64_PSEUDO: {
7044 return expand64BitScalarArithmetic(MI, BB);
7045 }
7046 case AMDGPU::V_ADD_U64_PSEUDO:
7047 case AMDGPU::V_SUB_U64_PSEUDO: {
7048 bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
7049
7050 MachineOperand &Dest = MI.getOperand(i: 0);
7051 MachineOperand &Src0 = MI.getOperand(i: 1);
7052 MachineOperand &Src1 = MI.getOperand(i: 2);
7053
7054 if (ST.hasAddSubU64Insts()) {
7055 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL,
7056 MCID: TII->get(Opcode: IsAdd ? AMDGPU::V_ADD_U64_e64
7057 : AMDGPU::V_SUB_U64_e64),
7058 DestReg: Dest.getReg())
7059 .add(MO: Src0)
7060 .add(MO: Src1)
7061 .addImm(Val: 0); // clamp
7062 TII->legalizeOperands(MI&: *I);
7063 MI.eraseFromParent();
7064 return BB;
7065 }
7066
7067 if (IsAdd && ST.hasLshlAddU64Inst()) {
7068 auto Add = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_LSHL_ADD_U64_e64),
7069 DestReg: Dest.getReg())
7070 .add(MO: Src0)
7071 .addImm(Val: 0)
7072 .add(MO: Src1);
7073 TII->legalizeOperands(MI&: *Add);
7074 MI.eraseFromParent();
7075 return BB;
7076 }
7077
7078 const auto *CarryRC = TRI->getWaveMaskRegClass();
7079
7080 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7081 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7082
7083 Register CarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
7084 Register DeadCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
7085
7086 const TargetRegisterClass *Src0RC = Src0.isReg()
7087 ? MRI.getRegClass(Reg: Src0.getReg())
7088 : &AMDGPU::VReg_64RegClass;
7089 const TargetRegisterClass *Src1RC = Src1.isReg()
7090 ? MRI.getRegClass(Reg: Src1.getReg())
7091 : &AMDGPU::VReg_64RegClass;
7092
7093 const TargetRegisterClass *Src0SubRC =
7094 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
7095 const TargetRegisterClass *Src1SubRC =
7096 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
7097
7098 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
7099 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
7100 MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
7101 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
7102
7103 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
7104 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
7105 MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
7106 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
7107
7108 unsigned LoOpc =
7109 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
7110 MachineInstr *LoHalf = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0)
7111 .addReg(RegNo: CarryReg, Flags: RegState::Define)
7112 .add(MO: SrcReg0Sub0)
7113 .add(MO: SrcReg1Sub0)
7114 .addImm(Val: 0); // clamp bit
7115
7116 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
7117 MachineInstr *HiHalf =
7118 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1)
7119 .addReg(RegNo: DeadCarryReg, Flags: RegState::Define | RegState::Dead)
7120 .add(MO: SrcReg0Sub1)
7121 .add(MO: SrcReg1Sub1)
7122 .addReg(RegNo: CarryReg)
7123 .addImm(Val: 0); // clamp bit
7124
7125 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
7126 .addReg(RegNo: DestSub0)
7127 .addImm(Val: AMDGPU::sub0)
7128 .addReg(RegNo: DestSub1)
7129 .addImm(Val: AMDGPU::sub1);
7130 TII->legalizeOperands(MI&: *LoHalf);
7131 TII->legalizeOperands(MI&: *HiHalf);
7132 MI.eraseFromParent();
7133 return BB;
7134 }
7135 case AMDGPU::S_ADD_CO_PSEUDO:
7136 case AMDGPU::S_SUB_CO_PSEUDO: {
7137 // This pseudo has a chance to be selected
7138 // only from uniform add/subcarry node. All the VGPR operands
7139 // therefore assumed to be splat vectors.
7140 MachineBasicBlock::iterator MII = MI;
7141 MachineOperand &Dest = MI.getOperand(i: 0);
7142 MachineOperand &CarryDest = MI.getOperand(i: 1);
7143 MachineOperand &Src0 = MI.getOperand(i: 2);
7144 MachineOperand &Src1 = MI.getOperand(i: 3);
7145 MachineOperand &Src2 = MI.getOperand(i: 4);
7146 if (Src0.isReg() && TRI->isVectorRegister(MRI, Reg: Src0.getReg())) {
7147 Register RegOp0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7148 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp0)
7149 .addReg(RegNo: Src0.getReg());
7150 Src0.setReg(RegOp0);
7151 }
7152 if (Src1.isReg() && TRI->isVectorRegister(MRI, Reg: Src1.getReg())) {
7153 Register RegOp1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7154 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp1)
7155 .addReg(RegNo: Src1.getReg());
7156 Src1.setReg(RegOp1);
7157 }
7158 Register RegOp2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7159 if (TRI->isVectorRegister(MRI, Reg: Src2.getReg())) {
7160 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp2)
7161 .addReg(RegNo: Src2.getReg());
7162 Src2.setReg(RegOp2);
7163 }
7164
7165 if (ST.isWave64()) {
7166 if (ST.hasScalarCompareEq64()) {
7167 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U64))
7168 .addReg(RegNo: Src2.getReg())
7169 .addImm(Val: 0);
7170 } else {
7171 const TargetRegisterClass *Src2RC = MRI.getRegClass(Reg: Src2.getReg());
7172 const TargetRegisterClass *SubRC =
7173 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
7174 MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
7175 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub0, SubRC);
7176 MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
7177 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub1, SubRC);
7178 Register Src2_32 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7179
7180 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_OR_B32), DestReg: Src2_32)
7181 .add(MO: Src2Sub0)
7182 .add(MO: Src2Sub1);
7183
7184 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7185 .addReg(RegNo: Src2_32)
7186 .addImm(Val: 0);
7187 }
7188 } else {
7189 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7190 .addReg(RegNo: Src2.getReg())
7191 .addImm(Val: 0);
7192 }
7193
7194 unsigned Opc = MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
7195 ? AMDGPU::S_ADDC_U32
7196 : AMDGPU::S_SUBB_U32;
7197
7198 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg()).add(MO: Src0).add(MO: Src1);
7199
7200 unsigned SelOpc =
7201 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7202
7203 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: CarryDest.getReg())
7204 .addImm(Val: -1)
7205 .addImm(Val: 0);
7206
7207 MI.eraseFromParent();
7208 return BB;
7209 }
7210 case AMDGPU::SI_INIT_M0: {
7211 MachineOperand &M0Init = MI.getOperand(i: 0);
7212 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7213 MCID: TII->get(Opcode: M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
7214 DestReg: AMDGPU::M0)
7215 .add(MO: M0Init);
7216 MI.eraseFromParent();
7217 return BB;
7218 }
7219 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
7220 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7221 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7222 MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7223 .addImm(Val: 0)
7224 .addImm(Val: 0);
7225 return BB;
7226 }
7227 case AMDGPU::GET_GROUPSTATICSIZE: {
7228 assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
7229 getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
7230 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32))
7231 .add(MO: MI.getOperand(i: 0))
7232 .addImm(Val: MFI->getLDSSize());
7233 MI.eraseFromParent();
7234 return BB;
7235 }
7236 case AMDGPU::GET_SHADERCYCLESHILO: {
7237 assert(MF->getSubtarget<GCNSubtarget>().hasShaderCyclesHiLoRegisters());
7238 // The algorithm is:
7239 //
7240 // hi1 = getreg(SHADER_CYCLES_HI)
7241 // lo1 = getreg(SHADER_CYCLES_LO)
7242 // hi2 = getreg(SHADER_CYCLES_HI)
7243 //
7244 // If hi1 == hi2 then there was no overflow and the result is hi2:lo1.
7245 // Otherwise there was overflow and the result is hi2:0. In both cases the
7246 // result should represent the actual time at some point during the sequence
7247 // of three getregs.
7248 using namespace AMDGPU::Hwreg;
7249 Register RegHi1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7250 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi1)
7251 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7252 Register RegLo1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7253 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegLo1)
7254 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES, Values: 0, Values: 32));
7255 Register RegHi2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7256 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi2)
7257 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7258 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7259 .addReg(RegNo: RegHi1)
7260 .addReg(RegNo: RegHi2);
7261 Register RegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7262 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: RegLo)
7263 .addReg(RegNo: RegLo1)
7264 .addImm(Val: 0);
7265 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE))
7266 .add(MO: MI.getOperand(i: 0))
7267 .addReg(RegNo: RegLo)
7268 .addImm(Val: AMDGPU::sub0)
7269 .addReg(RegNo: RegHi2)
7270 .addImm(Val: AMDGPU::sub1);
7271 MI.eraseFromParent();
7272 return BB;
7273 }
7274 case AMDGPU::SI_INDIRECT_SRC_V1:
7275 case AMDGPU::SI_INDIRECT_SRC_V2:
7276 case AMDGPU::SI_INDIRECT_SRC_V3:
7277 case AMDGPU::SI_INDIRECT_SRC_V4:
7278 case AMDGPU::SI_INDIRECT_SRC_V5:
7279 case AMDGPU::SI_INDIRECT_SRC_V6:
7280 case AMDGPU::SI_INDIRECT_SRC_V7:
7281 case AMDGPU::SI_INDIRECT_SRC_V8:
7282 case AMDGPU::SI_INDIRECT_SRC_V9:
7283 case AMDGPU::SI_INDIRECT_SRC_V10:
7284 case AMDGPU::SI_INDIRECT_SRC_V11:
7285 case AMDGPU::SI_INDIRECT_SRC_V12:
7286 case AMDGPU::SI_INDIRECT_SRC_V16:
7287 case AMDGPU::SI_INDIRECT_SRC_V32:
7288 return emitIndirectSrc(MI, MBB&: *BB, ST: *getSubtarget());
7289 case AMDGPU::SI_INDIRECT_DST_V1:
7290 case AMDGPU::SI_INDIRECT_DST_V2:
7291 case AMDGPU::SI_INDIRECT_DST_V3:
7292 case AMDGPU::SI_INDIRECT_DST_V4:
7293 case AMDGPU::SI_INDIRECT_DST_V5:
7294 case AMDGPU::SI_INDIRECT_DST_V6:
7295 case AMDGPU::SI_INDIRECT_DST_V7:
7296 case AMDGPU::SI_INDIRECT_DST_V8:
7297 case AMDGPU::SI_INDIRECT_DST_V9:
7298 case AMDGPU::SI_INDIRECT_DST_V10:
7299 case AMDGPU::SI_INDIRECT_DST_V11:
7300 case AMDGPU::SI_INDIRECT_DST_V12:
7301 case AMDGPU::SI_INDIRECT_DST_V16:
7302 case AMDGPU::SI_INDIRECT_DST_V32:
7303 return emitIndirectDst(MI, MBB&: *BB, ST: *getSubtarget());
7304 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
7305 case AMDGPU::SI_KILL_I1_PSEUDO:
7306 return splitKillBlock(MI, BB);
7307 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
7308 expand64BitV_CNDMASK(MI, BB);
7309 return BB;
7310 }
7311 case AMDGPU::SI_BR_UNDEF: {
7312 MachineInstr *Br = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
7313 .add(MO: MI.getOperand(i: 0));
7314 Br->getOperand(i: 1).setIsUndef(); // read undef SCC
7315 MI.eraseFromParent();
7316 return BB;
7317 }
7318 case AMDGPU::ADJCALLSTACKUP:
7319 case AMDGPU::ADJCALLSTACKDOWN: {
7320 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
7321 MachineInstrBuilder MIB(*MF, &MI);
7322 MIB.addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::ImplicitDefine)
7323 .addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::Implicit);
7324 return BB;
7325 }
7326 case AMDGPU::SI_CALL_ISEL: {
7327 unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(MF: *MF);
7328
7329 MachineInstrBuilder MIB;
7330 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_CALL))
7331 .addDef(RegNo: ReturnAddrReg, Flags: RegState::Dead);
7332
7333 for (const MachineOperand &MO : MI.operands())
7334 MIB.add(MO);
7335
7336 MIB.cloneMemRefs(OtherMI: MI);
7337 MIB.setMIFlags(MI.getFlags());
7338 MI.eraseFromParent();
7339 return BB;
7340 }
7341 case AMDGPU::V_ADDC_U32_e32:
7342 case AMDGPU::V_SUBB_U32_e32:
7343 case AMDGPU::V_SUBBREV_U32_e32:
7344 // These instructions have an implicit use of vcc which counts towards the
7345 // constant bus limit.
7346 TII->legalizeOperands(MI);
7347 return BB;
7348 case AMDGPU::DS_GWS_INIT:
7349 case AMDGPU::DS_GWS_SEMA_BR:
7350 case AMDGPU::DS_GWS_BARRIER:
7351 case AMDGPU::DS_GWS_SEMA_V:
7352 case AMDGPU::DS_GWS_SEMA_P:
7353 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
7354 // A s_waitcnt 0 is required to be the instruction immediately following.
7355 if (getSubtarget()->hasGWSAutoReplay()) {
7356 bundleInstWithWaitcnt(MI);
7357 return BB;
7358 }
7359
7360 return emitGWSMemViolTestLoop(MI, BB);
7361 case AMDGPU::S_SETREG_B32: {
7362 // Try to optimize cases that only set the denormal mode or rounding mode.
7363 //
7364 // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
7365 // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
7366 // instead.
7367 //
7368 // FIXME: This could be predicates on the immediate, but tablegen doesn't
7369 // allow you to have a no side effect instruction in the output of a
7370 // sideeffecting pattern.
7371 auto [ID, Offset, Width] =
7372 AMDGPU::Hwreg::HwregEncoding::decode(Encoded: MI.getOperand(i: 1).getImm());
7373 if (ID != AMDGPU::Hwreg::ID_MODE)
7374 return BB;
7375
7376 const unsigned WidthMask = maskTrailingOnes<unsigned>(N: Width);
7377 const unsigned SetMask = WidthMask << Offset;
7378
7379 if (getSubtarget()->hasDenormModeInst()) {
7380 unsigned SetDenormOp = 0;
7381 unsigned SetRoundOp = 0;
7382
7383 // The dedicated instructions can only set the whole denorm or round mode
7384 // at once, not a subset of bits in either.
7385 if (SetMask ==
7386 (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) {
7387 // If this fully sets both the round and denorm mode, emit the two
7388 // dedicated instructions for these.
7389 SetRoundOp = AMDGPU::S_ROUND_MODE;
7390 SetDenormOp = AMDGPU::S_DENORM_MODE;
7391 } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
7392 SetRoundOp = AMDGPU::S_ROUND_MODE;
7393 } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
7394 SetDenormOp = AMDGPU::S_DENORM_MODE;
7395 }
7396
7397 if (SetRoundOp || SetDenormOp) {
7398 MachineInstr *Def = MRI.getVRegDef(Reg: MI.getOperand(i: 0).getReg());
7399 if (Def && Def->isMoveImmediate() && Def->getOperand(i: 1).isImm()) {
7400 unsigned ImmVal = Def->getOperand(i: 1).getImm();
7401 if (SetRoundOp) {
7402 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetRoundOp))
7403 .addImm(Val: ImmVal & 0xf);
7404
7405 // If we also have the denorm mode, get just the denorm mode bits.
7406 ImmVal >>= 4;
7407 }
7408
7409 if (SetDenormOp) {
7410 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetDenormOp))
7411 .addImm(Val: ImmVal & 0xf);
7412 }
7413
7414 MI.eraseFromParent();
7415 return BB;
7416 }
7417 }
7418 }
7419
7420 // If only FP bits are touched, used the no side effects pseudo.
7421 if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
7422 AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
7423 MI.setDesc(TII->get(Opcode: AMDGPU::S_SETREG_B32_mode));
7424
7425 return BB;
7426 }
7427 case AMDGPU::S_INVERSE_BALLOT_U32:
7428 case AMDGPU::S_INVERSE_BALLOT_U64:
7429 // These opcodes only exist to let SIFixSGPRCopies insert a readfirstlane if
7430 // necessary. After that they are equivalent to a COPY.
7431 MI.setDesc(TII->get(Opcode: AMDGPU::COPY));
7432 return BB;
7433 case AMDGPU::ENDPGM_TRAP: {
7434 if (BB->succ_empty() && std::next(x: MI.getIterator()) == BB->end()) {
7435 MI.setDesc(TII->get(Opcode: AMDGPU::S_ENDPGM));
7436 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
7437 return BB;
7438 }
7439
7440 // We need a block split to make the real endpgm a terminator. We also don't
7441 // want to break phis in successor blocks, so we can't just delete to the
7442 // end of the block.
7443
7444 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false /*UpdateLiveIns*/);
7445 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
7446 MF->push_back(MBB: TrapBB);
7447 // clang-format off
7448 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ENDPGM))
7449 .addImm(Val: 0);
7450 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
7451 .addMBB(MBB: TrapBB);
7452 // clang-format on
7453
7454 BB->addSuccessor(Succ: TrapBB);
7455 MI.eraseFromParent();
7456 return SplitBB;
7457 }
7458 case AMDGPU::SIMULATED_TRAP: {
7459 assert(Subtarget->hasPrivEnabledTrap2NopBug());
7460 MachineBasicBlock *SplitBB =
7461 TII->insertSimulatedTrap(MRI, MBB&: *BB, MI, DL: MI.getDebugLoc());
7462 MI.eraseFromParent();
7463 return SplitBB;
7464 }
7465 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
7466 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
7467 assert(MFI->isWholeWaveFunction());
7468
7469 // During ISel, it's difficult to propagate the original EXEC mask to use as
7470 // an input to SI_WHOLE_WAVE_FUNC_RETURN. Set it up here instead.
7471 MachineInstr *Setup = TII->getWholeWaveFunctionSetup(MF&: *BB->getParent());
7472 assert(Setup && "Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
7473 Register OriginalExec = Setup->getOperand(i: 0).getReg();
7474 MF->getRegInfo().clearKillFlags(Reg: OriginalExec);
7475 MI.getOperand(i: 0).setReg(OriginalExec);
7476 return BB;
7477 }
7478 case AMDGPU::V_DOT2_F32_F16:
7479 case AMDGPU::V_DOT2_F32_BF16: {
7480 // Hint RA to assign dst and src2 the same physical register.
7481 // For targets without VOP2, but with VOPD, variant of the instruction this
7482 // is one of the conditions to attempt converting VOP3P to VOPD.
7483 MRI.setSimpleHint(VReg: MI.getOperand(i: 0).getReg(), PrefReg: MI.getOperand(i: 6).getReg());
7484 return BB;
7485 }
7486 case AMDGPU::SCHED_BARRIER:
7487 case AMDGPU::SCHED_GROUP_BARRIER:
7488 MI.getOperand(i: 0).setImm(MI.getOperand(i: 0).getImm() &
7489 static_cast<unsigned>(AMDGPU::SchedGroupMask::ALL));
7490 return BB;
7491 default:
7492 if (TII->isImage(MI) || TII->isMUBUF(MI)) {
7493 if (!MI.mayStore())
7494 AddMemOpInit(MI);
7495 return BB;
7496 }
7497 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, MBB: BB);
7498 }
7499}
7500
7501bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
7502 // This currently forces unfolding various combinations of fsub into fma with
7503 // free fneg'd operands. As long as we have fast FMA (controlled by
7504 // isFMAFasterThanFMulAndFAdd), we should perform these.
7505
7506 // When fma is quarter rate, for f64 where add / sub are at best half rate,
7507 // most of these combines appear to be cycle neutral but save on instruction
7508 // count / code size.
7509 return true;
7510}
7511
7512bool SITargetLowering::enableAggressiveFMAFusion(LLT Ty) const { return true; }
7513
7514EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
7515 EVT VT) const {
7516 if (!VT.isVector()) {
7517 return MVT::i1;
7518 }
7519 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i1, NumElements: VT.getVectorNumElements());
7520}
7521
7522MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
7523 // TODO: Should i16 be used always if legal? For now it would force VALU
7524 // shifts.
7525 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
7526}
7527
7528LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
7529 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
7530 ? Ty.changeElementSize(NewEltSize: 16)
7531 : Ty.changeElementSize(NewEltSize: 32);
7532}
7533
7534// Answering this is somewhat tricky and depends on the specific device which
7535// have different rates for fma or all f64 operations.
7536//
7537// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
7538// regardless of which device (although the number of cycles differs between
7539// devices), so it is always profitable for f64.
7540//
7541// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
7542// only on full rate devices. Normally, we should prefer selecting v_mad_f32
7543// which we can always do even without fused FP ops since it returns the same
7544// result as the separate operations and since it is always full
7545// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
7546// however does not support denormals, so we do report fma as faster if we have
7547// a fast fma device and require denormals.
7548//
7549bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT,
7550 DenormalFPEnv FPEnv) const {
7551 VT = VT.getScalarType();
7552 if (!VT.isSimple())
7553 return false;
7554
7555 switch (VT.getSimpleVT().SimpleTy) {
7556 case MVT::f32: {
7557 // If mad is not available this depends only on if f32 fma is full rate.
7558 if (!Subtarget->hasMadMacF32Insts())
7559 return Subtarget->hasFastFMAF32();
7560
7561 // Otherwise f32 mad is always full rate and returns the same result as
7562 // the separate operations so should be preferred over fma.
7563 // However does not support denormals.
7564 if (FPEnv.F32Mode != DenormalMode::getPreserveSign())
7565 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
7566
7567 // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
7568 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
7569 }
7570 case MVT::f64:
7571 return true;
7572 case MVT::f16:
7573 case MVT::bf16:
7574 return Subtarget->has16BitInsts() &&
7575 FPEnv.DefaultMode != DenormalMode::getPreserveSign();
7576 default:
7577 break;
7578 }
7579
7580 return false;
7581}
7582
7583bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7584 EVT VT) const {
7585 return isFMAFasterThanFMulAndFAdd(VT, FPEnv: getDenormalFPEnv(MF));
7586}
7587
7588bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
7589 Type *Ty) const {
7590 return isFMAFasterThanFMulAndFAdd(
7591 VT: getValueType(DL: F.getDataLayout(), Ty, /*AllowUnknown=*/true),
7592 FPEnv: F.getDenormalFPEnv());
7593}
7594
7595bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7596 LLT Ty) const {
7597 switch (Ty.getScalarSizeInBits()) {
7598 case 16:
7599 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f16);
7600 case 32:
7601 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f32);
7602 case 64:
7603 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f64);
7604 default:
7605 break;
7606 }
7607
7608 return false;
7609}
7610
7611bool SITargetLowering::isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const {
7612 // TODO: Check future ftz flag
7613 // v_mad_f32/v_mac_f32 do not support denormals.
7614 if (VT == MVT::f32)
7615 return Subtarget->hasMadMacF32Insts() &&
7616 FPEnv.F32Mode == DenormalMode::getPreserveSign();
7617 if (VT == MVT::f16)
7618 return Subtarget->hasMadF16() &&
7619 FPEnv.DefaultMode == DenormalMode::getPreserveSign();
7620
7621 return false;
7622}
7623
7624bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
7625 if (!Ty.isScalar())
7626 return false;
7627
7628 DenormalFPEnv FPEnv = getDenormalFPEnv(MF: *MI.getMF());
7629 if (Ty.getScalarSizeInBits() == 16)
7630 return isFMADLegal(VT: MVT::f16, FPEnv);
7631 if (Ty.getScalarSizeInBits() == 32)
7632 return isFMADLegal(VT: MVT::f32, FPEnv);
7633
7634 return false;
7635}
7636
7637bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
7638 const SDNode *N) const {
7639 return isFMADLegal(VT: N->getValueType(ResNo: 0),
7640 FPEnv: getDenormalFPEnv(MF: DAG.getMachineFunction()));
7641}
7642
7643bool SITargetLowering::isFMADLegal(const Function &F, Type *Ty) const {
7644 return isFMADLegal(VT: getValueType(DL: F.getDataLayout(), Ty: Ty->getScalarType(),
7645 /*AllowUnknown=*/true),
7646 FPEnv: F.getDenormalFPEnv());
7647}
7648
7649//===----------------------------------------------------------------------===//
7650// Custom DAG Lowering Operations
7651//===----------------------------------------------------------------------===//
7652
7653// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7654// wider vector type is legal.
7655SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
7656 SelectionDAG &DAG) const {
7657 unsigned Opc = Op.getOpcode();
7658 EVT VT = Op.getValueType();
7659 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7660
7661 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7662 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
7663
7664 SDLoc SL(Op);
7665
7666 // Forward any trailing scalar operands unchanged to both halves.
7667 SmallVector<SDValue, 2> LoOps = {Lo};
7668 SmallVector<SDValue, 2> HiOps = {Hi};
7669 auto TrailingOps = drop_begin(RangeOrContainer: Op->ops());
7670 LoOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7671 HiOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7672
7673 SDValue OpLo = DAG.getNode(Opcode: Opc, DL: SL, VT: LoVT, Ops: LoOps, Flags: Op->getFlags());
7674 SDValue OpHi = DAG.getNode(Opcode: Opc, DL: SL, VT: HiVT, Ops: HiOps, Flags: Op->getFlags());
7675
7676 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7677}
7678
7679// Enable lowering of ROTR for vxi32 types. This is a workaround for a
7680// regression whereby extra unnecessary instructions were added to codegen
7681// for rotr operations, casued by legalising v2i32 or. This resulted in extra
7682// instructions to extract the result from the vector.
7683SDValue SITargetLowering::lowerROTR(SDValue Op, SelectionDAG &DAG) const {
7684 [[maybe_unused]] EVT VT = Op.getValueType();
7685
7686 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
7687 VT == MVT::v16i32) &&
7688 "Unexpected ValueType.");
7689
7690 return DAG.UnrollVectorOp(N: Op.getNode());
7691}
7692
7693// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7694// wider vector type is legal.
7695SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
7696 SelectionDAG &DAG) const {
7697 unsigned Opc = Op.getOpcode();
7698 EVT VT = Op.getValueType();
7699 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7700
7701 auto [Lo0, Hi0] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7702 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7703
7704 SDLoc SL(Op);
7705
7706 SDValue OpLo =
7707 DAG.getNode(Opcode: Opc, DL: SL, VT: Lo0.getValueType(), N1: Lo0, N2: Lo1, Flags: Op->getFlags());
7708 SDValue OpHi =
7709 DAG.getNode(Opcode: Opc, DL: SL, VT: Hi0.getValueType(), N1: Hi0, N2: Hi1, Flags: Op->getFlags());
7710
7711 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7712}
7713
7714SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
7715 SelectionDAG &DAG) const {
7716 unsigned Opc = Op.getOpcode();
7717 EVT VT = Op.getValueType();
7718 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7719
7720 SDValue Op0 = Op.getOperand(i: 0);
7721 SDValue Lo0, Hi0;
7722 if (Op0.getValueType().isVector())
7723 std::tie(args&: Lo0, args&: Hi0) = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7724 else
7725 Lo0 = Hi0 = DAG.getFreeze(V: Op0);
7726
7727 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7728 auto [Lo2, Hi2] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 2);
7729
7730 SDLoc SL(Op);
7731 auto ResVT = DAG.GetSplitDestVTs(VT);
7732
7733 SDValue OpLo =
7734 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.first, N1: Lo0, N2: Lo1, N3: Lo2, Flags: Op->getFlags());
7735 SDValue OpHi =
7736 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.second, N1: Hi0, N2: Hi1, N3: Hi2, Flags: Op->getFlags());
7737
7738 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7739}
7740
7741SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
7742 switch (Op.getOpcode()) {
7743 default:
7744 return AMDGPUTargetLowering::LowerOperation(Op, DAG);
7745 case ISD::BRCOND:
7746 return LowerBRCOND(Op, DAG);
7747 case ISD::RETURNADDR:
7748 return LowerRETURNADDR(Op, DAG);
7749 case ISD::SPONENTRY:
7750 return LowerSPONENTRY(Op, DAG);
7751 case ISD::LOAD: {
7752 SDValue Result = LowerLOAD(Op, DAG);
7753 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
7754 "Load should return a value and a chain");
7755 return Result;
7756 }
7757 case ISD::FSQRT: {
7758 EVT VT = Op.getValueType();
7759 if (VT == MVT::f32)
7760 return lowerFSQRTF32(Op, DAG);
7761 if (VT == MVT::f64)
7762 return lowerFSQRTF64(Op, DAG);
7763 return SDValue();
7764 }
7765 case ISD::FSIN:
7766 case ISD::FCOS:
7767 return LowerTrig(Op, DAG);
7768 case ISD::SELECT:
7769 return LowerSELECT(Op, DAG);
7770 case ISD::FDIV:
7771 return LowerFDIV(Op, DAG);
7772 case ISD::FFREXP:
7773 return LowerFFREXP(Op, DAG);
7774 case ISD::ATOMIC_CMP_SWAP:
7775 return LowerATOMIC_CMP_SWAP(Op, DAG);
7776 case ISD::STORE:
7777 return LowerSTORE(Op, DAG);
7778 case ISD::GlobalAddress: {
7779 MachineFunction &MF = DAG.getMachineFunction();
7780 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
7781 return LowerGlobalAddress(MFI, Op, DAG);
7782 }
7783 case ISD::BlockAddress:
7784 return LowerBlockAddress(Op, DAG);
7785 case ISD::ExternalSymbol:
7786 return LowerExternalSymbol(Op, DAG);
7787 case ISD::INTRINSIC_WO_CHAIN:
7788 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7789 case ISD::CONVERT_FROM_ARBITRARY_FP:
7790 return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
7791 case ISD::CONVERT_TO_ARBITRARY_FP:
7792 return LowerCONVERT_TO_ARBITRARY_FP(Op, DAG);
7793 case ISD::INTRINSIC_W_CHAIN:
7794 return LowerINTRINSIC_W_CHAIN(Op, DAG);
7795 case ISD::INTRINSIC_VOID:
7796 return LowerINTRINSIC_VOID(Op, DAG);
7797 case ISD::ADDRSPACECAST:
7798 return lowerADDRSPACECAST(Op, DAG);
7799 case ISD::INSERT_SUBVECTOR:
7800 return lowerINSERT_SUBVECTOR(Op, DAG);
7801 case ISD::INSERT_VECTOR_ELT:
7802 return lowerINSERT_VECTOR_ELT(Op, DAG);
7803 case ISD::EXTRACT_VECTOR_ELT:
7804 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7805 case ISD::VECTOR_SHUFFLE:
7806 return lowerVECTOR_SHUFFLE(Op, DAG);
7807 case ISD::SCALAR_TO_VECTOR:
7808 return lowerSCALAR_TO_VECTOR(Op, DAG);
7809 case ISD::BUILD_VECTOR:
7810 return lowerBUILD_VECTOR(Op, DAG);
7811 case ISD::FP_ROUND:
7812 case ISD::STRICT_FP_ROUND:
7813 return lowerFP_ROUND(Op, DAG);
7814 case ISD::TRAP:
7815 return lowerTRAP(Op, DAG);
7816 case ISD::DEBUGTRAP:
7817 return lowerDEBUGTRAP(Op, DAG);
7818 case ISD::ABS:
7819 case ISD::FABS:
7820 case ISD::FNEG:
7821 case ISD::FCANONICALIZE:
7822 case ISD::BSWAP:
7823 return splitUnaryVectorOp(Op, DAG);
7824 case ISD::FP_TO_SINT_SAT:
7825 case ISD::FP_TO_UINT_SAT:
7826 if (Op.getValueType().isVector() && Op.getValueType() != MVT::v2i16 &&
7827 Op.getOperand(i: 0).getValueType().getScalarType() == MVT::f32)
7828 return splitUnaryVectorOp(Op, DAG);
7829 return LowerFP_TO_INT_SAT(Op, DAG);
7830 case ISD::FSUB:
7831 if (Op.getValueType() == MVT::bf16) {
7832 // Custom expansion:
7833 // fsub bf16 %a, %b -> fadd v2bf16(widen %a), fneg v2bf16(widen %b)
7834 // Then extract back to bf16.
7835 //
7836 // We create fneg on v2bf16 (not bf16) so the instruction selector can
7837 // fold the negation into the packed add's neg_lo/neg_hi modifiers,
7838 // generating a single v_pk_add_bf16 instruction. If we negate bf16 first,
7839 // it becomes a separate v_xor instruction before widening.
7840 SDLoc DL(Op);
7841 SDValue Op0 = Op.getOperand(i: 0);
7842 SDValue Op1 = Op.getOperand(i: 1);
7843
7844 // Widen both operands to v2bf16
7845 SDValue Vec0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Op0);
7846 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Op1);
7847
7848 // Create FNEG v2bf16 for the second operand
7849 SDValue NegVec1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::v2bf16, Operand: Vec1);
7850
7851 // Perform FADD v2bf16
7852 SDValue Result = DAG.getNode(Opcode: ISD::FADD, DL, VT: MVT::v2bf16, N1: Vec0, N2: NegVec1);
7853
7854 // Extract element 0 back to bf16
7855 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::bf16, N1: Result,
7856 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7857 }
7858 return SDValue();
7859 case ISD::FMINNUM:
7860 case ISD::FMAXNUM:
7861 return lowerFMINNUM_FMAXNUM(Op, DAG);
7862 case ISD::FMINIMUMNUM:
7863 case ISD::FMAXIMUMNUM:
7864 return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
7865 case ISD::FLDEXP:
7866 case ISD::STRICT_FLDEXP:
7867 return lowerFLDEXP(Op, DAG);
7868 case ISD::FMA:
7869 return splitTernaryVectorOp(Op, DAG);
7870 case ISD::FP_TO_SINT:
7871 case ISD::FP_TO_UINT:
7872 if (Subtarget->hasVCvtPkIU16F32() && Op.getValueType() == MVT::i16 &&
7873 Op.getOperand(i: 0).getValueType() == MVT::f32) {
7874 // Make f32->i16 legal so we can select V_CVT_PK_[IU]16_F32.
7875 return Op;
7876 }
7877 return LowerFP_TO_INT(Op, DAG);
7878 case ISD::SHL:
7879 case ISD::SRA:
7880 case ISD::SRL:
7881 case ISD::ADD:
7882 case ISD::SUB:
7883 case ISD::SMIN:
7884 case ISD::SMAX:
7885 case ISD::UMIN:
7886 case ISD::UMAX:
7887 case ISD::FMINNUM_IEEE:
7888 case ISD::FMAXNUM_IEEE:
7889 case ISD::FMINIMUM:
7890 case ISD::FMAXIMUM:
7891 case ISD::UADDSAT:
7892 case ISD::USUBSAT:
7893 case ISD::SADDSAT:
7894 case ISD::SSUBSAT:
7895 case ISD::FADD:
7896 case ISD::FMUL:
7897 return splitBinaryVectorOp(Op, DAG);
7898 case ISD::FCOPYSIGN:
7899 return lowerFCOPYSIGN(Op, DAG);
7900 case ISD::MUL:
7901 return lowerMUL(Op, DAG);
7902 case ISD::SMULO:
7903 case ISD::UMULO:
7904 return lowerXMULO(Op, DAG);
7905 case ISD::SMUL_LOHI:
7906 case ISD::UMUL_LOHI:
7907 return lowerXMUL_LOHI(Op, DAG);
7908 case ISD::DYNAMIC_STACKALLOC:
7909 return LowerDYNAMIC_STACKALLOC(Op, DAG);
7910 case ISD::STACKSAVE:
7911 return LowerSTACKSAVE(Op, DAG);
7912 case ISD::GET_ROUNDING:
7913 return lowerGET_ROUNDING(Op, DAG);
7914 case ISD::SET_ROUNDING:
7915 return lowerSET_ROUNDING(Op, DAG);
7916 case ISD::PREFETCH:
7917 return lowerPREFETCH(Op, DAG);
7918 case ISD::FP_EXTEND:
7919 case ISD::STRICT_FP_EXTEND:
7920 return lowerFP_EXTEND(Op, DAG);
7921 case ISD::GET_FPENV:
7922 return lowerGET_FPENV(Op, DAG);
7923 case ISD::SET_FPENV:
7924 return lowerSET_FPENV(Op, DAG);
7925 case ISD::ROTR:
7926 return lowerROTR(Op, DAG);
7927 case ISD::INLINEASM:
7928 return LowerINLINEASM(Op, DAG);
7929 }
7930 return SDValue();
7931}
7932
7933// TFE results are dword granular: value dwords followed by one status dword.
7934static std::pair<SDValue, SDValue>
7935splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL, SelectionDAG &DAG) {
7936 LLVMContext &C = *DAG.getContext();
7937 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
7938 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
7939 N2: DAG.getVectorIdxConstant(Val: NumValueDWords, DL));
7940 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
7941 SDValue ValueDWords =
7942 NumValueDWords == 1
7943 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op, N2: ZeroIdx)
7944 : DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL,
7945 VT: EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumValueDWords), N1: Op,
7946 N2: ZeroIdx);
7947 if (!VT.isVector() && VT.getSizeInBits() < 32)
7948 ValueDWords =
7949 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VT.changeTypeToInteger(), Operand: ValueDWords);
7950 return {DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ValueDWords), Status};
7951}
7952
7953// Used for D16: Casts the result of an instruction into the right vector,
7954// packs values if loads return unpacked values.
7955static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
7956 const SDLoc &DL, SelectionDAG &DAG,
7957 bool Unpacked) {
7958 if (!LoadVT.isVector())
7959 return Result;
7960
7961 // Cast back to the original packed type or to a larger type that is a
7962 // multiple of 32 bit for D16. Widening the return type is a required for
7963 // legalization.
7964 EVT FittingLoadVT = LoadVT;
7965 if ((LoadVT.getVectorNumElements() % 2) == 1) {
7966 FittingLoadVT =
7967 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7968 NumElements: LoadVT.getVectorNumElements() + 1);
7969 }
7970
7971 if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
7972 // Truncate to v2i16/v4i16.
7973 EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
7974
7975 // Workaround legalizer not scalarizing truncate after vector op
7976 // legalization but not creating intermediate vector trunc.
7977 SmallVector<SDValue, 4> Elts;
7978 DAG.ExtractVectorElements(Op: Result, Args&: Elts);
7979 for (SDValue &Elt : Elts)
7980 Elt = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Elt);
7981
7982 // Pad illegal v1i16/v3fi6 to v4i16
7983 if ((LoadVT.getVectorNumElements() % 2) == 1)
7984 Elts.push_back(Elt: DAG.getPOISON(VT: MVT::i16));
7985
7986 Result = DAG.getBuildVector(VT: IntLoadVT, DL, Ops: Elts);
7987
7988 // Bitcast to original type (v2f16/v4f16).
7989 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7990 }
7991
7992 // Cast back to the original packed type.
7993 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7994}
7995
7996SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
7997 SelectionDAG &DAG,
7998 ArrayRef<SDValue> Ops,
7999 bool IsIntrinsic) const {
8000 SDLoc DL(M);
8001
8002 bool IsTFE = M->getNumValues() == 3;
8003 bool Unpacked = Subtarget->hasUnpackedD16VMem();
8004 EVT LoadVT = M->getValueType(ResNo: 0);
8005
8006 EVT EquivLoadVT = LoadVT;
8007 if (LoadVT.isVector()) {
8008 if (Unpacked) {
8009 EquivLoadVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
8010 NumElements: LoadVT.getVectorNumElements());
8011 } else if ((LoadVT.getVectorNumElements() % 2) == 1) {
8012 // Widen v3f16 to legal type
8013 EquivLoadVT =
8014 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
8015 NumElements: LoadVT.getVectorNumElements() + 1);
8016 }
8017 }
8018
8019 if (IsTFE) {
8020 unsigned NumValueDWords = divideCeil(Numerator: EquivLoadVT.getSizeInBits(), Denominator: 32);
8021 EVT LoadDWordsVT =
8022 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumValueDWords + 1);
8023 SDVTList VTList = DAG.getVTList(VT1: LoadDWordsVT, VT2: MVT::Other);
8024 SDValue Load = DAG.getMemIntrinsicNode(
8025 Opcode, dl: DL, VTList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
8026 auto [Value, Status] = splitTFEValueAndStatus(Op: Load, VT: EquivLoadVT, DL, DAG);
8027 SDValue Adjusted =
8028 adjustLoadValueTypeImpl(Result: Value, LoadVT, DL, DAG, Unpacked);
8029 return DAG.getMergeValues(Ops: {Adjusted, Status, Load.getValue(R: 1)}, dl: DL);
8030 }
8031
8032 // Change from v4f16/v2f16 to EquivLoadVT.
8033 SDVTList VTList = DAG.getVTList(VT1: EquivLoadVT, VT2: MVT::Other);
8034
8035 SDValue Load = DAG.getMemIntrinsicNode(
8036 Opcode: IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, dl: DL, VTList, Ops,
8037 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
8038
8039 SDValue Adjusted = adjustLoadValueTypeImpl(Result: Load, LoadVT, DL, DAG, Unpacked);
8040
8041 return DAG.getMergeValues(Ops: {Adjusted, Load.getValue(R: 1)}, dl: DL);
8042}
8043
8044SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
8045 SelectionDAG &DAG,
8046 ArrayRef<SDValue> Ops) const {
8047 SDLoc DL(M);
8048 EVT LoadVT = M->getValueType(ResNo: 0);
8049 EVT EltType = LoadVT.getScalarType();
8050 EVT IntVT = LoadVT.changeTypeToInteger();
8051
8052 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
8053
8054 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
8055 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8056 DAG.getMachineFunction().getFunction(),
8057 "unsupported sub-dword format buffer load", DL.getDebugLoc()));
8058 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: LoadVT), M->getOperand(Num: 0)}, dl: DL);
8059 }
8060
8061 assert(M->getNumValues() == 2 || M->getNumValues() == 3);
8062 bool IsTFE = M->getNumValues() == 3;
8063
8064 if (IsD16 && IsTFE && !Subtarget->hasBufferTFEFormatD16()) {
8065 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8066 DAG.getMachineFunction().getFunction(),
8067 "TFE D16 format buffer load is not supported on this GPU",
8068 DL.getDebugLoc()));
8069 return DAG.getErrorMergeValues(ResultTypes: {M->value_begin(), M->value_end()},
8070 Chain: M->getOperand(Num: 0), dl: DL);
8071 }
8072
8073 unsigned Opc = IsD16 ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
8074 : AMDGPUISD::BUFFER_LOAD_FORMAT_D16)
8075 : IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
8076 : AMDGPUISD::BUFFER_LOAD_FORMAT)
8077 : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
8078 : AMDGPUISD::BUFFER_LOAD;
8079
8080 if (IsD16)
8081 return adjustLoadValueType(Opcode: Opc, M, DAG, Ops);
8082
8083 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
8084 if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
8085 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, MMO: M->getMemOperand(),
8086 IsTFE);
8087
8088 if (isTypeLegal(VT: LoadVT)) {
8089 return getMemIntrinsicNode(Opcode: Opc, DL, VTList: M->getVTList(), Ops, MemVT: IntVT,
8090 MMO: M->getMemOperand(), DAG);
8091 }
8092
8093 EVT CastVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: LoadVT);
8094 SDVTList VTList = IsTFE ? DAG.getVTList(VT1: CastVT, VT2: MVT::i32, VT3: MVT::Other)
8095 : DAG.getVTList(VT1: CastVT, VT2: MVT::Other);
8096 SDValue MemNode = getMemIntrinsicNode(Opcode: Opc, DL, VTList, Ops, MemVT: CastVT,
8097 MMO: M->getMemOperand(), DAG);
8098 SDValue Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: MemNode);
8099 if (IsTFE)
8100 return DAG.getMergeValues(Ops: {Data, MemNode.getValue(R: 1), MemNode.getValue(R: 2)},
8101 dl: DL);
8102 return DAG.getMergeValues(Ops: {Data, MemNode.getValue(R: 1)}, dl: DL);
8103}
8104
8105static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
8106 SelectionDAG &DAG) {
8107 EVT VT = N->getValueType(ResNo: 0);
8108 SDValue Src = N->getOperand(Num: 1);
8109 SDLoc SL(N);
8110
8111 if (Src.getOpcode() == ISD::SETCC) {
8112 SDValue Op0 = Src.getOperand(i: 0);
8113 SDValue Op1 = Src.getOperand(i: 1);
8114 // Need to expand bfloat to float for comparison (setcc).
8115 if (Op0.getValueType() == MVT::bf16) {
8116 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op0);
8117 Op1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op1);
8118 }
8119 // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
8120 return DAG.getNode(Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: Op0, N2: Op1, N3: Src.getOperand(i: 2));
8121 }
8122 if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Val&: Src)) {
8123 // (ballot 0) -> 0
8124 if (Arg->isZero())
8125 return DAG.getConstant(Val: 0, DL: SL, VT);
8126
8127 // (ballot 1) -> EXEC/EXEC_LO
8128 if (Arg->isOne()) {
8129 Register Exec;
8130 if (VT.getScalarSizeInBits() == 32)
8131 Exec = AMDGPU::EXEC_LO;
8132 else if (VT.getScalarSizeInBits() == 64)
8133 Exec = AMDGPU::EXEC;
8134 else
8135 return SDValue();
8136
8137 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: Exec, VT);
8138 }
8139 }
8140
8141 // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
8142 // ISD::SETNE)
8143 return DAG.getNode(
8144 Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: DAG.getZExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32),
8145 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), N3: DAG.getCondCode(Cond: ISD::SETNE));
8146}
8147
8148static SDValue lowerBFEIntrinsic(SDValue Op, SelectionDAG &DAG,
8149 Intrinsic::ID IntrinsicID) {
8150 bool Signed = IntrinsicID == Intrinsic::amdgcn_sbfe;
8151 SDLoc DL(Op);
8152 EVT VT = Op.getValueType();
8153 SDValue Src = Op.getOperand(i: 1);
8154 SDValue Offset = Op.getOperand(i: 2);
8155 SDValue Width = Op.getOperand(i: 3);
8156
8157 if (VT != MVT::i32) {
8158 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8159 DAG.getMachineFunction().getFunction(),
8160 Twine(Intrinsic::getBaseName(id: IntrinsicID)) + " only supports i32",
8161 DL.getDebugLoc()));
8162 return DAG.getPOISON(VT);
8163 }
8164
8165 return DAG.getNode(Opcode: Signed ? AMDGPUISD::BFE_I32 : AMDGPUISD::BFE_U32, DL, VT,
8166 N1: Src, N2: Offset, N3: Width);
8167}
8168
8169static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
8170 EVT VT);
8171
8172static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
8173 SelectionDAG &DAG) {
8174 EVT VT = N->getValueType(ResNo: 0);
8175 unsigned ValSize = VT.getSizeInBits();
8176 unsigned IID = N->getConstantOperandVal(Num: 0);
8177 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
8178 IID == Intrinsic::amdgcn_permlanex16;
8179 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
8180 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
8181 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
8182 IID == Intrinsic::amdgcn_permlane_up ||
8183 IID == Intrinsic::amdgcn_permlane_down ||
8184 IID == Intrinsic::amdgcn_permlane_xor;
8185 SDLoc SL(N);
8186 MVT IntVT = MVT::getIntegerVT(BitWidth: ValSize);
8187 const GCNSubtarget *ST = TLI.getSubtarget();
8188
8189 unsigned SplitSize = 32;
8190 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
8191 ST->hasDPALU_DPP() &&
8192 AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: N->getConstantOperandVal(Num: 3)))
8193 SplitSize = 64;
8194
8195 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
8196 SDValue Src2, MVT ValT) -> SDValue {
8197 SmallVector<SDValue, 8> Operands;
8198 switch (IID) {
8199 case Intrinsic::amdgcn_permlane16:
8200 case Intrinsic::amdgcn_permlanex16:
8201 case Intrinsic::amdgcn_update_dpp:
8202 Operands.push_back(Elt: N->getOperand(Num: 6));
8203 Operands.push_back(Elt: N->getOperand(Num: 5));
8204 Operands.push_back(Elt: N->getOperand(Num: 4));
8205 [[fallthrough]];
8206 case Intrinsic::amdgcn_writelane:
8207 case Intrinsic::amdgcn_permlane_bcast:
8208 case Intrinsic::amdgcn_permlane_up:
8209 case Intrinsic::amdgcn_permlane_down:
8210 case Intrinsic::amdgcn_permlane_xor:
8211 Operands.push_back(Elt: Src2);
8212 [[fallthrough]];
8213 case Intrinsic::amdgcn_readlane:
8214 case Intrinsic::amdgcn_set_inactive:
8215 case Intrinsic::amdgcn_set_inactive_chain_arg:
8216 case Intrinsic::amdgcn_mov_dpp8:
8217 Operands.push_back(Elt: Src1);
8218 [[fallthrough]];
8219 case Intrinsic::amdgcn_readfirstlane:
8220 case Intrinsic::amdgcn_permlane64:
8221 Operands.push_back(Elt: Src0);
8222 break;
8223 default:
8224 llvm_unreachable("unhandled lane op");
8225 }
8226
8227 Operands.push_back(Elt: DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32));
8228 std::reverse(first: Operands.begin(), last: Operands.end());
8229
8230 if (SDNode *GL = N->getGluedNode()) {
8231 assert(GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8232 GL = GL->getOperand(Num: 0).getNode();
8233 Operands.push_back(Elt: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8234 Operand: SDValue(GL, 0)));
8235 }
8236
8237 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: ValT, Ops: Operands);
8238 };
8239
8240 SDValue Src0 = N->getOperand(Num: 1);
8241 SDValue Src1, Src2;
8242 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
8243 IID == Intrinsic::amdgcn_mov_dpp8 ||
8244 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
8245 IsPermlaneShuffle) {
8246 Src1 = N->getOperand(Num: 2);
8247 if (IID == Intrinsic::amdgcn_writelane ||
8248 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
8249 IsPermlaneShuffle)
8250 Src2 = N->getOperand(Num: 3);
8251 }
8252
8253 if (ValSize == SplitSize) {
8254 // Already legal
8255 return SDValue();
8256 }
8257
8258 if (ValSize < 32) {
8259 bool IsFloat = VT.isFloatingPoint();
8260 Src0 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src0) : Src0,
8261 DL: SL, VT: MVT::i32);
8262
8263 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
8264 Src1 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src1) : Src1,
8265 DL: SL, VT: MVT::i32);
8266 }
8267
8268 if (IID == Intrinsic::amdgcn_writelane) {
8269 Src2 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src2) : Src2,
8270 DL: SL, VT: MVT::i32);
8271 }
8272
8273 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
8274 SDValue Trunc = DAG.getAnyExtOrTrunc(Op: LaneOp, DL: SL, VT: IntVT);
8275 return IsFloat ? DAG.getBitcast(VT, V: Trunc) : Trunc;
8276 }
8277
8278 if (ValSize % SplitSize != 0)
8279 return SDValue();
8280
8281 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
8282 EVT VT = N->getValueType(ResNo: 0);
8283 unsigned NE = VT.getVectorNumElements();
8284 EVT EltVT = VT.getVectorElementType();
8285 SmallVector<SDValue, 8> Scalars;
8286 unsigned NumOperands = N->getNumOperands();
8287 SmallVector<SDValue, 4> Operands(NumOperands);
8288 SDNode *GL = N->getGluedNode();
8289
8290 // only handle convergencectrl_glue
8291 assert(!GL || GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8292
8293 for (unsigned i = 0; i != NE; ++i) {
8294 for (unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
8295 ++j) {
8296 SDValue Operand = N->getOperand(Num: j);
8297 EVT OperandVT = Operand.getValueType();
8298 if (OperandVT.isVector()) {
8299 // A vector operand; extract a single element.
8300 EVT OperandEltVT = OperandVT.getVectorElementType();
8301 Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: OperandEltVT,
8302 N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: SL));
8303 } else {
8304 // A scalar operand; just use it as is.
8305 Operands[j] = Operand;
8306 }
8307 }
8308
8309 if (GL)
8310 Operands[NumOperands - 1] =
8311 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8312 Operand: SDValue(GL->getOperand(Num: 0).getNode(), 0));
8313
8314 Scalars.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: EltVT, Ops: Operands));
8315 }
8316
8317 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NE);
8318 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops: Scalars);
8319 };
8320
8321 if (VT.isVector()) {
8322 switch (MVT::SimpleValueType EltTy =
8323 VT.getVectorElementType().getSimpleVT().SimpleTy) {
8324 case MVT::i32:
8325 case MVT::f32:
8326 if (SplitSize == 32) {
8327 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT());
8328 return unrollLaneOp(LaneOp.getNode());
8329 }
8330 [[fallthrough]];
8331 case MVT::i16:
8332 case MVT::f16:
8333 case MVT::bf16: {
8334 unsigned SubVecNumElt =
8335 SplitSize / VT.getVectorElementType().getSizeInBits();
8336 MVT SubVecVT = MVT::getVectorVT(VT: EltTy, NumElements: SubVecNumElt);
8337 SmallVector<SDValue, 4> Pieces;
8338 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
8339 for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
8340 Src0SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src0,
8341 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8342
8343 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
8344 IsPermLane16) {
8345 Src1SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src1,
8346 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8347
8348 Pieces.push_back(
8349 Elt: createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
8350 } else if (IID == Intrinsic::amdgcn_writelane) {
8351 Src2SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src2,
8352 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8353 Pieces.push_back(
8354 Elt: createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
8355 } else {
8356 Pieces.push_back(Elt: createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
8357 }
8358
8359 EltIdx += SubVecNumElt;
8360 }
8361 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, Ops: Pieces);
8362 }
8363 default:
8364 // Handle all other cases by bitcasting to i32 vectors
8365 break;
8366 }
8367 }
8368
8369 MVT VecVT =
8370 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplitSize), NumElements: ValSize / SplitSize);
8371 Src0 = DAG.getBitcast(VT: VecVT, V: Src0);
8372
8373 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
8374 Src1 = DAG.getBitcast(VT: VecVT, V: Src1);
8375
8376 if (IID == Intrinsic::amdgcn_writelane)
8377 Src2 = DAG.getBitcast(VT: VecVT, V: Src2);
8378
8379 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
8380 SDValue UnrolledLaneOp = unrollLaneOp(LaneOp.getNode());
8381 return DAG.getBitcast(VT, V: UnrolledLaneOp);
8382}
8383
8384static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N,
8385 SelectionDAG &DAG) {
8386 EVT VT = N->getValueType(ResNo: 0);
8387
8388 if (VT.getSizeInBits() != 32)
8389 return SDValue();
8390
8391 SDLoc SL(N);
8392
8393 SDValue Value = N->getOperand(Num: 1);
8394 SDValue Index = N->getOperand(Num: 2);
8395
8396 // ds_bpermute requires index to be multiplied by 4
8397 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: 2, VT: MVT::i32, DL: SL);
8398 SDValue ShiftedIndex =
8399 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: Index.getValueType(), N1: Index, N2: ShiftAmount);
8400
8401 // Intrinsics will require i32 to operate on
8402 SDValue ValueI32 = DAG.getBitcast(VT: MVT::i32, V: Value);
8403
8404 auto MakeIntrinsic = [&DAG, &SL](unsigned IID, MVT RetVT,
8405 SmallVector<SDValue> IntrinArgs) -> SDValue {
8406 SmallVector<SDValue> Operands(1);
8407 Operands[0] = DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32);
8408 Operands.append(RHS: IntrinArgs);
8409 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: RetVT, Ops: Operands);
8410 };
8411
8412 // If we can bpermute across the whole wave, then just do that
8413 if (TLI.getSubtarget()->supportsWaveWideBPermute()) {
8414 SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8415 {ShiftedIndex, ValueI32});
8416 return DAG.getBitcast(VT, V: BPermute);
8417 }
8418
8419 assert(TLI.getSubtarget()->isWave64());
8420
8421 // Otherwise, we need to make use of whole wave mode
8422 SDValue PoisonVal = DAG.getPOISON(VT: ValueI32->getValueType(ResNo: 0));
8423
8424 // Set inactive lanes to poison
8425 SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8426 {ValueI32, PoisonVal});
8427 SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8428 {ShiftedIndex, PoisonVal});
8429
8430 SDValue Swapped =
8431 MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
8432
8433 // Get permutation of each half, then we'll select which one to use
8434 SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8435 {WWMIndex, WWMValue});
8436 SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
8437 MVT::i32, {WWMIndex, Swapped});
8438 SDValue BPermOtherHalfWWM =
8439 MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
8440
8441 // Select which side to take the permute from
8442 SDValue ThreadIDMask = DAG.getAllOnesConstant(DL: SL, VT: MVT::i32);
8443 // We can get away with only using mbcnt_lo here since we're only
8444 // trying to detect which side of 32 each lane is on, and mbcnt_lo
8445 // returns 32 for lanes 32-63.
8446 SDValue ThreadID =
8447 MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
8448 {ThreadIDMask, DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32)});
8449
8450 SDValue SameOrOtherHalf =
8451 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32,
8452 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: ThreadID, N2: Index),
8453 N2: DAG.getTargetConstant(Val: 32, DL: SL, VT: MVT::i32));
8454 SDValue UseSameHalf =
8455 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SameOrOtherHalf,
8456 RHS: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond: ISD::SETEQ);
8457 SDValue Result = DAG.getSelect(DL: SL, VT: MVT::i32, Cond: UseSameHalf, LHS: BPermSameHalf,
8458 RHS: BPermOtherHalfWWM);
8459 return DAG.getBitcast(VT, V: Result);
8460}
8461
8462void SITargetLowering::ReplaceNodeResults(SDNode *N,
8463 SmallVectorImpl<SDValue> &Results,
8464 SelectionDAG &DAG) const {
8465 switch (N->getOpcode()) {
8466 case ISD::INSERT_VECTOR_ELT: {
8467 if (SDValue Res = lowerINSERT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8468 Results.push_back(Elt: Res);
8469 return;
8470 }
8471 case ISD::EXTRACT_VECTOR_ELT: {
8472 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8473 Results.push_back(Elt: Res);
8474 return;
8475 }
8476 case ISD::CONVERT_TO_ARBITRARY_FP: {
8477 if (SDValue Res = LowerCONVERT_TO_ARBITRARY_FP(Op: SDValue(N, 0), DAG))
8478 Results.push_back(Elt: Res);
8479 return;
8480 }
8481 case ISD::INTRINSIC_WO_CHAIN: {
8482 unsigned IID = N->getConstantOperandVal(Num: 0);
8483 switch (IID) {
8484 case Intrinsic::amdgcn_wave_reduce_min:
8485 case Intrinsic::amdgcn_wave_reduce_umin:
8486 case Intrinsic::amdgcn_wave_reduce_max:
8487 case Intrinsic::amdgcn_wave_reduce_umax:
8488 case Intrinsic::amdgcn_wave_reduce_add:
8489 case Intrinsic::amdgcn_wave_reduce_sub:
8490 case Intrinsic::amdgcn_wave_reduce_and:
8491 case Intrinsic::amdgcn_wave_reduce_or:
8492 case Intrinsic::amdgcn_wave_reduce_xor: {
8493 EVT VT = N->getValueType(ResNo: 0);
8494 if (isTypeLegal(VT))
8495 return;
8496 SDLoc SL(N);
8497 bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
8498 IID == Intrinsic::amdgcn_wave_reduce_max ||
8499 IID == Intrinsic::amdgcn_wave_reduce_add ||
8500 IID == Intrinsic::amdgcn_wave_reduce_sub;
8501 unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8502 SDValue ExtSrc = DAG.getNode(Opcode: ExtOpc, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 1));
8503 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
8504 N1: N->getOperand(Num: 0), N2: ExtSrc, N3: N->getOperand(Num: 2));
8505 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Result));
8506 return;
8507 }
8508 case Intrinsic::amdgcn_make_buffer_rsrc:
8509 Results.push_back(Elt: lowerPointerAsRsrcIntrin(Op: N, DAG));
8510 return;
8511 case Intrinsic::amdgcn_cvt_pkrtz: {
8512 SDValue Src0 = N->getOperand(Num: 1);
8513 SDValue Src1 = N->getOperand(Num: 2);
8514 SDLoc SL(N);
8515 SDValue Cvt =
8516 DAG.getNode(Opcode: AMDGPUISD::CVT_PKRTZ_F16_F32, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8517 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Cvt));
8518 return;
8519 }
8520 case Intrinsic::amdgcn_cvt_pknorm_i16:
8521 case Intrinsic::amdgcn_cvt_pknorm_u16:
8522 case Intrinsic::amdgcn_cvt_pk_i16:
8523 case Intrinsic::amdgcn_cvt_pk_u16: {
8524 SDValue Src0 = N->getOperand(Num: 1);
8525 SDValue Src1 = N->getOperand(Num: 2);
8526 SDLoc SL(N);
8527 unsigned Opcode;
8528
8529 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
8530 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
8531 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
8532 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
8533 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
8534 Opcode = AMDGPUISD::CVT_PK_I16_I32;
8535 else
8536 Opcode = AMDGPUISD::CVT_PK_U16_U32;
8537
8538 EVT VT = N->getValueType(ResNo: 0);
8539 if (isTypeLegal(VT))
8540 Results.push_back(Elt: DAG.getNode(Opcode, DL: SL, VT, N1: Src0, N2: Src1));
8541 else {
8542 SDValue Cvt = DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8543 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: Cvt));
8544 }
8545 return;
8546 }
8547 case Intrinsic::amdgcn_s_buffer_load: {
8548 SDValue Op = SDValue(N, 0);
8549 EVT VT = Op.getValueType();
8550 Results.push_back(Elt: lowerSBuffer(VT, MemVT: VT, DL: SDLoc(Op), Chain: DAG.getEntryNode(),
8551 Rsrc: Op.getOperand(i: 1), Offset: Op.getOperand(i: 2),
8552 CachePolicy: Op.getOperand(i: 3), DAG));
8553 return;
8554 }
8555 case Intrinsic::amdgcn_dead: {
8556 for (unsigned I = 0, E = N->getNumValues(); I < E; ++I)
8557 Results.push_back(Elt: DAG.getPOISON(VT: N->getValueType(ResNo: I)));
8558 return;
8559 }
8560 }
8561 break;
8562 }
8563 case ISD::INTRINSIC_W_CHAIN: {
8564 if (N->getConstantOperandVal(Num: 1) != Intrinsic::amdgcn_ptr_s_buffer_load &&
8565 N->getValueType(ResNo: 0).isSimple() &&
8566 SBufferLoadDiagnosticVTs[N->getSimpleValueType(ResNo: 0).SimpleTy])
8567 break;
8568 if (SDValue Res = LowerINTRINSIC_W_CHAIN(Op: SDValue(N, 0), DAG)) {
8569 if (Res.getOpcode() == ISD::MERGE_VALUES) {
8570 // FIXME: Hacky
8571 for (unsigned I = 0; I < Res.getNumOperands(); I++) {
8572 Results.push_back(Elt: Res.getOperand(i: I));
8573 }
8574 } else {
8575 for (unsigned I = 0; I < N->getNumValues(); ++I)
8576 Results.push_back(Elt: Res.getValue(R: I));
8577 }
8578 return;
8579 }
8580
8581 break;
8582 }
8583 case ISD::SELECT: {
8584 SDLoc SL(N);
8585 EVT VT = N->getValueType(ResNo: 0);
8586 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT);
8587 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 1));
8588 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 2));
8589
8590 EVT SelectVT = NewVT;
8591 if (NewVT.bitsLT(VT: MVT::i32)) {
8592 LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: LHS);
8593 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: RHS);
8594 SelectVT = MVT::i32;
8595 }
8596
8597 SDValue NewSelect =
8598 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: SelectVT, N1: N->getOperand(Num: 0), N2: LHS, N3: RHS);
8599
8600 if (NewVT != SelectVT)
8601 NewSelect = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: NewVT, Operand: NewSelect);
8602 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewSelect));
8603 return;
8604 }
8605 case ISD::FNEG: {
8606 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8607 break;
8608
8609 SDLoc SL(N);
8610 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8611
8612 SDValue Op = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: BC,
8613 N2: DAG.getConstant(Val: 0x80008000, DL: SL, VT: MVT::i32));
8614 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8615 return;
8616 }
8617 case ISD::FABS: {
8618 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8619 break;
8620
8621 SDLoc SL(N);
8622 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8623
8624 SDValue Op = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: BC,
8625 N2: DAG.getConstant(Val: 0x7fff7fff, DL: SL, VT: MVT::i32));
8626 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8627 return;
8628 }
8629 case ISD::FSQRT: {
8630 if (N->getValueType(ResNo: 0) != MVT::f16)
8631 break;
8632 Results.push_back(Elt: lowerFSQRTF16(Op: SDValue(N, 0), DAG));
8633 break;
8634 }
8635 default:
8636 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
8637 break;
8638 }
8639}
8640
8641/// Helper function for LowerBRCOND
8642static SDNode *findUser(SDValue Value, unsigned Opcode) {
8643
8644 for (SDUse &U : Value->uses()) {
8645 if (U.get() != Value)
8646 continue;
8647
8648 if (U.getUser()->getOpcode() == Opcode)
8649 return U.getUser();
8650 }
8651 return nullptr;
8652}
8653
8654unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
8655 if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
8656 switch (Intr->getConstantOperandVal(Num: 1)) {
8657 case Intrinsic::amdgcn_if:
8658 return AMDGPUISD::IF;
8659 case Intrinsic::amdgcn_else:
8660 return AMDGPUISD::ELSE;
8661 case Intrinsic::amdgcn_loop:
8662 return AMDGPUISD::LOOP;
8663 case Intrinsic::amdgcn_end_cf:
8664 llvm_unreachable("should not occur");
8665 default:
8666 return 0;
8667 }
8668 }
8669
8670 // break, if_break, else_break are all only used as inputs to loop, not
8671 // directly as branch conditions.
8672 return 0;
8673}
8674
8675bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
8676 const Triple &TT = GV->getParent()->getTargetTriple();
8677 return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
8678 GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
8679 AMDGPU::shouldEmitConstantsToTextSection(TT);
8680}
8681
8682bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
8683 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
8684 return false;
8685
8686 // FIXME: Either avoid relying on address space here or change the default
8687 // address space for functions to avoid the explicit check.
8688 return (GV->getValueType()->isFunctionTy() ||
8689 !isNonGlobalAddrSpace(AS: GV->getAddressSpace())) &&
8690 !shouldEmitFixup(GV) && !getTargetMachine().shouldAssumeDSOLocal(GV);
8691}
8692
8693bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
8694 return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
8695}
8696
8697bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const {
8698 if (!GV->hasExternalLinkage())
8699 return true;
8700
8701 // With object linking, external LDS declarations need relocations so the
8702 // linker can assign their offsets.
8703 if (AMDGPUTargetMachine::EnableObjectLinking) {
8704 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: GV)) {
8705 if (GVar->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
8706 GVar->getAddressSpace() == AMDGPUAS::BARRIER) {
8707 assert(GVar->isDeclaration() &&
8708 "AS 3 & 13 GVs should be declaration here "
8709 "when object linking is enabled");
8710 return false;
8711 }
8712 }
8713 }
8714
8715 const auto OS = getTargetMachine().getTargetTriple().getOS();
8716 return OS == Triple::AMDHSA || OS == Triple::AMDPAL;
8717}
8718
8719/// This transforms the control flow intrinsics to get the branch destination as
8720/// last parameter, also switches branch target with BR if the need arise
8721SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, SelectionDAG &DAG) const {
8722 SDLoc DL(BRCOND);
8723
8724 SDNode *Intr = BRCOND.getOperand(i: 1).getNode();
8725 SDValue Target = BRCOND.getOperand(i: 2);
8726 SDNode *BR = nullptr;
8727 SDNode *SetCC = nullptr;
8728
8729 switch (Intr->getOpcode()) {
8730 case ISD::SETCC: {
8731 // As long as we negate the condition everything is fine
8732 SetCC = Intr;
8733 Intr = SetCC->getOperand(Num: 0).getNode();
8734 break;
8735 }
8736 case ISD::XOR: {
8737 // Similar to SETCC, if we have (xor c, -1), we will be fine.
8738 SDValue LHS = Intr->getOperand(Num: 0);
8739 SDValue RHS = Intr->getOperand(Num: 1);
8740 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS); C && C->getZExtValue()) {
8741 Intr = LHS.getNode();
8742 break;
8743 }
8744 [[fallthrough]];
8745 }
8746 default: {
8747 // Get the target from BR if we don't negate the condition
8748 BR = findUser(Value: BRCOND, Opcode: ISD::BR);
8749 assert(BR && "brcond missing unconditional branch user");
8750 Target = BR->getOperand(Num: 1);
8751 }
8752 }
8753
8754 unsigned CFNode = isCFIntrinsic(Intr);
8755 if (CFNode == 0) {
8756 // This is a uniform branch so we don't need to legalize.
8757 return BRCOND;
8758 }
8759
8760 bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
8761 Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
8762
8763 assert(!SetCC ||
8764 (SetCC->getConstantOperandVal(1) == 1 &&
8765 cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
8766 ISD::SETNE));
8767
8768 // operands of the new intrinsic call
8769 SmallVector<SDValue, 4> Ops;
8770 if (HaveChain)
8771 Ops.push_back(Elt: BRCOND.getOperand(i: 0));
8772
8773 Ops.append(in_start: Intr->op_begin() + (HaveChain ? 2 : 1), in_end: Intr->op_end());
8774 Ops.push_back(Elt: Target);
8775
8776 ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
8777
8778 // build the new intrinsic call
8779 SDNode *Result = DAG.getNode(Opcode: CFNode, DL, VTList: DAG.getVTList(VTs: Res), Ops).getNode();
8780
8781 if (!HaveChain) {
8782 SDValue Ops[] = {SDValue(Result, 0), BRCOND.getOperand(i: 0)};
8783
8784 Result = DAG.getMergeValues(Ops, dl: DL).getNode();
8785 }
8786
8787 if (BR) {
8788 // Give the branch instruction our target
8789 SDValue Ops[] = {BR->getOperand(Num: 0), BRCOND.getOperand(i: 2)};
8790 SDValue NewBR = DAG.getNode(Opcode: ISD::BR, DL, VTList: BR->getVTList(), Ops);
8791 DAG.ReplaceAllUsesWith(From: BR, To: NewBR.getNode());
8792 }
8793
8794 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
8795
8796 // Copy the intrinsic results to registers
8797 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
8798 SDNode *CopyToReg = findUser(Value: SDValue(Intr, i), Opcode: ISD::CopyToReg);
8799 if (!CopyToReg)
8800 continue;
8801
8802 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CopyToReg->getOperand(Num: 1),
8803 N: SDValue(Result, i - 1), Glue: SDValue());
8804
8805 DAG.ReplaceAllUsesWith(From: SDValue(CopyToReg, 0), To: CopyToReg->getOperand(Num: 0));
8806 }
8807
8808 // Remove the old intrinsic from the chain
8809 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Intr, Intr->getNumValues() - 1),
8810 To: Intr->getOperand(Num: 0));
8811
8812 return Chain;
8813}
8814
8815SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
8816 MVT VT = Op.getSimpleValueType();
8817 SDLoc DL(Op);
8818 // Checking the depth
8819 if (Op.getConstantOperandVal(i: 0) != 0)
8820 return DAG.getConstant(Val: 0, DL, VT);
8821
8822 MachineFunction &MF = DAG.getMachineFunction();
8823 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8824 // Check for kernel and shader functions
8825 if (Info->isEntryFunction())
8826 return DAG.getConstant(Val: 0, DL, VT);
8827
8828 MachineFrameInfo &MFI = MF.getFrameInfo();
8829 // There is a call to @llvm.returnaddress in this function
8830 MFI.setReturnAddressIsTaken(true);
8831
8832 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
8833 // Get the return address reg and mark it as an implicit live-in
8834 Register Reg = MF.addLiveIn(PReg: TRI->getReturnAddressReg(MF),
8835 RC: getRegClassFor(VT, isDivergent: Op.getNode()->isDivergent()));
8836
8837 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT);
8838}
8839
8840SDValue SITargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
8841 MachineFunction &MF = DAG.getMachineFunction();
8842 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8843
8844 // For functions that set up their own stack, select the GET_STACK_BASE
8845 // pseudo.
8846 if (MFI->isBottomOfStack())
8847 return Op;
8848
8849 // For everything else, create a dummy stack object.
8850 int FI = MF.getFrameInfo().CreateFixedObject(Size: 1, SPOffset: 0, /*IsImmutable=*/false);
8851 return DAG.getFrameIndex(FI, VT: Op.getValueType());
8852}
8853
8854SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op,
8855 const SDLoc &DL, EVT VT) const {
8856 return Op.getValueType().bitsLE(VT)
8857 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
8858 : DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
8859 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8860}
8861
8862SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
8863 SelectionDAG &DAG) const {
8864 EVT DstVT = Op.getValueType();
8865 unsigned NumElts = DstVT.getVectorNumElements();
8866 assert(NumElts > 2 && isPowerOf2_32(NumElts));
8867
8868 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
8869
8870 SDLoc DL(Op);
8871 unsigned Opc = Op.getOpcode();
8872 SDValue Flags = Op.getOperand(i: 1);
8873 EVT HalfDstVT =
8874 EVT::getVectorVT(Context&: *DAG.getContext(), VT: DstVT.getScalarType(), NumElements: NumElts / 2);
8875 SDValue OpLo = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Lo, N2: Flags);
8876 SDValue OpHi = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Hi, N2: Flags);
8877
8878 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, N1: OpLo, N2: OpHi);
8879}
8880
8881SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
8882 bool IsStrict = Op->isStrictFPOpcode();
8883 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8884 EVT SrcVT = Src.getValueType();
8885 EVT DstVT = Op.getValueType();
8886
8887 if (DstVT.isVectorOf(EltVT: MVT::f16)) {
8888 assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32");
8889 if (SrcVT.getScalarType() != MVT::f32)
8890 return SDValue();
8891 return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG);
8892 }
8893
8894 if (SrcVT.getScalarType() != MVT::f64)
8895 return Op;
8896
8897 SDLoc DL(Op);
8898 if (DstVT == MVT::f16) {
8899 // TODO: Handle strictfp
8900 if (Op.getOpcode() != ISD::FP_ROUND)
8901 return Op;
8902
8903 if (!Subtarget->has16BitInsts()) {
8904 SDValue FpToFp16 = DAG.getNode(Opcode: ISD::FP_TO_FP16, DL, VT: MVT::i32, Operand: Src);
8905 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8906 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8907 }
8908 if (Op->getFlags().hasApproximateFuncs()) {
8909 SDValue Flags = Op.getOperand(i: 1);
8910 SDValue Src32 = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f32, N1: Src, N2: Flags);
8911 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Src32, N2: Flags);
8912 }
8913 SDValue FpToFp16 = LowerF64ToF16Safe(Src, DL, DAG);
8914 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8915 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8916 }
8917
8918 assert(DstVT.getScalarType() == MVT::bf16 &&
8919 "custom lower FP_ROUND for f16 or bf16");
8920 assert(Subtarget->hasBF16ConversionInsts() && "f32 -> bf16 is legal");
8921
8922 // Round-inexact-to-odd f64 to f32, then do the final rounding using the
8923 // hardware f32 -> bf16 instruction.
8924 EVT F32VT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
8925 SDValue Rod = expandRoundInexactToOdd(ResultVT: F32VT, Op: Src, DL, DAG);
8926 if (IsStrict) {
8927 return DAG.getNode(
8928 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
8929 Ops: {Op.getOperand(i: 0), Rod, DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)});
8930 }
8931 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: Rod,
8932 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8933}
8934
8935SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
8936 SelectionDAG &DAG) const {
8937 EVT VT = Op.getValueType();
8938 const MachineFunction &MF = DAG.getMachineFunction();
8939 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8940 bool IsIEEEMode = Info->getMode().IEEE;
8941
8942 // FIXME: Assert during selection that this is only selected for
8943 // ieee_mode. Currently a combine can produce the ieee version for non-ieee
8944 // mode functions, but this happens to be OK since it's only done in cases
8945 // where there is known no sNaN.
8946 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8947 return expandFMINNUM_FMAXNUM(N: Op.getNode(), DAG);
8948
8949 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8950 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8951 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8952 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8953 return splitBinaryVectorOp(Op, DAG);
8954 return Op;
8955}
8956
8957SDValue
8958SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
8959 SelectionDAG &DAG) const {
8960 EVT VT = Op.getValueType();
8961 const MachineFunction &MF = DAG.getMachineFunction();
8962 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8963 bool IsIEEEMode = Info->getMode().IEEE;
8964
8965 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8966 return expandFMINIMUMNUM_FMAXIMUMNUM(N: Op.getNode(), DAG);
8967
8968 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8969 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8970 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8971 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8972 return splitBinaryVectorOp(Op, DAG);
8973 return Op;
8974}
8975
8976SDValue SITargetLowering::lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const {
8977 bool IsStrict = Op.getOpcode() == ISD::STRICT_FLDEXP;
8978 EVT VT = Op.getValueType();
8979 assert(VT == MVT::f16);
8980
8981 SDValue Exp = Op.getOperand(i: IsStrict ? 2 : 1);
8982 EVT ExpVT = Exp.getValueType();
8983 if (ExpVT == MVT::i16)
8984 return Op;
8985
8986 SDLoc DL(Op);
8987
8988 // Correct the exponent type for f16 to i16.
8989 // Clamp the range of the exponent to the instruction's range.
8990
8991 // TODO: This should be a generic narrowing legalization, and can easily be
8992 // for GlobalISel.
8993
8994 SDValue MinExp = DAG.getSignedConstant(Val: minIntN(N: 16), DL, VT: ExpVT);
8995 SDValue ClampMin = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ExpVT, N1: Exp, N2: MinExp);
8996
8997 SDValue MaxExp = DAG.getSignedConstant(Val: maxIntN(N: 16), DL, VT: ExpVT);
8998 SDValue Clamp = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ExpVT, N1: ClampMin, N2: MaxExp);
8999
9000 SDValue TruncExp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Clamp);
9001
9002 if (IsStrict) {
9003 return DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL, ResultTys: {VT, MVT::Other},
9004 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), TruncExp});
9005 }
9006
9007 return DAG.getNode(Opcode: ISD::FLDEXP, DL, VT, N1: Op.getOperand(i: 0), N2: TruncExp);
9008}
9009
9010static unsigned getExtOpcodeForPromotedOp(SDValue Op) {
9011 switch (Op->getOpcode()) {
9012 case ISD::ABS:
9013 case ISD::SRA:
9014 case ISD::SMIN:
9015 case ISD::SMAX:
9016 return ISD::SIGN_EXTEND;
9017 case ISD::SRL:
9018 case ISD::UMIN:
9019 case ISD::UMAX:
9020 case ISD::USUBSAT:
9021 case ISD::UADDSAT:
9022 return ISD::ZERO_EXTEND;
9023 case ISD::ADD:
9024 case ISD::SUB:
9025 case ISD::AND:
9026 case ISD::OR:
9027 case ISD::XOR:
9028 case ISD::SHL:
9029 case ISD::SELECT:
9030 case ISD::MUL:
9031 // operation result won't be influenced by garbage high bits.
9032 // TODO: are all of those cases correct, and are there more?
9033 return ISD::ANY_EXTEND;
9034 case ISD::SETCC: {
9035 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
9036 return ISD::isSignedIntSetCC(Code: CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
9037 }
9038 default:
9039 llvm_unreachable("unexpected opcode!");
9040 }
9041}
9042
9043SDValue
9044SITargetLowering::promoteUniformUnaryOpToI32(SDValue Op,
9045 DAGCombinerInfo &DCI) const {
9046 EVT OpTy = Op.getValueType();
9047 SelectionDAG &DAG = DCI.DAG;
9048 EVT ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
9049
9050 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
9051 return SDValue();
9052
9053 SDLoc DL(Op);
9054 SDValue Input = Op.getOperand(i: 0);
9055 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
9056 Input = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: Input);
9057
9058 SDValue NewVal = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: ExtTy, Operand: Input);
9059
9060 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OpTy, Operand: NewVal);
9061}
9062
9063SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
9064 DAGCombinerInfo &DCI) const {
9065 const unsigned Opc = Op.getOpcode();
9066 assert(Opc == ISD::ADD || Opc == ISD::SUB || Opc == ISD::SHL ||
9067 Opc == ISD::SRL || Opc == ISD::SRA || Opc == ISD::AND ||
9068 Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL ||
9069 Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN ||
9070 Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX ||
9071 Opc == ISD::USUBSAT || Opc == ISD::UADDSAT);
9072
9073 EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType()
9074 : Op->getOperand(Num: 0).getValueType();
9075 auto &DAG = DCI.DAG;
9076 auto ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
9077
9078 if (DCI.isBeforeLegalizeOps() ||
9079 isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
9080 return SDValue();
9081
9082 SDLoc DL(Op);
9083 SDValue LHS;
9084 SDValue RHS;
9085 if (Opc == ISD::SELECT) {
9086 LHS = Op->getOperand(Num: 1);
9087 RHS = Op->getOperand(Num: 2);
9088 } else {
9089 LHS = Op->getOperand(Num: 0);
9090 RHS = Op->getOperand(Num: 1);
9091 }
9092
9093 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
9094 LHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {LHS});
9095
9096 // Special case: for shifts, the RHS always needs a zext.
9097 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
9098 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtTy, Operand: {RHS});
9099 else
9100 RHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {RHS});
9101
9102 // setcc always return i1/i1 vec so no need to truncate after.
9103 if (Opc == ISD::SETCC) {
9104 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
9105 return DAG.getSetCC(DL, VT: Op.getValueType(), LHS, RHS, Cond: CC);
9106 }
9107
9108 // For other ops, we extend the operation's return type as well so we need to
9109 // truncate back to the original type.
9110 SDValue NewVal;
9111 if (Opc == ISD::SELECT)
9112 NewVal = DAG.getNode(Opcode: ISD::SELECT, DL, VT: ExtTy, Ops: {Op->getOperand(Num: 0), LHS, RHS});
9113 else if (Opc == ISD::UADDSAT) {
9114 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ExtTy, N1: LHS, N2: RHS);
9115 SDValue MaxVal = DAG.getConstant(
9116 Val: APInt::getMaxValue(numBits: OpTy.getScalarSizeInBits()).zext(width: 32), DL, VT: ExtTy);
9117 NewVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ExtTy, N1: Sum, N2: MaxVal);
9118 } else
9119 NewVal = DAG.getNode(Opcode: Opc, DL, VT: ExtTy, Ops: {LHS, RHS});
9120
9121 return DAG.getZExtOrTrunc(Op: NewVal, DL, VT: OpTy);
9122}
9123
9124SDValue SITargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
9125 SDValue Mag = Op.getOperand(i: 0);
9126 EVT MagVT = Mag.getValueType();
9127
9128 if (MagVT.getVectorNumElements() > 2)
9129 return splitBinaryVectorOp(Op, DAG);
9130
9131 SDValue Sign = Op.getOperand(i: 1);
9132 EVT SignVT = Sign.getValueType();
9133
9134 if (MagVT == SignVT)
9135 return Op;
9136
9137 // fcopysign v2f16:mag, v2f32:sign ->
9138 // fcopysign v2f16:mag,
9139 // bitcast (trunc (srl (bitcast sign to v2i32), 16) to v2i16)
9140
9141 SDLoc SL(Op);
9142 SDValue SignAsInt32 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Sign);
9143 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: 16, VT: MVT::v2i32, DL: SL);
9144 SDValue SignShifted =
9145 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::v2i32, N1: SignAsInt32, N2: ShiftAmt);
9146 SDValue SignAsInt16 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::v2i16, Operand: SignShifted);
9147
9148 SDValue SignAsHalf16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MagVT, Operand: SignAsInt16);
9149
9150 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MagVT, N1: Mag, N2: SignAsHalf16);
9151}
9152
9153// Custom lowering for vector multiplications and s_mul_u64.
9154SDValue SITargetLowering::lowerMUL(SDValue Op, SelectionDAG &DAG) const {
9155 EVT VT = Op.getValueType();
9156
9157 // Split vector operands.
9158 if (VT.isVector())
9159 return splitBinaryVectorOp(Op, DAG);
9160
9161 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64");
9162
9163 // There are four ways to lower s_mul_u64:
9164 //
9165 // 1. If all the operands are uniform, then we lower it as it is.
9166 //
9167 // 2. If the operands are divergent, then we have to split s_mul_u64 in 32-bit
9168 // multiplications because there is not a vector equivalent of s_mul_u64.
9169 //
9170 // 3. If the cost model decides that it is more efficient to use vector
9171 // registers, then we have to split s_mul_u64 in 32-bit multiplications.
9172 // This happens in splitScalarSMULU64() in SIInstrInfo.cpp .
9173 //
9174 // 4. If the cost model decides to use vector registers and both of the
9175 // operands are zero-extended/sign-extended from 32-bits, then we split the
9176 // s_mul_u64 in two 32-bit multiplications. The problem is that it is not
9177 // possible to check if the operands are zero-extended or sign-extended in
9178 // SIInstrInfo.cpp. For this reason, here, we replace s_mul_u64 with
9179 // s_mul_u64_u32_pseudo if both operands are zero-extended and we replace
9180 // s_mul_u64 with s_mul_i64_i32_pseudo if both operands are sign-extended.
9181 // If the cost model decides that we have to use vector registers, then
9182 // splitScalarSMulPseudo() (in SIInstrInfo.cpp) split s_mul_u64_u32/
9183 // s_mul_i64_i32_pseudo in two vector multiplications. If the cost model
9184 // decides that we should use scalar registers, then s_mul_u64_u32_pseudo/
9185 // s_mul_i64_i32_pseudo is lowered as s_mul_u64 in expandPostRAPseudo() in
9186 // SIInstrInfo.cpp .
9187
9188 if (Op->isDivergent())
9189 return SDValue();
9190
9191 SDValue Op0 = Op.getOperand(i: 0);
9192 SDValue Op1 = Op.getOperand(i: 1);
9193 // If all the operands are zero-enteted to 32-bits, then we replace s_mul_u64
9194 // with s_mul_u64_u32_pseudo. If all the operands are sign-extended to
9195 // 32-bits, then we replace s_mul_u64 with s_mul_i64_i32_pseudo.
9196 KnownBits Op0KnownBits = DAG.computeKnownBits(Op: Op0);
9197 unsigned Op0LeadingZeros = Op0KnownBits.countMinLeadingZeros();
9198 KnownBits Op1KnownBits = DAG.computeKnownBits(Op: Op1);
9199 unsigned Op1LeadingZeros = Op1KnownBits.countMinLeadingZeros();
9200 SDLoc SL(Op);
9201 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
9202 return SDValue(
9203 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_U64_U32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9204 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op0);
9205 unsigned Op1SignBits = DAG.ComputeNumSignBits(Op: Op1);
9206 if (Op0SignBits >= 33 && Op1SignBits >= 33)
9207 return SDValue(
9208 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_I64_I32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9209 // If all the operands are uniform, then we lower s_mul_u64 as it is.
9210 return Op;
9211}
9212
9213SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
9214 EVT VT = Op.getValueType();
9215 SDLoc SL(Op);
9216 SDValue LHS = Op.getOperand(i: 0);
9217 SDValue RHS = Op.getOperand(i: 1);
9218 bool isSigned = Op.getOpcode() == ISD::SMULO;
9219
9220 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
9221 const APInt &C = RHSC->getAPIntValue();
9222 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
9223 if (C.isPowerOf2()) {
9224 // smulo(x, signed_min) is same as umulo(x, signed_min).
9225 bool UseArithShift = isSigned && !C.isMinSignedValue();
9226 SDValue ShiftAmt = DAG.getConstant(Val: C.logBase2(), DL: SL, VT: MVT::i32);
9227 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: LHS, N2: ShiftAmt);
9228 SDValue Overflow =
9229 DAG.getSetCC(DL: SL, VT: MVT::i1,
9230 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL, DL: SL, VT,
9231 N1: Result, N2: ShiftAmt),
9232 RHS: LHS, Cond: ISD::SETNE);
9233 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9234 }
9235 }
9236
9237 SDValue Result = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: LHS, N2: RHS);
9238 SDValue Top =
9239 DAG.getNode(Opcode: isSigned ? ISD::MULHS : ISD::MULHU, DL: SL, VT, N1: LHS, N2: RHS);
9240
9241 SDValue Sign = isSigned
9242 ? DAG.getNode(Opcode: ISD::SRA, DL: SL, VT, N1: Result,
9243 N2: DAG.getConstant(Val: VT.getScalarSizeInBits() - 1,
9244 DL: SL, VT: MVT::i32))
9245 : DAG.getConstant(Val: 0, DL: SL, VT);
9246 SDValue Overflow = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Top, RHS: Sign, Cond: ISD::SETNE);
9247
9248 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9249}
9250
9251SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
9252 if (Op->isDivergent()) {
9253 // Select to V_MAD_[IU]64_[IU]32.
9254 return Op;
9255 }
9256 if (Subtarget->hasSMulHi()) {
9257 // Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
9258 return SDValue();
9259 }
9260 // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
9261 // calculate the high part, so we might as well do the whole thing with
9262 // V_MAD_[IU]64_[IU]32.
9263 return Op;
9264}
9265
9266SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
9267 if (!Subtarget->hasTrapHandler() ||
9268 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
9269 return lowerTrapEndpgm(Op, DAG);
9270
9271 return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(Op, DAG)
9272 : lowerTrapHsaQueuePtr(Op, DAG);
9273}
9274
9275SDValue SITargetLowering::lowerTrapEndpgm(SDValue Op, SelectionDAG &DAG) const {
9276 SDLoc SL(Op);
9277 SDValue Chain = Op.getOperand(i: 0);
9278 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9279}
9280
9281SDValue
9282SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT,
9283 const SDLoc &DL, Align Alignment,
9284 ImplicitParameter Param) const {
9285 MachineFunction &MF = DAG.getMachineFunction();
9286 uint64_t Offset = getImplicitParameterOffset(MF, Param);
9287 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain: DAG.getEntryNode(), Offset);
9288 MachinePointerInfo PtrInfo =
9289 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
9290 return DAG.getLoad(
9291 VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
9292 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
9293}
9294
9295SDValue SITargetLowering::lowerTrapHsaQueuePtr(SDValue Op,
9296 SelectionDAG &DAG) const {
9297 SDLoc SL(Op);
9298 SDValue Chain = Op.getOperand(i: 0);
9299
9300 SDValue QueuePtr;
9301 // For code object version 5, QueuePtr is passed through implicit kernarg.
9302 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9303 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9304 QueuePtr =
9305 loadImplicitKernelArgument(DAG, VT: MVT::i64, DL: SL, Alignment: Align(8), Param: QUEUE_PTR);
9306 } else {
9307 MachineFunction &MF = DAG.getMachineFunction();
9308 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9309 Register UserSGPR = Info->getQueuePtrUserSGPR();
9310
9311 if (UserSGPR == AMDGPU::NoRegister) {
9312 // We probably are in a function incorrectly marked with
9313 // amdgpu-no-queue-ptr. This is undefined. We don't want to delete the
9314 // trap, so just use a null pointer.
9315 QueuePtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9316 } else {
9317 QueuePtr = CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR,
9318 VT: MVT::i64);
9319 }
9320 }
9321
9322 SDValue SGPR01 = DAG.getRegister(Reg: AMDGPU::SGPR0_SGPR1, VT: MVT::i64);
9323 SDValue ToReg = DAG.getCopyToReg(Chain, dl: SL, Reg: SGPR01, N: QueuePtr, Glue: SDValue());
9324
9325 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9326 SDValue Ops[] = {ToReg, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16), SGPR01,
9327 ToReg.getValue(R: 1)};
9328 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9329}
9330
9331SDValue SITargetLowering::lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const {
9332 SDLoc SL(Op);
9333 SDValue Chain = Op.getOperand(i: 0);
9334
9335 // We need to simulate the 's_trap 2' instruction on targets that run in
9336 // PRIV=1 (where it is treated as a nop).
9337 if (Subtarget->hasPrivEnabledTrap2NopBug())
9338 return DAG.getNode(Opcode: AMDGPUISD::SIMULATED_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9339
9340 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9341 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9342 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9343}
9344
9345SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
9346 SDLoc SL(Op);
9347 SDValue Chain = Op.getOperand(i: 0);
9348 MachineFunction &MF = DAG.getMachineFunction();
9349
9350 if (!Subtarget->hasTrapHandler() ||
9351 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
9352 LLVMContext &Ctx = MF.getFunction().getContext();
9353 Ctx.diagnose(DI: DiagnosticInfoUnsupported(MF.getFunction(),
9354 "debugtrap handler not supported",
9355 Op.getDebugLoc(), DS_Warning));
9356 return Chain;
9357 }
9358
9359 uint64_t TrapID =
9360 static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap);
9361 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9362 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9363}
9364
9365/// When a divergent value (in VGPR) is passed to an inline asm with an SGPR
9366/// constraint ('s'), we need to insert v_readfirstlane to move the value from
9367/// VGPR to SGPR. This is done by modifying the CopyToReg nodes in the glue
9368/// chain that feed into the INLINEASM node.
9369SDValue SITargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
9370 unsigned NumOps = Op.getNumOperands();
9371
9372 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
9373 SmallSet<Register, 8> SGPRInputRegs;
9374
9375 unsigned NumVals = 0;
9376 for (unsigned I = InlineAsm::Op_FirstOperand; I < NumOps - 1;
9377 I += 1 + NumVals) {
9378 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i: I));
9379 NumVals = Flags.getNumOperandRegisters();
9380
9381 unsigned RCID;
9382 bool IsSGPRInput = Flags.getKind() == InlineAsm::Kind::RegUse &&
9383 NumVals > 0 && Flags.hasRegClassConstraint(RC&: RCID) &&
9384 TRI->isSGPRClass(RC: TRI->getRegClass(i: RCID));
9385
9386 for (unsigned J = 0; J < NumVals; ++J) {
9387 SDValue Val = Op.getOperand(i: I + 1 + J);
9388 if (const RegisterSDNode *RegNode =
9389 dyn_cast<RegisterSDNode>(Val: Val.getNode())) {
9390 Register Reg = RegNode->getReg();
9391 if (IsSGPRInput || (Reg.isPhysical() && TRI->isSGPRPhysReg(Reg)))
9392 SGPRInputRegs.insert(V: Reg);
9393 }
9394 }
9395 }
9396
9397 if (SGPRInputRegs.empty())
9398 return Op;
9399
9400 // Walk the glue chain and insert readfirstlane for divergent SGPR inputs.
9401 SDLoc DL(Op);
9402 SDNode *N = Op.getOperand(i: NumOps - 1).getNode();
9403
9404 while (N && N->getOpcode() == ISD::CopyToReg) {
9405 Register Reg = cast<RegisterSDNode>(Val: N->getOperand(Num: 1))->getReg();
9406 SDValue SrcVal = N->getOperand(Num: 2);
9407
9408 // Insert readfirstlane if copying a divergent value to an SGPR input.
9409 if (SrcVal->isDivergent() && SGPRInputRegs.count(V: Reg)) {
9410 SDValue ReadFirstLaneID =
9411 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
9412 SDValue ReadFirstLane =
9413 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: SrcVal.getValueType(),
9414 N1: ReadFirstLaneID, N2: SrcVal);
9415
9416 SmallVector<SDValue, 4> Ops = {N->getOperand(Num: 0), N->getOperand(Num: 1),
9417 ReadFirstLane};
9418 if (N->getNumOperands() > 3)
9419 Ops.push_back(Elt: N->getOperand(Num: 3)); // Glue input
9420
9421 DAG.UpdateNodeOperands(N, Ops);
9422 }
9423
9424 // Follow glue chain to next CopyToReg.
9425 SDNode *Next = nullptr;
9426 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I) {
9427 if (N->getOperand(Num: I).getValueType() == MVT::Glue) {
9428 Next = N->getOperand(Num: I).getNode();
9429 break;
9430 }
9431 }
9432 N = Next;
9433 }
9434
9435 return Op;
9436}
9437
9438SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
9439 SelectionDAG &DAG) const {
9440 unsigned BaseAS = AS;
9441 unsigned SANum = AMDGPU::getSyntheticApertureNumber(AS);
9442 if (SANum != AMDGPU::SyntheticAperture::None)
9443 BaseAS = AMDGPUAS::LOCAL_ADDRESS;
9444
9445 SDValue Aperture = getBaseSegmentAperture(AS: BaseAS, DL, DAG);
9446
9447 if (SANum != AMDGPU::SyntheticAperture::None) {
9448 SDValue Tag = DAG.getConstant(Val: SANum, DL, VT: MVT::i32);
9449 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Aperture, N2: Tag);
9450 }
9451
9452 return Aperture;
9453}
9454
9455SDValue SITargetLowering::getBaseSegmentAperture(unsigned AS, const SDLoc &DL,
9456 SelectionDAG &DAG) const {
9457 const bool IsLDS = (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER);
9458
9459 if (Subtarget->hasApertureRegs()) {
9460 const unsigned ApertureRegNo =
9461 IsLDS ? AMDGPU::SRC_SHARED_BASE : AMDGPU::SRC_PRIVATE_BASE;
9462 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
9463 !Subtarget->hasGloballyAddressableScratch()) &&
9464 "Cannot use src_private_base with globally addressable scratch!");
9465 // Note: this feature (register) is broken. When used as a 32-bit operand,
9466 // it returns a wrong value (all zeroes?). The real value is in the upper 32
9467 // bits.
9468 //
9469 // To work around the issue, emit a 64 bit copy from this register
9470 // then extract the high bits. Note that this shouldn't even result in a
9471 // shift being emitted and simply become a pair of registers (e.g.):
9472 // s_mov_b64 s[6:7], src_shared_base
9473 // v_mov_b32_e32 v1, s7
9474 SDValue Copy =
9475 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: ApertureRegNo, VT: MVT::v2i32);
9476 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Copy, Idx: 1);
9477 }
9478
9479 // For code object version 5, private_base and shared_base are passed through
9480 // implicit kernargs.
9481 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9482 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9483 ImplicitParameter Param = IsLDS ? SHARED_BASE : PRIVATE_BASE;
9484 return loadImplicitKernelArgument(DAG, VT: MVT::i32, DL, Alignment: Align(4), Param);
9485 }
9486
9487 MachineFunction &MF = DAG.getMachineFunction();
9488 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9489 Register UserSGPR = Info->getQueuePtrUserSGPR();
9490 if (UserSGPR == AMDGPU::NoRegister) {
9491 // We probably are in a function incorrectly marked with
9492 // amdgpu-no-queue-ptr. This is undefined.
9493 return DAG.getPOISON(VT: MVT::i32);
9494 }
9495
9496 SDValue QueuePtr =
9497 CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR, VT: MVT::i64);
9498
9499 // Offset into amd_queue_t for group_segment_aperture_base_hi /
9500 // private_segment_aperture_base_hi.
9501 uint32_t StructOffset = IsLDS ? 0x40 : 0x44;
9502
9503 SDValue Ptr =
9504 DAG.getObjectPtrOffset(SL: DL, Ptr: QueuePtr, Offset: TypeSize::getFixed(ExactSize: StructOffset));
9505
9506 // TODO: Use custom target PseudoSourceValue.
9507 // TODO: We should use the value from the IR intrinsic call, but it might not
9508 // be available and how do we get it?
9509 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
9510 return DAG.getLoad(VT: MVT::i32, dl: DL, Chain: QueuePtr.getValue(R: 1), Ptr, PtrInfo,
9511 Alignment: commonAlignment(A: Align(64), Offset: StructOffset),
9512 MMOFlags: MachineMemOperand::MODereferenceable |
9513 MachineMemOperand::MOInvariant);
9514}
9515
9516/// Return true if the value is a known valid address, such that a null check is
9517/// not necessary.
9518static bool isKnownNonNull(SDValue Val, SelectionDAG &DAG,
9519 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
9520 if (isa<FrameIndexSDNode, GlobalAddressSDNode, BasicBlockSDNode>(Val))
9521 return true;
9522
9523 if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
9524 return ConstVal->getSExtValue() != AMDGPU::getNullPointerValue(AS: AddrSpace);
9525
9526 // TODO: Search through arithmetic, handle arguments and loads
9527 // marked nonnull.
9528 return false;
9529}
9530
9531SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
9532 SelectionDAG &DAG) const {
9533 SDLoc SL(Op);
9534
9535 const AMDGPUTargetMachine &TM =
9536 static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
9537
9538 const auto *ASC = cast<AddrSpaceCastSDNode>(Val&: Op);
9539 unsigned SrcAS = ASC->getSrcAddressSpace();
9540 SDValue Src = ASC->getOperand(Num: 0);
9541 unsigned DestAS = ASC->getDestAddressSpace();
9542 bool IsNonNull = ASC->getFlags().hasNonNull();
9543
9544 SDValue FlatNullPtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9545
9546 // flat -> local/private/barrier
9547 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
9548 if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
9549 DestAS == AMDGPUAS::PRIVATE_ADDRESS || DestAS == AMDGPUAS::BARRIER) {
9550 SDValue Ptr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9551
9552 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
9553 Subtarget->hasGloballyAddressableScratch()) {
9554 // flat -> private with globally addressable scratch: subtract
9555 // src_flat_scratch_base_lo.
9556 SDValue FlatScratchBaseLo(
9557 DAG.getMachineNode(
9558 Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: MVT::i32,
9559 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, VT: MVT::i32)),
9560 0);
9561 Ptr = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: Ptr, N2: FlatScratchBaseLo);
9562 }
9563
9564 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9565 return Ptr;
9566
9567 unsigned NullVal = AMDGPU::getNullPointerValue(AS: DestAS);
9568 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9569 SDValue NonNull = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: FlatNullPtr, Cond: ISD::SETNE);
9570
9571 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: NonNull, N2: Ptr,
9572 N3: SegmentNullPtr);
9573 }
9574 }
9575
9576 // local/private/barrier -> flat
9577 if (DestAS == AMDGPUAS::FLAT_ADDRESS) {
9578 if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
9579 SrcAS == AMDGPUAS::PRIVATE_ADDRESS || SrcAS == AMDGPUAS::BARRIER) {
9580 SDValue CvtPtr;
9581 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
9582 Subtarget->hasGloballyAddressableScratch()) {
9583 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
9584 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
9585 SDValue AllOnes = DAG.getSignedTargetConstant(Val: -1, DL: SL, VT: MVT::i32);
9586 SDValue ThreadID = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
9587 ThreadID = DAG.getNode(
9588 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9589 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_lo, DL: SL, VT: MVT::i32),
9590 N2: AllOnes, N3: ThreadID);
9591 if (Subtarget->isWave64())
9592 ThreadID = DAG.getNode(
9593 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9594 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_hi, DL: SL, VT: MVT::i32),
9595 N2: AllOnes, N3: ThreadID);
9596 SDValue ShAmt = DAG.getShiftAmountConstant(
9597 Val: 57 - 32 - Subtarget->getWavefrontSizeLog2(), VT: MVT::i32, DL: SL);
9598 SDValue SrcHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ThreadID, N2: ShAmt);
9599 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: SrcHi);
9600 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9601 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
9602 // 64-bit hi:lo value.
9603 SDValue FlatScratchBase = {
9604 DAG.getMachineNode(
9605 Opcode: AMDGPU::S_MOV_B64, dl: SL, VT: MVT::i64,
9606 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE, VT: MVT::i64)),
9607 0};
9608 CvtPtr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: CvtPtr, N2: FlatScratchBase);
9609 } else {
9610 SDValue Aperture = getSegmentAperture(AS: SrcAS, DL: SL, DAG);
9611
9612 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Aperture);
9613 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9614 }
9615
9616 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9617 return CvtPtr;
9618
9619 unsigned NullVal = AMDGPU::getNullPointerValue(AS: SrcAS);
9620 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9621
9622 SDValue NonNull =
9623 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: SegmentNullPtr, Cond: ISD::SETNE);
9624
9625 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: NonNull, N2: CvtPtr,
9626 N3: FlatNullPtr);
9627 }
9628 }
9629
9630 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9631 Op.getValueType() == MVT::i64) {
9632 const SIMachineFunctionInfo *Info =
9633 DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
9634 if (Info->get32BitAddressHighBits() == 0)
9635 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: Src);
9636
9637 SDValue Hi = DAG.getConstant(Val: Info->get32BitAddressHighBits(), DL: SL, VT: MVT::i32);
9638 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Hi);
9639 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
9640 }
9641
9642 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9643 Src.getValueType() == MVT::i64)
9644 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9645
9646 // global <-> flat are no-ops and never emitted.
9647
9648 // Invalid casts are poison.
9649 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
9650}
9651
9652// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
9653// the small vector and inserting them into the big vector. That is better than
9654// the default expansion of doing it via a stack slot. Even though the use of
9655// the stack slot would be optimized away afterwards, the stack slot itself
9656// remains.
9657SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
9658 SelectionDAG &DAG) const {
9659 SDValue Vec = Op.getOperand(i: 0);
9660 SDValue Ins = Op.getOperand(i: 1);
9661 SDValue Idx = Op.getOperand(i: 2);
9662 EVT VecVT = Vec.getValueType();
9663 EVT InsVT = Ins.getValueType();
9664 EVT EltVT = VecVT.getVectorElementType();
9665 unsigned InsNumElts = InsVT.getVectorNumElements();
9666 unsigned IdxVal = Idx->getAsZExtVal();
9667 SDLoc SL(Op);
9668
9669 if (EltVT.getScalarSizeInBits() == 16 && IdxVal % 2 == 0) {
9670 // Insert 32-bit registers at a time.
9671 assert(InsNumElts % 2 == 0 && "expect legal vector types");
9672
9673 unsigned VecNumElts = VecVT.getVectorNumElements();
9674 EVT NewVecVT =
9675 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: VecNumElts / 2);
9676 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
9677 : EVT::getVectorVT(Context&: *DAG.getContext(),
9678 VT: MVT::i32, NumElements: InsNumElts / 2);
9679
9680 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVecVT, Operand: Vec);
9681 Ins = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewInsVT, Operand: Ins);
9682
9683 for (unsigned I = 0; I != InsNumElts / 2; ++I) {
9684 SDValue Elt;
9685 if (InsNumElts == 2) {
9686 Elt = Ins;
9687 } else {
9688 Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Ins,
9689 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9690 }
9691 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NewVecVT, N1: Vec, N2: Elt,
9692 N3: DAG.getConstant(Val: IdxVal / 2 + I, DL: SL, VT: MVT::i32));
9693 }
9694
9695 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Vec);
9696 }
9697
9698 for (unsigned I = 0; I != InsNumElts; ++I) {
9699 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Ins,
9700 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9701 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: VecVT, N1: Vec, N2: Elt,
9702 N3: DAG.getConstant(Val: IdxVal + I, DL: SL, VT: MVT::i32));
9703 }
9704 return Vec;
9705}
9706
9707SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
9708 SelectionDAG &DAG) const {
9709 SDValue Vec = Op.getOperand(i: 0);
9710 SDValue InsVal = Op.getOperand(i: 1);
9711 SDValue Idx = Op.getOperand(i: 2);
9712 EVT VecVT = Vec.getValueType();
9713 EVT EltVT = VecVT.getVectorElementType();
9714 unsigned VecSize = VecVT.getSizeInBits();
9715 unsigned EltSize = EltVT.getSizeInBits();
9716 SDLoc SL(Op);
9717
9718 // Specially handle the case of v4i16 with static indexing.
9719 unsigned NumElts = VecVT.getVectorNumElements();
9720 auto *KIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
9721 if (NumElts == 4 && EltSize == 16 && KIdx) {
9722 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Vec);
9723
9724 SDValue LoHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9725 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
9726 SDValue HiHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9727 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
9728
9729 SDValue LoVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: LoHalf);
9730 SDValue HiVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: HiHalf);
9731
9732 unsigned Idx = KIdx->getZExtValue();
9733 bool InsertLo = Idx < 2;
9734 SDValue InsHalf = DAG.getNode(
9735 Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: MVT::v2i16, N1: InsertLo ? LoVec : HiVec,
9736 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: InsVal),
9737 N3: DAG.getConstant(Val: InsertLo ? Idx : (Idx - 2), DL: SL, VT: MVT::i32));
9738
9739 InsHalf = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: InsHalf);
9740
9741 SDValue Concat =
9742 InsertLo ? DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {InsHalf, HiHalf})
9743 : DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoHalf, InsHalf});
9744
9745 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Concat);
9746 }
9747
9748 // Static indexing does not lower to stack access, and hence there is no need
9749 // for special custom lowering to avoid stack access.
9750 if (isa<ConstantSDNode>(Val: Idx))
9751 return SDValue();
9752
9753 // Avoid stack access for dynamic indexing by custom lowering to
9754 // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
9755
9756 assert(VecSize <= 64 && "Expected target vector size to be <= 64 bits");
9757
9758 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9759
9760 // Convert vector index to bit-index and get the required bit mask.
9761 assert(isPowerOf2_32(EltSize));
9762 const auto EltMask = maskTrailingOnes<uint64_t>(N: EltSize);
9763 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9764 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9765 SDValue BFM = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT,
9766 N1: DAG.getConstant(Val: EltMask, DL: SL, VT: IntVT), N2: ScaledIdx);
9767
9768 // 1. Create a congruent vector with the target value in each element.
9769 SDValue ExtVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT,
9770 Operand: DAG.getSplatBuildVector(VT: VecVT, DL: SL, Op: InsVal));
9771
9772 // 2. Mask off all other indices except the required index within (1).
9773 SDValue LHS = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: BFM, N2: ExtVal);
9774
9775 // 3. Mask off the required index within the target vector.
9776 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9777 SDValue RHS =
9778 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: DAG.getNOT(DL: SL, Val: BFM, VT: IntVT), N2: BCVec);
9779
9780 // 4. Get (2) and (3) ORed into the target vector.
9781 SDValue BFI =
9782 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: LHS, N2: RHS, Flags: SDNodeFlags::Disjoint);
9783
9784 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: BFI);
9785}
9786
9787SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
9788 SelectionDAG &DAG) const {
9789 SDLoc SL(Op);
9790
9791 EVT ResultVT = Op.getValueType();
9792 SDValue Vec = Op.getOperand(i: 0);
9793 SDValue Idx = Op.getOperand(i: 1);
9794 EVT VecVT = Vec.getValueType();
9795 unsigned VecSize = VecVT.getSizeInBits();
9796 EVT EltVT = VecVT.getVectorElementType();
9797
9798 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
9799
9800 // Make sure we do any optimizations that will make it easier to fold
9801 // source modifiers before obscuring it with bit operations.
9802
9803 // XXX - Why doesn't this get called when vector_shuffle is expanded?
9804 if (SDValue Combined = performExtractVectorEltCombine(N: Op.getNode(), DCI))
9805 return Combined;
9806
9807 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
9808 SDValue Lo, Hi;
9809 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
9810
9811 if (VecSize == 128) {
9812 SDValue V2 = DAG.getBitcast(VT: MVT::v2i64, V: Vec);
9813 Lo = DAG.getBitcast(VT: LoVT,
9814 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9815 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32)));
9816 Hi = DAG.getBitcast(VT: HiVT,
9817 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9818 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32)));
9819 } else if (VecSize == 256) {
9820 SDValue V2 = DAG.getBitcast(VT: MVT::v4i64, V: Vec);
9821 SDValue Parts[4];
9822 for (unsigned P = 0; P < 4; ++P) {
9823 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9824 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9825 }
9826
9827 Lo = DAG.getBitcast(VT: LoVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9828 N1: Parts[0], N2: Parts[1]));
9829 Hi = DAG.getBitcast(VT: HiVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9830 N1: Parts[2], N2: Parts[3]));
9831 } else {
9832 assert(VecSize == 512);
9833
9834 SDValue V2 = DAG.getBitcast(VT: MVT::v8i64, V: Vec);
9835 SDValue Parts[8];
9836 for (unsigned P = 0; P < 8; ++P) {
9837 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9838 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9839 }
9840
9841 Lo = DAG.getBitcast(VT: LoVT,
9842 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9843 N1: Parts[0], N2: Parts[1], N3: Parts[2], N4: Parts[3]));
9844 Hi = DAG.getBitcast(VT: HiVT,
9845 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9846 N1: Parts[4], N2: Parts[5], N3: Parts[6], N4: Parts[7]));
9847 }
9848
9849 EVT IdxVT = Idx.getValueType();
9850 unsigned NElem = VecVT.getVectorNumElements();
9851 assert(isPowerOf2_32(NElem));
9852 SDValue IdxMask = DAG.getConstant(Val: NElem / 2 - 1, DL: SL, VT: IdxVT);
9853 SDValue NewIdx = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IdxVT, N1: Idx, N2: IdxMask);
9854 SDValue Half = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IdxMask, True: Hi, False: Lo, Cond: ISD::SETUGT);
9855 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Half, N2: NewIdx);
9856 }
9857
9858 assert(VecSize <= 64);
9859
9860 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9861
9862 // If Vec is just a SCALAR_TO_VECTOR, then use the scalar integer directly.
9863 SDValue VecBC = peekThroughBitcasts(V: Vec);
9864 if (VecBC.getOpcode() == ISD::SCALAR_TO_VECTOR) {
9865 SDValue Src = VecBC.getOperand(i: 0);
9866 Src = DAG.getBitcast(VT: Src.getValueType().changeTypeToInteger(), V: Src);
9867 Vec = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: IntVT);
9868 }
9869
9870 unsigned EltSize = EltVT.getSizeInBits();
9871 assert(isPowerOf2_32(EltSize));
9872
9873 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9874
9875 // Convert vector index to bit-index (* EltSize)
9876 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9877
9878 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9879 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: IntVT, N1: BC, N2: ScaledIdx);
9880
9881 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
9882 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Elt);
9883 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResultVT, Operand: Result);
9884 }
9885
9886 return DAG.getAnyExtOrTrunc(Op: Elt, DL: SL, VT: ResultVT);
9887}
9888
9889static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
9890 assert(Elt % 2 == 0);
9891 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
9892}
9893
9894static bool elementPairIsOddToEven(ArrayRef<int> Mask, int Elt) {
9895 assert(Elt % 2 == 0);
9896 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
9897 !(Mask[Elt + 1] & 1);
9898}
9899
9900SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
9901 SelectionDAG &DAG) const {
9902 SDLoc SL(Op);
9903 EVT ResultVT = Op.getValueType();
9904 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
9905 MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
9906 const int NewSrcNumElts = 2;
9907 MVT PackVT = MVT::getVectorVT(VT: EltVT, NumElements: NewSrcNumElts);
9908 int SrcNumElts = Op.getOperand(i: 0).getValueType().getVectorNumElements();
9909
9910 // Break up the shuffle into registers sized pieces.
9911 //
9912 // We're trying to form sub-shuffles that the register allocation pipeline
9913 // won't be able to figure out, like how to use v_pk_mov_b32 to do a register
9914 // blend or 16-bit op_sel. It should be able to figure out how to reassemble a
9915 // pair of copies into a consecutive register copy, so use the ordinary
9916 // extract_vector_elt lowering unless we can use the shuffle.
9917 //
9918 // TODO: This is a bit of hack, and we should probably always use
9919 // extract_subvector for the largest possible subvector we can (or at least
9920 // use it for PackVT aligned pieces). However we have worse support for
9921 // combines on them don't directly treat extract_subvector / insert_subvector
9922 // as legal. The DAG scheduler also ends up doing a worse job with the
9923 // extract_subvectors.
9924 const bool ShouldUseConsecutiveExtract = EltVT.getSizeInBits() == 16;
9925
9926 // vector_shuffle <0,1,6,7> lhs, rhs
9927 // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
9928 //
9929 // vector_shuffle <6,7,2,3> lhs, rhs
9930 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
9931 //
9932 // vector_shuffle <6,7,0,1> lhs, rhs
9933 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
9934
9935 // Avoid scalarizing when both halves are reading from consecutive elements.
9936
9937 // If we're treating 2 element shuffles as legal, also create odd-to-even
9938 // shuffles of neighboring pairs.
9939 //
9940 // vector_shuffle <3,2,7,6> lhs, rhs
9941 // -> concat_vectors vector_shuffle <1, 0> (extract_subvector lhs, 0)
9942 // vector_shuffle <1, 0> (extract_subvector rhs, 2)
9943
9944 SmallVector<SDValue, 16> Pieces;
9945 for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
9946 if (ShouldUseConsecutiveExtract &&
9947 elementPairIsContiguous(Mask: SVN->getMask(), Elt: I)) {
9948 const int Idx = SVN->getMaskElt(Idx: I);
9949 int VecIdx = Idx < SrcNumElts ? 0 : 1;
9950 int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
9951 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT,
9952 N1: SVN->getOperand(Num: VecIdx),
9953 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
9954 Pieces.push_back(Elt: SubVec);
9955 } else if (elementPairIsOddToEven(Mask: SVN->getMask(), Elt: I) &&
9956 isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: PackVT)) {
9957 int Idx0 = SVN->getMaskElt(Idx: I);
9958 int Idx1 = SVN->getMaskElt(Idx: I + 1);
9959
9960 SDValue SrcOp0 = SVN->getOperand(Num: 0);
9961 SDValue SrcOp1 = SrcOp0;
9962 if (Idx0 >= SrcNumElts) {
9963 SrcOp0 = SVN->getOperand(Num: 1);
9964 Idx0 -= SrcNumElts;
9965 }
9966
9967 if (Idx1 >= SrcNumElts) {
9968 SrcOp1 = SVN->getOperand(Num: 1);
9969 Idx1 -= SrcNumElts;
9970 }
9971
9972 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
9973 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
9974
9975 // Extract nearest even aligned piece.
9976 SDValue SubVec0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp0,
9977 N2: DAG.getConstant(Val: AlignedIdx0, DL: SL, VT: MVT::i32));
9978 SDValue SubVec1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp1,
9979 N2: DAG.getConstant(Val: AlignedIdx1, DL: SL, VT: MVT::i32));
9980
9981 int NewMaskIdx0 = Idx0 - AlignedIdx0;
9982 int NewMaskIdx1 = Idx1 - AlignedIdx1;
9983
9984 SDValue Result0 = SubVec0;
9985 SDValue Result1 = SubVec0;
9986
9987 if (SubVec0 != SubVec1) {
9988 NewMaskIdx1 += NewSrcNumElts;
9989 Result1 = SubVec1;
9990 } else {
9991 Result1 = DAG.getPOISON(VT: PackVT);
9992 }
9993
9994 SDValue Shuf = DAG.getVectorShuffle(VT: PackVT, dl: SL, N1: Result0, N2: Result1,
9995 Mask: {NewMaskIdx0, NewMaskIdx1});
9996 Pieces.push_back(Elt: Shuf);
9997 } else {
9998 const int Idx0 = SVN->getMaskElt(Idx: I);
9999 const int Idx1 = SVN->getMaskElt(Idx: I + 1);
10000 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
10001 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
10002 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
10003 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
10004
10005 SDValue Vec0 = SVN->getOperand(Num: VecIdx0);
10006 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec0,
10007 N2: DAG.getSignedConstant(Val: EltIdx0, DL: SL, VT: MVT::i32));
10008
10009 SDValue Vec1 = SVN->getOperand(Num: VecIdx1);
10010 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec1,
10011 N2: DAG.getSignedConstant(Val: EltIdx1, DL: SL, VT: MVT::i32));
10012 Pieces.push_back(Elt: DAG.getBuildVector(VT: PackVT, DL: SL, Ops: {Elt0, Elt1}));
10013 }
10014 }
10015
10016 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: ResultVT, Ops: Pieces);
10017}
10018
10019SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
10020 SelectionDAG &DAG) const {
10021 SDValue SVal = Op.getOperand(i: 0);
10022 EVT ResultVT = Op.getValueType();
10023 EVT SValVT = SVal.getValueType();
10024 SDValue UndefVal = DAG.getPOISON(VT: SValVT);
10025 SDLoc SL(Op);
10026
10027 SmallVector<SDValue, 8> VElts;
10028 VElts.push_back(Elt: SVal);
10029 for (int I = 1, E = ResultVT.getVectorNumElements(); I < E; ++I)
10030 VElts.push_back(Elt: UndefVal);
10031
10032 return DAG.getBuildVector(VT: ResultVT, DL: SL, Ops: VElts);
10033}
10034
10035SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
10036 SelectionDAG &DAG) const {
10037 SDLoc SL(Op);
10038 EVT VT = Op.getValueType();
10039
10040 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
10041 assert(!Subtarget->hasVOP3PInsts() && "this should be legal");
10042
10043 SDValue Lo = Op.getOperand(i: 0);
10044 SDValue Hi = Op.getOperand(i: 1);
10045
10046 // Avoid adding defined bits with the zero_extend.
10047 if (Hi.isUndef()) {
10048 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
10049 SDValue ExtLo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
10050 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ExtLo);
10051 }
10052
10053 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Hi);
10054 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Hi);
10055
10056 SDValue ShlHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Hi,
10057 N2: DAG.getConstant(Val: 16, DL: SL, VT: MVT::i32));
10058 if (Lo.isUndef())
10059 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ShlHi);
10060
10061 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
10062 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
10063
10064 SDValue Or =
10065 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Lo, N2: ShlHi, Flags: SDNodeFlags::Disjoint);
10066 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Or);
10067 }
10068
10069 // Split into 2-element chunks.
10070 const unsigned NumParts = VT.getVectorNumElements() / 2;
10071 EVT PartVT = MVT::getVectorVT(VT: VT.getVectorElementType().getSimpleVT(), NumElements: 2);
10072 MVT PartIntVT = MVT::getIntegerVT(BitWidth: PartVT.getSizeInBits());
10073
10074 SmallVector<SDValue> Casts;
10075 for (unsigned P = 0; P < NumParts; ++P) {
10076 SDValue Vec = DAG.getBuildVector(
10077 VT: PartVT, DL: SL, Ops: {Op.getOperand(i: P * 2), Op.getOperand(i: P * 2 + 1)});
10078 Casts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: PartIntVT, Operand: Vec));
10079 }
10080
10081 SDValue Blend =
10082 DAG.getBuildVector(VT: MVT::getVectorVT(VT: PartIntVT, NumElements: NumParts), DL: SL, Ops: Casts);
10083 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Blend);
10084}
10085
10086bool SITargetLowering::isOffsetFoldingLegal(
10087 const GlobalAddressSDNode *GA) const {
10088 // Named barriers have fixed, non-relocated LDS addresses, so a constant
10089 // offset into an array of them can be folded into the address.
10090 if (GA->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
10091 const auto *GV = dyn_cast<GlobalVariable>(Val: GA->getGlobal());
10092 return GV && AMDGPU::isNamedBarrier(GV: *GV);
10093 }
10094
10095 // OSes that use ELF REL relocations (instead of RELA) can only store a
10096 // 32-bit addend in the instruction, so it is not safe to allow offset folding
10097 // which can create arbitrary 64-bit addends. (This is only a problem for
10098 // R_AMDGPU_*32_HI relocations since other relocation types are unaffected by
10099 // the high 32 bits of the addend.)
10100 //
10101 // This should be kept in sync with how HasRelocationAddend is initialized in
10102 // the constructor of ELFAMDGPUAsmBackend.
10103 if (!Subtarget->isAmdHsaOS())
10104 return false;
10105
10106 // We can fold offsets for anything that doesn't require a GOT relocation.
10107 return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
10108 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
10109 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
10110 !shouldEmitGOTReloc(GV: GA->getGlobal());
10111}
10112
10113static SDValue
10114buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
10115 const SDLoc &DL, int64_t Offset, EVT PtrVT,
10116 unsigned GAFlags = SIInstrInfo::MO_NONE) {
10117 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
10118 // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
10119 // lowered to the following code sequence:
10120 //
10121 // For constant address space:
10122 // s_getpc_b64 s[0:1]
10123 // s_add_u32 s0, s0, $symbol
10124 // s_addc_u32 s1, s1, 0
10125 //
10126 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10127 // a fixup or relocation is emitted to replace $symbol with a literal
10128 // constant, which is a pc-relative offset from the encoding of the $symbol
10129 // operand to the global variable.
10130 //
10131 // For global address space:
10132 // s_getpc_b64 s[0:1]
10133 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
10134 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
10135 //
10136 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10137 // fixups or relocations are emitted to replace $symbol@*@lo and
10138 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
10139 // which is a 64-bit pc-relative offset from the encoding of the $symbol
10140 // operand to the global variable.
10141 if (((const GCNSubtarget &)DAG.getSubtarget()).has64BitLiterals()) {
10142 assert(GAFlags != SIInstrInfo::MO_NONE);
10143
10144 SDValue Ptr =
10145 DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64, offset: Offset, TargetFlags: GAFlags + 2);
10146 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET64, DL, VT: PtrVT, Operand: Ptr);
10147 }
10148
10149 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags);
10150 SDValue PtrHi;
10151 if (GAFlags == SIInstrInfo::MO_NONE)
10152 PtrHi = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
10153 else
10154 PtrHi = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags + 1);
10155 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET, DL, VT: PtrVT, N1: PtrLo, N2: PtrHi);
10156}
10157
10158SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
10159 SDValue Op,
10160 SelectionDAG &DAG) const {
10161 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Val&: Op);
10162 SDLoc DL(GSD);
10163 EVT PtrVT = Op.getValueType();
10164
10165 const GlobalValue *GV = GSD->getGlobal();
10166 const unsigned AS = GSD->getAddressSpace();
10167 if (((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER) &&
10168 shouldUseLDSConstAddress(GV)) ||
10169 AS == AMDGPUAS::REGION_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS) {
10170 if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
10171 const GlobalVariable &GVar = *cast<GlobalVariable>(Val: GV);
10172 // HIP uses an unsized array `extern __shared__ T s[]` or similar
10173 // zero-sized type in other languages to declare the dynamic shared
10174 // memory which size is not known at the compile time. They will be
10175 // allocated by the runtime and placed directly after the static
10176 // allocated ones. They all share the same offset.
10177 if (GVar.getGlobalSize(DL: GVar.getDataLayout()) == 0) {
10178 assert(PtrVT == MVT::i32 && "32-bit pointer is expected.");
10179 // Adjust alignment for that dynamic shared memory array.
10180 Function &F = DAG.getMachineFunction().getFunction();
10181 MFI->setDynLDSAlign(F, GV: GVar);
10182 MFI->setUsesDynamicLDS(true);
10183 return SDValue(
10184 DAG.getMachineNode(Opcode: AMDGPU::GET_GROUPSTATICSIZE, dl: DL, VT: PtrVT), 0);
10185 }
10186 }
10187 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
10188 }
10189
10190 if (AS == AMDGPUAS::BARRIER) {
10191 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
10192 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10193 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0);
10194 }
10195
10196 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
10197 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
10198 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10199 return DAG.getNode(Opcode: AMDGPUISD::LDS, DL, VT: MVT::i32, Operand: GA);
10200 }
10201
10202 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS()) {
10203 if (Subtarget->has64BitLiterals()) {
10204 SDValue Addr = DAG.getTargetGlobalAddress(
10205 GV, DL, VT: MVT::i64, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS64);
10206 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B64, dl: DL, VT: MVT::i64, Op1: Addr),
10207 0);
10208 }
10209
10210 SDValue AddrLo = DAG.getTargetGlobalAddress(
10211 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_LO);
10212 AddrLo = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrLo), 0};
10213
10214 SDValue AddrHi = DAG.getTargetGlobalAddress(
10215 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_HI);
10216 AddrHi = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrHi), 0};
10217
10218 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: AddrLo, N2: AddrHi);
10219 }
10220
10221 if (shouldEmitFixup(GV))
10222 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT);
10223
10224 if (shouldEmitPCReloc(GV))
10225 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT,
10226 GAFlags: SIInstrInfo::MO_REL32);
10227
10228 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, Offset: 0, PtrVT,
10229 GAFlags: SIInstrInfo::MO_GOTPCREL32);
10230 PointerType *PtrTy =
10231 PointerType::get(C&: *DAG.getContext(), AddressSpace: AMDGPUAS::CONSTANT_ADDRESS);
10232 const DataLayout &DataLayout = DAG.getDataLayout();
10233 Align Alignment = DataLayout.getABITypeAlign(Ty: PtrTy);
10234 MachinePointerInfo PtrInfo =
10235 MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction());
10236
10237 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: GOTAddr, PtrInfo, Alignment,
10238 MMOFlags: MachineMemOperand::MODereferenceable |
10239 MachineMemOperand::MOInvariant);
10240}
10241
10242SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
10243 SelectionDAG &DAG) const {
10244 // TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
10245 const Function &Fn = DAG.getMachineFunction().getFunction();
10246 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10247 Fn, "unsupported external symbol", Op.getDebugLoc()));
10248 return DAG.getPOISON(VT: Op.getValueType());
10249}
10250
10251SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
10252 const SDLoc &DL, SDValue V) const {
10253 // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
10254 // the destination register.
10255 //
10256 // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
10257 // so we will end up with redundant moves to m0.
10258 //
10259 // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
10260
10261 // A Null SDValue creates a glue result.
10262 SDNode *M0 = DAG.getMachineNode(Opcode: AMDGPU::SI_INIT_M0, dl: DL, VT1: MVT::Other, VT2: MVT::Glue,
10263 Op1: V, Op2: Chain);
10264 return SDValue(M0, 0);
10265}
10266
10267SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
10268 MVT VT,
10269 unsigned Offset) const {
10270 SDLoc SL(Op);
10271 SDValue Param = lowerKernargMemParameter(
10272 DAG, VT: MVT::i32, MemVT: MVT::i32, SL, Chain: DAG.getEntryNode(), Offset, Alignment: Align(4), Signed: false);
10273 // The local size values will have the hi 16-bits as zero.
10274 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Param,
10275 N2: DAG.getValueType(VT));
10276}
10277
10278static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10279 EVT VT) {
10280 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10281 DAG.getMachineFunction().getFunction(),
10282 "non-hsa intrinsic with hsa target", DL.getDebugLoc()));
10283 return DAG.getPOISON(VT);
10284}
10285
10286static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10287 EVT VT) {
10288 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10289 DAG.getMachineFunction().getFunction(),
10290 "intrinsic not supported on subtarget", DL.getDebugLoc()));
10291 return DAG.getPOISON(VT);
10292}
10293
10294static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
10295 ArrayRef<SDValue> Elts) {
10296 assert(!Elts.empty());
10297 MVT Type;
10298 unsigned NumElts = Elts.size();
10299
10300 if (NumElts <= 12) {
10301 Type = MVT::getVectorVT(VT: MVT::f32, NumElements: NumElts);
10302 } else {
10303 assert(Elts.size() <= 16);
10304 Type = MVT::v16f32;
10305 NumElts = 16;
10306 }
10307
10308 SmallVector<SDValue, 16> VecElts(NumElts);
10309 for (unsigned i = 0; i < Elts.size(); ++i) {
10310 SDValue Elt = Elts[i];
10311 if (Elt.getValueType() != MVT::f32)
10312 Elt = DAG.getBitcast(VT: MVT::f32, V: Elt);
10313 VecElts[i] = Elt;
10314 }
10315 for (unsigned i = Elts.size(); i < NumElts; ++i)
10316 VecElts[i] = DAG.getPOISON(VT: MVT::f32);
10317
10318 if (NumElts == 1)
10319 return VecElts[0];
10320 return DAG.getBuildVector(VT: Type, DL, Ops: VecElts);
10321}
10322
10323static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT,
10324 SDValue Src, int ExtraElts) {
10325 EVT SrcVT = Src.getValueType();
10326
10327 SmallVector<SDValue, 8> Elts;
10328
10329 if (SrcVT.isVector())
10330 DAG.ExtractVectorElements(Op: Src, Args&: Elts);
10331 else
10332 Elts.push_back(Elt: Src);
10333
10334 SDValue Undef = DAG.getPOISON(VT: SrcVT.getScalarType());
10335 while (ExtraElts--)
10336 Elts.push_back(Elt: Undef);
10337
10338 return DAG.getBuildVector(VT: CastVT, DL, Ops: Elts);
10339}
10340
10341// Re-construct the required return value for a image load intrinsic.
10342// This is more complicated due to the optional use TexFailCtrl which means the
10343// required return type is an aggregate
10344static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result,
10345 ArrayRef<EVT> ResultTypes, bool IsTexFail,
10346 bool Unpacked, bool IsD16, int DMaskPop,
10347 int NumVDataDwords, bool IsAtomicPacked16Bit,
10348 const SDLoc &DL) {
10349 // Determine the required return type. This is the same regardless of
10350 // IsTexFail flag
10351 EVT ReqRetVT = ResultTypes[0];
10352 int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1;
10353 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
10354 ? (ReqRetNumElts + 1) / 2
10355 : ReqRetNumElts;
10356
10357 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
10358
10359 MVT DataDwordVT =
10360 NumDataDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: NumDataDwords);
10361
10362 MVT MaskPopVT =
10363 MaskPopDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: MaskPopDwords);
10364
10365 SDValue Data(Result, 0);
10366 SDValue TexFail;
10367
10368 if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) {
10369 SDValue ZeroIdx = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
10370 if (MaskPopVT.isVector()) {
10371 Data = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: MaskPopVT,
10372 N1: SDValue(Result, 0), N2: ZeroIdx);
10373 } else {
10374 Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MaskPopVT,
10375 N1: SDValue(Result, 0), N2: ZeroIdx);
10376 }
10377 }
10378
10379 if (DataDwordVT.isVector() && !IsAtomicPacked16Bit)
10380 Data = padEltsToUndef(DAG, DL, CastVT: DataDwordVT, Src: Data,
10381 ExtraElts: NumDataDwords - MaskPopDwords);
10382
10383 if (IsD16)
10384 Data = adjustLoadValueTypeImpl(Result: Data, LoadVT: ReqRetVT, DL, DAG, Unpacked);
10385
10386 EVT LegalReqRetVT = ReqRetVT;
10387 if (!ReqRetVT.isVector()) {
10388 if (!Data.getValueType().isInteger())
10389 Data = DAG.getNode(Opcode: ISD::BITCAST, DL,
10390 VT: Data.getValueType().changeTypeToInteger(), Operand: Data);
10391 Data = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ReqRetVT.changeTypeToInteger(), Operand: Data);
10392 } else {
10393 // We need to widen the return vector to a legal type
10394 if ((ReqRetVT.getVectorNumElements() % 2) == 1 &&
10395 ReqRetVT.getVectorElementType().getSizeInBits() == 16) {
10396 LegalReqRetVT =
10397 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ReqRetVT.getVectorElementType(),
10398 NumElements: ReqRetVT.getVectorNumElements() + 1);
10399 }
10400 }
10401 Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LegalReqRetVT, Operand: Data);
10402
10403 if (IsTexFail) {
10404 TexFail =
10405 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: SDValue(Result, 0),
10406 N2: DAG.getConstant(Val: MaskPopDwords, DL, VT: MVT::i32));
10407
10408 return DAG.getMergeValues(Ops: {Data, TexFail, SDValue(Result, 1)}, dl: DL);
10409 }
10410
10411 if (Result->getNumValues() == 1)
10412 return Data;
10413
10414 return DAG.getMergeValues(Ops: {Data, SDValue(Result, 1)}, dl: DL);
10415}
10416
10417static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
10418 SDValue *LWE, bool &IsTexFail) {
10419 auto *TexFailCtrlConst = cast<ConstantSDNode>(Val: TexFailCtrl.getNode());
10420
10421 uint64_t Value = TexFailCtrlConst->getZExtValue();
10422 if (Value) {
10423 IsTexFail = true;
10424 }
10425
10426 SDLoc DL(TexFailCtrlConst);
10427 *TFE = DAG.getTargetConstant(Val: (Value & 0x1) ? 1 : 0, DL, VT: MVT::i32);
10428 Value &= ~(uint64_t)0x1;
10429 *LWE = DAG.getTargetConstant(Val: (Value & 0x2) ? 1 : 0, DL, VT: MVT::i32);
10430 Value &= ~(uint64_t)0x2;
10431
10432 return Value == 0;
10433}
10434
10435static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op,
10436 MVT PackVectorVT,
10437 SmallVectorImpl<SDValue> &PackedAddrs,
10438 unsigned DimIdx, unsigned EndIdx,
10439 unsigned NumGradients) {
10440 SDLoc DL(Op);
10441 for (unsigned I = DimIdx; I < EndIdx; I++) {
10442 SDValue Addr = Op.getOperand(i: I);
10443
10444 // Gradients are packed with undef for each coordinate.
10445 // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this:
10446 // 1D: undef,dx/dh; undef,dx/dv
10447 // 2D: dy/dh,dx/dh; dy/dv,dx/dv
10448 // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv
10449 if (((I + 1) >= EndIdx) ||
10450 ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 ||
10451 I == DimIdx + NumGradients - 1))) {
10452 if (Addr.getValueType() != MVT::i16)
10453 Addr = DAG.getBitcast(VT: MVT::i16, V: Addr);
10454 Addr = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Addr);
10455 } else {
10456 Addr = DAG.getBuildVector(VT: PackVectorVT, DL, Ops: {Addr, Op.getOperand(i: I + 1)});
10457 I++;
10458 }
10459 Addr = DAG.getBitcast(VT: MVT::f32, V: Addr);
10460 PackedAddrs.push_back(Elt: Addr);
10461 }
10462}
10463
10464/// Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and
10465/// return poison values of \p ResultTypes, preserving the chain if present.
10466static SDValue diagnoseUnsupportedImage(SelectionDAG &DAG, SDValue Op,
10467 ArrayRef<EVT> ResultTypes,
10468 const SDLoc &DL, const Twine &Msg) {
10469 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10470 DAG.getMachineFunction().getFunction(), Msg, DL.getDebugLoc()));
10471 return DAG.getErrorMergeValues(ResultTypes, Chain: Op.getOperand(i: 0), dl: DL);
10472}
10473
10474SDValue SITargetLowering::lowerImage(SDValue Op,
10475 const AMDGPU::ImageDimIntrinsicInfo *Intr,
10476 SelectionDAG &DAG, bool WithChain) const {
10477 SDLoc DL(Op);
10478 MachineFunction &MF = DAG.getMachineFunction();
10479 const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
10480 unsigned IntrOpcode = Intr->BaseOpcode;
10481 // For image atomic: use no-return opcode if result is unused.
10482 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode &&
10483 !Op.getNode()->hasAnyUseOfValue(Value: 0))
10484 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
10485 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
10486 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
10487 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
10488 bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI: *Subtarget);
10489 bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
10490 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
10491 bool IsGFX13 = AMDGPU::isGFX13(STI: *Subtarget);
10492
10493 SmallVector<EVT, 3> ResultTypes(Op->values());
10494 SmallVector<EVT, 3> OrigResultTypes(Op->values());
10495 if (BaseOpcode->NoReturn && BaseOpcode->Atomic)
10496 ResultTypes.erase(CI: &ResultTypes[0]);
10497
10498 bool IsD16 = false;
10499 bool IsG16 = false;
10500 bool IsA16 = false;
10501 SDValue VData;
10502 int NumVDataDwords = 0;
10503 bool AdjustRetType = false;
10504 bool IsAtomicPacked16Bit = false;
10505
10506 // Offset of intrinsic arguments
10507 const unsigned ArgOffset = WithChain ? 2 : 1;
10508
10509 unsigned DMask;
10510 unsigned DMaskLanes = 0;
10511
10512 if (BaseOpcode->Atomic) {
10513 VData = Op.getOperand(i: 2);
10514
10515 IsAtomicPacked16Bit =
10516 (IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
10517 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16_NORTN ||
10518 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
10519 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
10520
10521 if (!IsAtomicPacked16Bit && VData.getValueSizeInBits() != 32 &&
10522 VData.getValueSizeInBits() != 64) {
10523 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10524 Msg: "unsupported image atomic data type");
10525 }
10526
10527 bool Is64Bit = VData.getValueSizeInBits() == 64;
10528 if (BaseOpcode->AtomicX2) {
10529 SDValue VData2 = Op.getOperand(i: 3);
10530 VData = DAG.getBuildVector(VT: Is64Bit ? MVT::v2i64 : MVT::v2i32, DL,
10531 Ops: {VData, VData2});
10532 if (Is64Bit)
10533 VData = DAG.getBitcast(VT: MVT::v4i32, V: VData);
10534
10535 if (!BaseOpcode->NoReturn)
10536 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
10537
10538 DMask = Is64Bit ? 0xf : 0x3;
10539 NumVDataDwords = Is64Bit ? 4 : 2;
10540 } else {
10541 DMask = Is64Bit ? 0x3 : 0x1;
10542 NumVDataDwords = Is64Bit ? 2 : 1;
10543 }
10544 } else {
10545 DMask = Op->getConstantOperandVal(Num: ArgOffset + Intr->DMaskIndex);
10546 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
10547
10548 if (BaseOpcode->Store) {
10549 VData = Op.getOperand(i: 2);
10550
10551 MVT StoreVT = VData.getSimpleValueType();
10552 MVT StoreScalarVT = StoreVT.getScalarType();
10553 if (StoreScalarVT != MVT::f16 && StoreScalarVT.getSizeInBits() != 32 &&
10554 StoreScalarVT.getSizeInBits() != 64) {
10555 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10556 Msg: "unsupported image store data type");
10557 }
10558 if (StoreScalarVT == MVT::f16) {
10559 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10560 return Op; // D16 is unsupported for this instruction
10561
10562 IsD16 = true;
10563 VData = handleD16VData(VData, DAG, ImageStore: true);
10564 }
10565
10566 NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
10567 } else if (!BaseOpcode->NoReturn) {
10568 // Work out the num dwords based on the dmask popcount and underlying type
10569 // and whether packing is supported.
10570 MVT LoadVT = ResultTypes[0].getSimpleVT();
10571 MVT LoadScalarVT = LoadVT.getScalarType();
10572 if (LoadScalarVT != MVT::f16 && LoadScalarVT.getSizeInBits() != 32 &&
10573 LoadScalarVT.getSizeInBits() != 64) {
10574 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10575 Msg: "unsupported image load data type");
10576 }
10577 if (LoadScalarVT == MVT::f16) {
10578 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10579 return Op; // D16 is unsupported for this instruction
10580
10581 IsD16 = true;
10582 }
10583
10584 // Confirm that the return type is large enough for the dmask specified
10585 if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) ||
10586 (!LoadVT.isVector() && DMaskLanes > 1))
10587 return Op;
10588
10589 // The sq block of gfx8 and gfx9 do not estimate register use correctly
10590 // for d16 image_gather4, image_gather4_l, and image_gather4_lz
10591 // instructions.
10592 if (IsD16 && !Subtarget->hasUnpackedD16VMem() &&
10593 !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug()))
10594 NumVDataDwords = (DMaskLanes + 1) / 2;
10595 else
10596 NumVDataDwords = DMaskLanes;
10597
10598 AdjustRetType = true;
10599 }
10600 }
10601
10602 unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd;
10603 SmallVector<SDValue, 4> VAddrs;
10604
10605 // Check for 16 bit addresses or derivatives and pack if true.
10606 MVT VAddrVT =
10607 Op.getOperand(i: ArgOffset + Intr->GradientStart).getSimpleValueType();
10608 MVT VAddrScalarVT = VAddrVT.getScalarType();
10609 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10610 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10611
10612 VAddrVT = Op.getOperand(i: ArgOffset + Intr->CoordStart).getSimpleValueType();
10613 VAddrScalarVT = VAddrVT.getScalarType();
10614 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10615 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10616
10617 // Push back extra arguments.
10618 for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) {
10619 if (IsA16 && (Op.getOperand(i: ArgOffset + I).getValueType() == MVT::f16)) {
10620 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
10621 // Special handling of bias when A16 is on. Bias is of type half but
10622 // occupies full 32-bit.
10623 SDValue Bias = DAG.getBuildVector(
10624 VT: MVT::v2f16, DL,
10625 Ops: {Op.getOperand(i: ArgOffset + I), DAG.getPOISON(VT: MVT::f16)});
10626 VAddrs.push_back(Elt: Bias);
10627 } else {
10628 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
10629 "Bias needs to be converted to 16 bit in A16 mode");
10630 VAddrs.push_back(Elt: Op.getOperand(i: ArgOffset + I));
10631 }
10632 }
10633
10634 if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) {
10635 // 16 bit gradients are supported, but are tied to the A16 control
10636 // so both gradients and addresses must be 16 bit
10637 LLVM_DEBUG(
10638 dbgs() << "Failed to lower image intrinsic: 16 bit addresses "
10639 "require 16 bit args for both gradients and addresses");
10640 return Op;
10641 }
10642
10643 if (IsA16) {
10644 if (!ST->hasA16()) {
10645 LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not "
10646 "support 16 bit addresses\n");
10647 return Op;
10648 }
10649 }
10650
10651 // We've dealt with incorrect input so we know that if IsA16, IsG16
10652 // are set then we have to compress/pack operands (either address,
10653 // gradient or both)
10654 // In the case where a16 and gradients are tied (no G16 support) then we
10655 // have already verified that both IsA16 and IsG16 are true
10656 if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) {
10657 // Activate g16
10658 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
10659 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
10660 IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16
10661 }
10662
10663 // Add gradients (packed or unpacked)
10664 if (IsG16) {
10665 // Pack the gradients
10666 // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart);
10667 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: GradPackVectorVT, PackedAddrs&: VAddrs,
10668 DimIdx: ArgOffset + Intr->GradientStart,
10669 EndIdx: ArgOffset + Intr->CoordStart, NumGradients: Intr->NumGradients);
10670 } else {
10671 for (unsigned I = ArgOffset + Intr->GradientStart;
10672 I < ArgOffset + Intr->CoordStart; I++)
10673 VAddrs.push_back(Elt: Op.getOperand(i: I));
10674 }
10675
10676 // Add addresses (packed or unpacked)
10677 if (IsA16) {
10678 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: AddrPackVectorVT, PackedAddrs&: VAddrs,
10679 DimIdx: ArgOffset + Intr->CoordStart, EndIdx: VAddrEnd,
10680 NumGradients: 0 /* No gradients */);
10681 } else {
10682 // Add uncompressed address
10683 for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++)
10684 VAddrs.push_back(Elt: Op.getOperand(i: I));
10685 }
10686
10687 // If the register allocator cannot place the address registers contiguously
10688 // without introducing moves, then using the non-sequential address encoding
10689 // is always preferable, since it saves VALU instructions and is usually a
10690 // wash in terms of code size or even better.
10691 //
10692 // However, we currently have no way of hinting to the register allocator that
10693 // MIMG addresses should be placed contiguously when it is possible to do so,
10694 // so force non-NSA for the common 2-address case as a heuristic.
10695 //
10696 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
10697 // allocation when possible.
10698 //
10699 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
10700 // set of the remaining addresses.
10701 const unsigned NSAMaxSize = ST->getNSAMaxSize(HasSampler: BaseOpcode->Sampler);
10702 const bool HasPartialNSAEncoding = ST->hasPartialNSAEncoding();
10703 const bool UseNSA = ST->hasNSAEncoding() &&
10704 VAddrs.size() >= ST->getNSAThreshold(MF) &&
10705 (VAddrs.size() <= NSAMaxSize || HasPartialNSAEncoding);
10706 const bool UsePartialNSA =
10707 UseNSA && HasPartialNSAEncoding && VAddrs.size() > NSAMaxSize;
10708
10709 SDValue VAddr;
10710 if (UsePartialNSA) {
10711 VAddr = getBuildDwordsVector(DAG, DL,
10712 Elts: ArrayRef(VAddrs).drop_front(N: NSAMaxSize - 1));
10713 } else if (!UseNSA) {
10714 VAddr = getBuildDwordsVector(DAG, DL, Elts: VAddrs);
10715 }
10716
10717 SDValue True = DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1);
10718 SDValue False = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1);
10719 SDValue Unorm;
10720 if (!BaseOpcode->Sampler) {
10721 Unorm = True;
10722 } else {
10723 uint64_t UnormConst =
10724 Op.getConstantOperandVal(i: ArgOffset + Intr->UnormIndex);
10725
10726 Unorm = UnormConst ? True : False;
10727 }
10728
10729 SDValue TFE;
10730 SDValue LWE;
10731 SDValue TexFail = Op.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex);
10732 bool IsTexFail = false;
10733 if (!parseTexFail(TexFailCtrl: TexFail, DAG, TFE: &TFE, LWE: &LWE, IsTexFail))
10734 return Op;
10735
10736 if (IsTexFail) {
10737 if (!DMaskLanes) {
10738 // Expecting to get an error flag since TFC is on - and dmask is 0
10739 // Force dmask to be at least 1 otherwise the instruction will fail
10740 DMask = 0x1;
10741 DMaskLanes = 1;
10742 NumVDataDwords = 1;
10743 }
10744 NumVDataDwords += 1;
10745 AdjustRetType = true;
10746 }
10747
10748 // Has something earlier tagged that the return type needs adjusting
10749 // This happens if the instruction is a load or has set TexFailCtrl flags
10750 if (AdjustRetType) {
10751 // NumVDataDwords reflects the true number of dwords required in the return
10752 // type
10753 if (DMaskLanes == 0 && !BaseOpcode->Store) {
10754 // This is a no-op load. This can be eliminated
10755 SDValue Undef = DAG.getPOISON(VT: Op.getValueType());
10756 if (isa<MemSDNode>(Val: Op))
10757 return DAG.getMergeValues(Ops: {Undef, Op.getOperand(i: 0)}, dl: DL);
10758 return Undef;
10759 }
10760
10761 EVT NewVT = NumVDataDwords > 1 ? EVT::getVectorVT(Context&: *DAG.getContext(),
10762 VT: MVT::i32, NumElements: NumVDataDwords)
10763 : MVT::i32;
10764
10765 ResultTypes[0] = NewVT;
10766 if (ResultTypes.size() == 3) {
10767 // Original result was aggregate type used for TexFailCtrl results
10768 // The actual instruction returns as a vector type which has now been
10769 // created. Remove the aggregate result.
10770 ResultTypes.erase(CI: &ResultTypes[1]);
10771 }
10772 }
10773
10774 unsigned CPol = Op.getConstantOperandVal(i: ArgOffset + Intr->CachePolicyIndex);
10775 // Keep GLC only when the atomic's result is actually used.
10776 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
10777 CPol |= AMDGPU::CPol::GLC;
10778 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
10779 AMDGPU::CPol::VOLATILE))
10780 return Op;
10781
10782 SmallVector<SDValue, 26> Ops;
10783 if (BaseOpcode->Store || BaseOpcode->Atomic)
10784 Ops.push_back(Elt: VData); // vdata
10785 if (UsePartialNSA) {
10786 append_range(C&: Ops, R: ArrayRef(VAddrs).take_front(N: NSAMaxSize - 1));
10787 Ops.push_back(Elt: VAddr);
10788 } else if (UseNSA)
10789 append_range(C&: Ops, R&: VAddrs);
10790 else
10791 Ops.push_back(Elt: VAddr);
10792 SDValue Rsrc = Op.getOperand(i: ArgOffset + Intr->RsrcIndex);
10793 EVT RsrcVT = Rsrc.getValueType();
10794 bool IndexedRsrc = IsGFX13 && RsrcVT == MVT::i32;
10795 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32 && !IndexedRsrc)
10796 return Op;
10797 Ops.push_back(Elt: Rsrc);
10798 bool IsSampIndexed = false;
10799 if (BaseOpcode->Sampler) {
10800 SDValue Samp = Op.getOperand(i: ArgOffset + Intr->SampIndex);
10801 EVT SampVT = Samp.getValueType();
10802 IsSampIndexed = IsGFX13 && SampVT == MVT::i32;
10803 if (SampVT != MVT::v4i32 && !IsSampIndexed)
10804 return Op;
10805 Ops.push_back(Elt: Samp);
10806 }
10807 Ops.push_back(Elt: DAG.getTargetConstant(Val: DMask, DL, VT: MVT::i32));
10808 if (IsGFX10Plus)
10809 Ops.push_back(Elt: DAG.getTargetConstant(Val: DimInfo->Encoding, DL, VT: MVT::i32));
10810 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10811 Ops.push_back(Elt: Unorm);
10812 Ops.push_back(Elt: DAG.getTargetConstant(Val: CPol, DL, VT: MVT::i32));
10813 Ops.push_back(Elt: IsA16 && // r128, a16 for gfx9
10814 ST->hasFeature(Feature: AMDGPU::FeatureR128A16)
10815 ? True
10816 : False);
10817 if (IsGFX10Plus)
10818 Ops.push_back(Elt: IsA16 ? True : False);
10819
10820 if (!Subtarget->hasGFX90AInsts())
10821 Ops.push_back(Elt: TFE); // tfe
10822 else if (TFE->getAsZExtVal()) {
10823 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10824 DAG.getMachineFunction().getFunction(),
10825 "TFE is not supported on this GPU", DL.getDebugLoc()));
10826 }
10827
10828 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10829 Ops.push_back(Elt: LWE); // lwe
10830 if (!IsGFX10Plus)
10831 Ops.push_back(Elt: DimInfo->DA ? True : False);
10832 if (BaseOpcode->HasD16)
10833 Ops.push_back(Elt: IsD16 ? True : False);
10834 if (isa<MemSDNode>(Val: Op))
10835 Ops.push_back(Elt: Op.getOperand(i: 0)); // chain
10836
10837 int NumVAddrDwords =
10838 UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32;
10839 int Opcode = -1;
10840
10841 if (IsGFX13) {
10842 Opcode =
10843 AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13, VDataDwords: NumVDataDwords,
10844 VAddrDwords: NumVAddrDwords, IndexedRsrc, IndexedSamp: IsSampIndexed);
10845 } else if (IsGFX12Plus) {
10846 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
10847 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10848 } else if (IsGFX11Plus) {
10849 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10850 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
10851 : AMDGPU::MIMGEncGfx11Default,
10852 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10853 } else if (IsGFX10Plus) {
10854 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10855 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
10856 : AMDGPU::MIMGEncGfx10Default,
10857 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10858 } else {
10859 if (Subtarget->hasGFX90AInsts()) {
10860 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
10861 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10862 if (Opcode == -1) {
10863 return diagnoseUnsupportedImage(
10864 DAG, Op, ResultTypes: OrigResultTypes, DL,
10865 Msg: "requested image instruction is not supported on this GPU");
10866 }
10867 }
10868 if (Opcode == -1 &&
10869 Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10870 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
10871 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10872 if (Opcode == -1)
10873 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
10874 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10875 }
10876 if (Opcode == -1)
10877 return Op;
10878
10879 MachineSDNode *NewNode = DAG.getMachineNode(Opcode, dl: DL, ResultTys: ResultTypes, Ops);
10880 if (auto *MemOp = dyn_cast<MemSDNode>(Val&: Op)) {
10881 MachineMemOperand *MemRef = MemOp->getMemOperand();
10882 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
10883 }
10884
10885 if (BaseOpcode->NoReturn) {
10886 if (BaseOpcode->Atomic)
10887 return DAG.getMergeValues(
10888 Ops: {DAG.getPOISON(VT: OrigResultTypes[0]), SDValue(NewNode, 0)}, dl: DL);
10889
10890 return SDValue(NewNode, 0);
10891 }
10892
10893 if (BaseOpcode->AtomicX2) {
10894 SmallVector<SDValue, 1> Elt;
10895 DAG.ExtractVectorElements(Op: SDValue(NewNode, 0), Args&: Elt, Start: 0, Count: 1);
10896 return DAG.getMergeValues(Ops: {Elt[0], SDValue(NewNode, 1)}, dl: DL);
10897 }
10898
10899 return constructRetValue(DAG, Result: NewNode, ResultTypes: OrigResultTypes, IsTexFail,
10900 Unpacked: Subtarget->hasUnpackedD16VMem(), IsD16, DMaskPop: DMaskLanes,
10901 NumVDataDwords, IsAtomicPacked16Bit, DL);
10902}
10903
10904SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
10905 SDValue Chain, SDValue Rsrc,
10906 SDValue Offset, SDValue CachePolicy,
10907 SelectionDAG &DAG,
10908 MachineMemOperand *MMO) const {
10909 MachineFunction &MF = DAG.getMachineFunction();
10910 bool HasChainResult = MMO != nullptr;
10911
10912 // SBUFFER_LOAD only produces values that fill whole SGPRs, apart from the
10913 // subword loads below.
10914 bool IsSubwordLoad = (MemVT == MVT::i8 || MemVT == MVT::i16) &&
10915 Subtarget->hasScalarSubwordLoads();
10916 if ((!isTypeLegal(VT) || VT.getSizeInBits() % 32 != 0) && !IsSubwordLoad) {
10917 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10918 MF.getFunction(), "unsupported s_buffer_load result type",
10919 DL.getDebugLoc()));
10920 EVT ResultTypes[] = {VT, MVT::Other};
10921 return DAG.getErrorMergeValues(
10922 ResultTypes: ArrayRef(ResultTypes, HasChainResult ? 2 : 1), Chain, dl: DL);
10923 }
10924
10925 if (!HasChainResult) {
10926 const DataLayout &DataLayout = DAG.getDataLayout();
10927 Align Alignment =
10928 DataLayout.getABITypeAlign(Ty: MemVT.getTypeForEVT(Context&: *DAG.getContext()));
10929
10930 MMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
10931 F: MachineMemOperand::MOLoad |
10932 MachineMemOperand::MODereferenceable |
10933 MachineMemOperand::MOInvariant,
10934 Size: MemVT.getStoreSize(), BaseAlignment: Alignment);
10935 }
10936
10937 if (!Offset->isDivergent()) {
10938 SDValue Ops[] = {Chain, Rsrc, Offset, CachePolicy};
10939
10940 // Lower llvm.amdgcn.*s.buffer.load.{i,u}N intrinsics. First, generate
10941 // s_buffer_load_u* for signed and unsigned load instructions. Next, DAG
10942 // combiner tries to merge the s_buffer_load_uN with a sext instruction
10943 // (performSignExtendInRegCombine()) and it replaces s_buffer_load_uN with
10944 // s_buffer_load_iN.
10945 auto HandleScalarSubwordLoads = [&](unsigned Opcode) -> SDValue {
10946 SDValue BufferLoad = DAG.getMemIntrinsicNode(
10947 Opcode, dl: DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops, MemVT, MMO);
10948 SDValue LoadVal = DAG.getAnyExtOrTrunc(
10949 Op: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: BufferLoad), DL, VT);
10950 if (HasChainResult)
10951 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
10952 return LoadVal;
10953 };
10954 if (MemVT == MVT::i8 && Subtarget->hasScalarSubwordLoads())
10955 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_UBYTE);
10956
10957 if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
10958 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
10959
10960 // Widen vec3 load to vec4. Only 32-bit elements have a vec4 pattern.
10961 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
10962 VT.getVectorElementType().getSizeInBits() == 32 &&
10963 !Subtarget->hasScalarDwordx3Loads()) {
10964 EVT WidenedVT =
10965 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
10966 auto WidenedOp = DAG.getMemIntrinsicNode(
10967 Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL, VTList: DAG.getVTList(VT1: WidenedVT, VT2: MVT::Other),
10968 Ops, MemVT: WidenedVT,
10969 MMO: MF.getMachineMemOperand(MMO, Offset: 0, Size: WidenedVT.getStoreSize()));
10970 auto Subvector = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WidenedOp,
10971 N2: DAG.getVectorIdxConstant(Val: 0, DL));
10972 if (HasChainResult)
10973 return DAG.getMergeValues(Ops: {Subvector, WidenedOp.getValue(R: 1)}, dl: DL);
10974 return Subvector;
10975 }
10976
10977 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL,
10978 VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops, MemVT,
10979 MMO);
10980 }
10981
10982 // We have a divergent offset. Emit a MUBUF buffer load instead. We can
10983 // assume that the buffer is unswizzled.
10984 SDValue Ops[] = {
10985 Chain, // Chain
10986 Rsrc, // rsrc
10987 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
10988 {}, // voffset
10989 {}, // soffset
10990 {}, // offset
10991 CachePolicy, // cachepolicy
10992 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
10993 };
10994 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
10995 Subtarget->hasScalarSubwordLoads()) {
10996 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
10997 SDValue Load = handleByteShortBufferLoads(DAG, LoadVT: MemVT, DL, Ops, MMO);
10998 SDValue LoadVal = DAG.getAnyExtOrTrunc(Op: Load.getOperand(i: 0), DL, VT);
10999 if (HasChainResult)
11000 return DAG.getMergeValues(Ops: {LoadVal, Load.getOperand(i: 1)}, dl: DL);
11001 return LoadVal;
11002 }
11003
11004 SmallVector<SDValue, 4> Loads;
11005 unsigned NumLoads = 1;
11006 MVT LoadVT = VT.getSimpleVT();
11007 unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
11008 assert((LoadVT.getScalarType() == MVT::i32 ||
11009 LoadVT.getScalarType() == MVT::f32));
11010
11011 if (NumElts == 8 || NumElts == 16) {
11012 NumLoads = NumElts / 4;
11013 LoadVT = MVT::getVectorVT(VT: LoadVT.getScalarType(), NumElements: 4);
11014 }
11015
11016 SDVTList VTList = DAG.getVTList(VTs: {LoadVT, MVT::Other});
11017
11018 // Use the alignment to ensure that the required offsets will fit into the
11019 // immediate offsets.
11020 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3],
11021 Alignment: NumLoads > 1 ? Align(16 * NumLoads) : Align(4));
11022
11023 uint64_t InstOffset = Ops[5]->getAsZExtVal();
11024 unsigned LoadSize = LoadVT.getStoreSize();
11025 for (unsigned i = 0; i < NumLoads; ++i) {
11026 Ops[5] = DAG.getTargetConstant(Val: InstOffset + 16 * i, DL, VT: MVT::i32);
11027 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(MMO, Offset: 16 * i, Size: LoadSize);
11028 Loads.push_back(Elt: getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops,
11029 MemVT: LoadVT, MMO: LoadMMO, DAG));
11030 }
11031
11032 if (NumElts == 8 || NumElts == 16) {
11033 SDValue LoadVal = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Loads);
11034 if (HasChainResult) {
11035 SmallVector<SDValue, 4> LoadChains;
11036 for (SDValue Load : Loads)
11037 LoadChains.push_back(Elt: Load.getValue(R: 1));
11038 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: LoadChains);
11039 return DAG.getMergeValues(Ops: {LoadVal, Chain}, dl: DL);
11040 }
11041 return LoadVal;
11042 }
11043
11044 return Loads[0];
11045}
11046
11047SDValue SITargetLowering::lowerWaveID(SelectionDAG &DAG, SDValue Op) const {
11048 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
11049 if (!Subtarget->hasArchitectedSGPRs())
11050 return {};
11051 SDLoc SL(Op);
11052 MVT VT = MVT::i32;
11053 SDValue TTMP8 = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: AMDGPU::TTMP8, VT);
11054 return DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT, N1: TTMP8,
11055 N2: DAG.getConstant(Val: 25, DL: SL, VT), N3: DAG.getConstant(Val: 5, DL: SL, VT));
11056}
11057
11058SDValue SITargetLowering::lowerConstHwRegRead(SelectionDAG &DAG, SDValue Op,
11059 AMDGPU::Hwreg::Id HwReg,
11060 unsigned LowBit,
11061 unsigned Width) const {
11062 SDLoc SL(Op);
11063 using namespace AMDGPU::Hwreg;
11064 return {DAG.getMachineNode(
11065 Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT: MVT::i32,
11066 Op1: DAG.getTargetConstant(Val: HwregEncoding::encode(Values: HwReg, Values: LowBit, Values: Width),
11067 DL: SL, VT: MVT::i32)),
11068 0};
11069}
11070
11071SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
11072 unsigned Dim,
11073 const ArgDescriptor &Arg) const {
11074 SDLoc SL(Op);
11075 MachineFunction &MF = DAG.getMachineFunction();
11076 unsigned MaxID = Subtarget->getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: Dim);
11077 if (MaxID == 0)
11078 return DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
11079
11080 // It's undefined behavior if a function marked with the amdgpu-no-*
11081 // attributes uses the corresponding intrinsic.
11082 if (!Arg)
11083 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
11084
11085 SDValue Val = loadInputValue(DAG, RC: &AMDGPU::VGPR_32RegClass, VT: MVT::i32,
11086 SL: SDLoc(DAG.getEntryNode()), Arg);
11087
11088 // Don't bother inserting AssertZext for packed IDs since we're emitting the
11089 // masking operations anyway.
11090 //
11091 // TODO: We could assert the top bit is 0 for the source copy.
11092 if (Arg.isMasked())
11093 return Val;
11094
11095 // Preserve the known bits after expansion to a copy.
11096 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_width(Value: MaxID));
11097 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Val,
11098 N2: DAG.getValueType(SmallVT));
11099}
11100
11101SDValue SITargetLowering::lowerFromFP8(SDValue Op, bool IsBF8,
11102 SelectionDAG &DAG) const {
11103 SDLoc SL(Op);
11104 SDValue Src = Op.getOperand(i: 0);
11105 EVT DstVT = Op.getValueType();
11106 bool IsF16 = DstVT.getVectorElementType() == MVT::f16;
11107 assert((!IsF16 || Subtarget->hasFP8F16ConversionInsts()) &&
11108 "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
11109
11110 unsigned Opc;
11111 if (IsF16)
11112 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
11113 else
11114 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
11115
11116 // Pack the two i8 lanes into the integer type the packed HW node reads. The
11117 // f16 form takes i16 and the f32 form takes i32. v2i8 bitcasts to i16
11118 // directly and the f32 node reads the low half of an any-extended i32.
11119 EVT PackedVT =
11120 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstVT.getScalarSizeInBits());
11121 SDValue AsI16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
11122 SDValue Packed = DAG.getAnyExtOrTrunc(Op: AsI16, DL: SL, VT: PackedVT);
11123 return DAG.getNode(Opcode: Opc, DL: SL, VT: DstVT, Operand: Packed);
11124}
11125
11126SDValue
11127SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
11128 SelectionDAG &DAG) const {
11129 // Handle the OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 on subtargets
11130 // with matching HW conversions. Other formats use the generic expansion.
11131 APFloatBase::Semantics FPSemantic =
11132 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(i: 1));
11133 const bool IsFP8 = FPSemantic == APFloatBase::S_Float8E4M3FN;
11134 const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
11135 const bool IsE5M3 = FPSemantic == APFloatBase::S_Float8E5M3FNU;
11136 const bool HasE5M3ConversionInsts =
11137 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11138 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11139 if (!IsSupported)
11140 return SDValue();
11141
11142 EVT DstVT = Op.getValueType();
11143 // The custom action for a v2i8 source also reaches half conversions on
11144 // targets which only have FP8-to-f32 instructions.
11145 if (DstVT.getScalarType() == MVT::f16 &&
11146 !Subtarget->hasFP8F16ConversionInsts())
11147 return SDValue();
11148
11149 if (IsE5M3) {
11150 if (DstVT.getScalarType() != MVT::f32)
11151 return SDValue();
11152
11153 SDLoc SL(Op);
11154 SDValue Src = Op.getOperand(i: 0);
11155 assert((!DstVT.isVector() || DstVT == MVT::v2f32) &&
11156 "only the v2f32 vector result is custom lowered");
11157
11158 if (DstVT.isVector())
11159 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
11160 Src = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
11161
11162 auto ConvertByte = [&](unsigned ByteSel) {
11163 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_FP8_E5M3, DL: SL, VT: MVT::f32, N1: Src,
11164 N2: DAG.getTargetConstant(Val: ByteSel, DL: SL, VT: MVT::i32));
11165 };
11166
11167 if (!DstVT.isVector())
11168 return ConvertByte(0);
11169 return DAG.getBuildVector(VT: DstVT, DL: SL, Ops: {ConvertByte(0), ConvertByte(1)});
11170 }
11171
11172 if (!DstVT.isVector()) {
11173 SDValue Src = Op.getOperand(i: 0);
11174 if (Src.getValueType() != MVT::i32) {
11175 SDLoc SL(Op);
11176 SDValue SrcI32 = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
11177 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: SL, VT: DstVT, N1: SrcI32,
11178 N2: Op.getOperand(i: 1));
11179 }
11180 return Op;
11181 }
11182
11183 EVT EltVT = DstVT.getVectorElementType();
11184 if (EltVT == MVT::f16 || EltVT == MVT::f32)
11185 return lowerFromFP8(Op, IsBF8, DAG);
11186 return SDValue();
11187}
11188
11189SDValue SITargetLowering::lowerToFP8(SDValue Op, bool IsBF8, bool IsE5M3,
11190 SelectionDAG &DAG) const {
11191 SDLoc SL(Op);
11192 SDValue Src = Op.getOperand(i: 0);
11193 EVT ResVT = Op.getValueType();
11194 bool IsF16 = Src.getValueType().getScalarType() == MVT::f16;
11195 assert((!IsF16 || Subtarget->hasF16FP8ConversionInsts()) &&
11196 "f16 -> fp8/bf8 conversion requires F16FP8ConversionInsts");
11197 assert((!ResVT.isVector() || ResVT == MVT::v2i8) &&
11198 "only the v2i8 vector result is custom lowered");
11199
11200 if (IsF16) {
11201 unsigned Opc =
11202 IsBF8 ? AMDGPUISD::CVT_PK_BF8_F16 : AMDGPUISD::CVT_PK_FP8_F16;
11203 SDValue Bytes = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i16, Operand: Src);
11204 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResVT, Operand: Bytes);
11205 }
11206
11207 unsigned Opc = IsBF8 ? AMDGPUISD::CVT_PK_BF8_F32
11208 : IsE5M3 ? AMDGPUISD::CVT_PK_FP8_F32_E5M3
11209 : AMDGPUISD::CVT_PK_FP8_F32;
11210 SDValue PoisonI32 = DAG.getPOISON(VT: MVT::i32);
11211 SDValue WordSel = DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1);
11212
11213 if (!ResVT.isVector()) {
11214 // Convert one lane, the second is unused. Feed it the same source so the
11215 // instruction does not read an undefined register.
11216 SDValue Packed =
11217 DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Src, N2: Src, N3: PoisonI32, N4: WordSel);
11218 return DAG.getAnyExtOrTrunc(Op: Packed, DL: SL, VT: ResVT);
11219 }
11220
11221 SDValue A = DAG.getExtractVectorElt(DL: SL, VT: MVT::f32, Vec: Src, Idx: 0);
11222 SDValue B = DAG.getExtractVectorElt(DL: SL, VT: MVT::f32, Vec: Src, Idx: 1);
11223 SDValue Packed = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: A, N2: B, N3: PoisonI32, N4: WordSel);
11224 SDValue Bytes = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Packed);
11225 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResVT, Operand: Bytes);
11226}
11227
11228SDValue
11229SITargetLowering::LowerCONVERT_TO_ARBITRARY_FP(SDValue Op,
11230 SelectionDAG &DAG) const {
11231 // The OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 map to HW conversions
11232 // on subtargets that support them. Everything else uses generic expansion.
11233 APFloatBase::Semantics Sem =
11234 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(i: 1));
11235 const bool IsFP8 = Sem == APFloatBase::S_Float8E4M3FN;
11236 const bool IsBF8 = Sem == APFloatBase::S_Float8E5M2;
11237 const bool IsE5M3 = Sem == APFloatBase::S_Float8E5M3FNU;
11238 const bool HasE5M3ConversionInsts =
11239 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11240 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11241 if (!IsSupported)
11242 return SDValue();
11243
11244 // The HW conversions only support nearest-even. The OCP conversions do not
11245 // saturate. The unsigned E5M3 conversion always clamps out-of-range inputs,
11246 // which also refines the non-saturating form where those inputs are poison.
11247 if (static_cast<RoundingMode>(Op.getConstantOperandVal(i: 2)) !=
11248 RoundingMode::NearestTiesToEven)
11249 return SDValue();
11250 if (!IsE5M3 && Op.getConstantOperandVal(i: 3) != 0)
11251 return SDValue();
11252
11253 EVT SrcEltVT = Op.getOperand(i: 0).getValueType().getScalarType();
11254 // The f32 form is built here rather than by a tablegen pattern because the
11255 // HW result is i32 while the node result is i16 after the i8 promotion.
11256 if (SrcEltVT == MVT::f32)
11257 return lowerToFP8(Op, IsBF8, IsE5M3, DAG);
11258 if (!IsE5M3 && SrcEltVT == MVT::f16 &&
11259 Subtarget->hasF16FP8ConversionInsts()) {
11260 // A scalar conversion is selected from the generic node by tablegen, only
11261 // the illegal v2i8 result type needs lowering here.
11262 if (!Op.getValueType().isVector())
11263 return Op;
11264 return lowerToFP8(Op, IsBF8, IsE5M3: false, DAG);
11265 }
11266 return SDValue();
11267}
11268
11269SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11270 SelectionDAG &DAG) const {
11271 MachineFunction &MF = DAG.getMachineFunction();
11272 auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
11273
11274 EVT VT = Op.getValueType();
11275 SDLoc DL(Op);
11276 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
11277
11278 // TODO: Should this propagate fast-math-flags?
11279
11280 switch (IntrinsicID) {
11281 case Intrinsic::amdgcn_wave_reduce_min:
11282 case Intrinsic::amdgcn_wave_reduce_umin:
11283 case Intrinsic::amdgcn_wave_reduce_fmin:
11284 case Intrinsic::amdgcn_wave_reduce_max:
11285 case Intrinsic::amdgcn_wave_reduce_umax:
11286 case Intrinsic::amdgcn_wave_reduce_fmax:
11287 case Intrinsic::amdgcn_wave_reduce_add:
11288 case Intrinsic::amdgcn_wave_reduce_fadd:
11289 case Intrinsic::amdgcn_wave_reduce_sub:
11290 case Intrinsic::amdgcn_wave_reduce_fsub:
11291 case Intrinsic::amdgcn_wave_reduce_and:
11292 case Intrinsic::amdgcn_wave_reduce_or:
11293 case Intrinsic::amdgcn_wave_reduce_xor: {
11294 EVT SrcVT = Op.getOperand(i: 1).getValueType();
11295 if (SrcVT.getFixedSizeInBits() == 16) {
11296 bool IsFPOp = SrcVT.isFloatingPoint();
11297 bool NeedsSignExt = IntrinsicID == Intrinsic::amdgcn_wave_reduce_min ||
11298 IntrinsicID == Intrinsic::amdgcn_wave_reduce_max ||
11299 IntrinsicID == Intrinsic::amdgcn_wave_reduce_add ||
11300 IntrinsicID == Intrinsic::amdgcn_wave_reduce_sub;
11301 unsigned ExtOpc = IsFPOp ? ISD::FP_EXTEND
11302 : NeedsSignExt ? ISD::SIGN_EXTEND
11303 : ISD::ZERO_EXTEND;
11304 auto SrcType = IsFPOp ? MVT::f16 : MVT::i16;
11305 auto ExtType = IsFPOp ? MVT::f32 : MVT::i32;
11306 SDValue ExtendedSrc = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtType, Operand: Op.getOperand(i: 1));
11307 SDValue Strategy = Op.getOperand(i: 2);
11308 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ExtType,
11309 N1: Op.getOperand(i: 0), N2: ExtendedSrc, N3: Strategy);
11310 if (IsFPOp)
11311 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: SrcType, N1: Result,
11312 N2: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32));
11313 else
11314 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcType, Operand: Result);
11315 }
11316 return SDValue();
11317 }
11318 case Intrinsic::amdgcn_implicit_buffer_ptr: {
11319 if (getSubtarget()->isAmdHsaOrMesa(F: MF.getFunction()))
11320 return emitNonHSAIntrinsicError(DAG, DL, VT);
11321 return getPreloadedValue(DAG, MFI: *MFI, VT,
11322 PVID: AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
11323 }
11324 case Intrinsic::amdgcn_dispatch_ptr:
11325 case Intrinsic::amdgcn_queue_ptr: {
11326 if (!Subtarget->isAmdHsaOrMesa(F: MF.getFunction())) {
11327 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11328 MF.getFunction(), "unsupported hsa intrinsic without hsa target",
11329 DL.getDebugLoc()));
11330 return DAG.getPOISON(VT);
11331 }
11332
11333 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
11334 ? AMDGPUFunctionArgInfo::DISPATCH_PTR
11335 : AMDGPUFunctionArgInfo::QUEUE_PTR;
11336 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: RegID);
11337 }
11338 case Intrinsic::amdgcn_implicitarg_ptr: {
11339 if (MFI->isEntryFunction())
11340 return getImplicitArgPtr(DAG, SL: DL);
11341 return getPreloadedValue(DAG, MFI: *MFI, VT,
11342 PVID: AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
11343 }
11344 case Intrinsic::amdgcn_kernarg_segment_ptr: {
11345 if (!AMDGPU::isKernel(F: MF.getFunction())) {
11346 // This only makes sense to call in a kernel, so just lower to null.
11347 return DAG.getConstant(Val: 0, DL, VT);
11348 }
11349
11350 return getPreloadedValue(DAG, MFI: *MFI, VT,
11351 PVID: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
11352 }
11353 case Intrinsic::amdgcn_dispatch_id: {
11354 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: AMDGPUFunctionArgInfo::DISPATCH_ID);
11355 }
11356 case Intrinsic::amdgcn_rcp:
11357 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT, Operand: Op.getOperand(i: 1));
11358 case Intrinsic::amdgcn_rsq:
11359 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11360 case Intrinsic::amdgcn_rsq_legacy:
11361 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11362 return emitRemovedIntrinsicError(DAG, DL, VT);
11363 return SDValue();
11364 case Intrinsic::amdgcn_rcp_legacy:
11365 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11366 return emitRemovedIntrinsicError(DAG, DL, VT);
11367 return DAG.getNode(Opcode: AMDGPUISD::RCP_LEGACY, DL, VT, Operand: Op.getOperand(i: 1));
11368 case Intrinsic::amdgcn_fma_legacy:
11369 case Intrinsic::amdgcn_sudot4:
11370 case Intrinsic::amdgcn_sudot8:
11371 case Intrinsic::amdgcn_tanh:
11372 return SDValue();
11373 case Intrinsic::amdgcn_rsq_clamp: {
11374 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11375 return DAG.getNode(Opcode: AMDGPUISD::RSQ_CLAMP, DL, VT, Operand: Op.getOperand(i: 1));
11376
11377 Type *Type = VT.getTypeForEVT(Context&: *DAG.getContext());
11378 APFloat Max = APFloat::getLargest(Sem: Type->getFltSemantics());
11379 APFloat Min = APFloat::getLargest(Sem: Type->getFltSemantics(), Negative: true);
11380
11381 SDValue Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11382 SDValue Tmp =
11383 DAG.getNode(Opcode: ISD::FMINNUM, DL, VT, N1: Rsq, N2: DAG.getConstantFP(Val: Max, DL, VT));
11384 return DAG.getNode(Opcode: ISD::FMAXNUM, DL, VT, N1: Tmp,
11385 N2: DAG.getConstantFP(Val: Min, DL, VT));
11386 }
11387 case Intrinsic::r600_read_ngroups_x:
11388 if (Subtarget->isAmdHsaOS())
11389 return emitNonHSAIntrinsicError(DAG, DL, VT);
11390
11391 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11392 Offset: SI::KernelInputOffsets::NGROUPS_X, Alignment: Align(4),
11393 Signed: false);
11394 case Intrinsic::r600_read_ngroups_y:
11395 if (Subtarget->isAmdHsaOS())
11396 return emitNonHSAIntrinsicError(DAG, DL, VT);
11397
11398 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11399 Offset: SI::KernelInputOffsets::NGROUPS_Y, Alignment: Align(4),
11400 Signed: false);
11401 case Intrinsic::r600_read_ngroups_z:
11402 if (Subtarget->isAmdHsaOS())
11403 return emitNonHSAIntrinsicError(DAG, DL, VT);
11404
11405 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11406 Offset: SI::KernelInputOffsets::NGROUPS_Z, Alignment: Align(4),
11407 Signed: false);
11408 case Intrinsic::r600_read_local_size_x:
11409 if (Subtarget->isAmdHsaOS())
11410 return emitNonHSAIntrinsicError(DAG, DL, VT);
11411
11412 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11413 Offset: SI::KernelInputOffsets::LOCAL_SIZE_X);
11414 case Intrinsic::r600_read_local_size_y:
11415 if (Subtarget->isAmdHsaOS())
11416 return emitNonHSAIntrinsicError(DAG, DL, VT);
11417
11418 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11419 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Y);
11420 case Intrinsic::r600_read_local_size_z:
11421 if (Subtarget->isAmdHsaOS())
11422 return emitNonHSAIntrinsicError(DAG, DL, VT);
11423
11424 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11425 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Z);
11426 case Intrinsic::amdgcn_workgroup_id_x:
11427 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11428 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
11429 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
11430 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
11431 case Intrinsic::amdgcn_workgroup_id_y:
11432 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11433 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
11434 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
11435 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
11436 case Intrinsic::amdgcn_workgroup_id_z:
11437 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11438 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
11439 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
11440 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
11441 case Intrinsic::amdgcn_cluster_id_x:
11442 return Subtarget->hasClusters()
11443 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11444 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_X)
11445 : DAG.getPOISON(VT);
11446 case Intrinsic::amdgcn_cluster_id_y:
11447 return Subtarget->hasClusters()
11448 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11449 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y)
11450 : DAG.getPOISON(VT);
11451 case Intrinsic::amdgcn_cluster_id_z:
11452 return Subtarget->hasClusters()
11453 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11454 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z)
11455 : DAG.getPOISON(VT);
11456 case Intrinsic::amdgcn_cluster_workgroup_id_x:
11457 return Subtarget->hasClusters()
11458 ? getPreloadedValue(
11459 DAG, MFI: *MFI, VT,
11460 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X)
11461 : DAG.getPOISON(VT);
11462 case Intrinsic::amdgcn_cluster_workgroup_id_y:
11463 return Subtarget->hasClusters()
11464 ? getPreloadedValue(
11465 DAG, MFI: *MFI, VT,
11466 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y)
11467 : DAG.getPOISON(VT);
11468 case Intrinsic::amdgcn_cluster_workgroup_id_z:
11469 return Subtarget->hasClusters()
11470 ? getPreloadedValue(
11471 DAG, MFI: *MFI, VT,
11472 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z)
11473 : DAG.getPOISON(VT);
11474 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
11475 return Subtarget->hasClusters()
11476 ? lowerConstHwRegRead(DAG, Op, HwReg: AMDGPU::Hwreg::ID_IB_STS2, LowBit: 21, Width: 4)
11477 : SDValue();
11478 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
11479 return Subtarget->hasClusters()
11480 ? getPreloadedValue(
11481 DAG, MFI: *MFI, VT,
11482 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X)
11483 : DAG.getPOISON(VT);
11484 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
11485 return Subtarget->hasClusters()
11486 ? getPreloadedValue(
11487 DAG, MFI: *MFI, VT,
11488 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y)
11489 : DAG.getPOISON(VT);
11490 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
11491 return Subtarget->hasClusters()
11492 ? getPreloadedValue(
11493 DAG, MFI: *MFI, VT,
11494 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z)
11495 : DAG.getPOISON(VT);
11496 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
11497 return Subtarget->hasClusters()
11498 ? getPreloadedValue(
11499 DAG, MFI: *MFI, VT,
11500 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID)
11501 : DAG.getPOISON(VT);
11502 case Intrinsic::amdgcn_wave_id:
11503 return lowerWaveID(DAG, Op);
11504 case Intrinsic::amdgcn_lds_kernel_id: {
11505 if (MFI->isEntryFunction())
11506 return getLDSKernelId(DAG, SL: DL);
11507 return getPreloadedValue(DAG, MFI: *MFI, VT,
11508 PVID: AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
11509 }
11510 case Intrinsic::amdgcn_workitem_id_x:
11511 return lowerWorkitemID(DAG, Op, Dim: 0, Arg: MFI->getArgInfo().WorkItemIDX);
11512 case Intrinsic::amdgcn_workitem_id_y:
11513 return lowerWorkitemID(DAG, Op, Dim: 1, Arg: MFI->getArgInfo().WorkItemIDY);
11514 case Intrinsic::amdgcn_workitem_id_z:
11515 return lowerWorkitemID(DAG, Op, Dim: 2, Arg: MFI->getArgInfo().WorkItemIDZ);
11516 case Intrinsic::amdgcn_wavefrontsize:
11517 return DAG.getConstant(Val: MF.getSubtarget<GCNSubtarget>().getWavefrontSize(),
11518 DL: SDLoc(Op), VT: MVT::i32);
11519 case Intrinsic::amdgcn_s_buffer_load: {
11520 unsigned CPol = Op.getConstantOperandVal(i: 3);
11521 // s_buffer_load, because of how it's optimized, can't be volatile
11522 // so reject ones with the volatile bit set.
11523 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11524 ? AMDGPU::CPol::ALL
11525 : AMDGPU::CPol::ALL_pregfx12))
11526 return Op;
11527 return lowerSBuffer(VT, MemVT: VT, DL, Chain: DAG.getEntryNode(), Rsrc: Op.getOperand(i: 1),
11528 Offset: Op.getOperand(i: 2), CachePolicy: Op.getOperand(i: 3), DAG);
11529 }
11530 case Intrinsic::amdgcn_fdiv_fast:
11531 return lowerFDIV_FAST(Op, DAG);
11532 case Intrinsic::amdgcn_sin:
11533 return DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL, VT, Operand: Op.getOperand(i: 1));
11534
11535 case Intrinsic::amdgcn_cos:
11536 return DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL, VT, Operand: Op.getOperand(i: 1));
11537
11538 case Intrinsic::amdgcn_mul_u24:
11539 return DAG.getNode(Opcode: AMDGPUISD::MUL_U24, DL, VT, N1: Op.getOperand(i: 1),
11540 N2: Op.getOperand(i: 2));
11541 case Intrinsic::amdgcn_mul_i24:
11542 return DAG.getNode(Opcode: AMDGPUISD::MUL_I24, DL, VT, N1: Op.getOperand(i: 1),
11543 N2: Op.getOperand(i: 2));
11544
11545 case Intrinsic::amdgcn_log_clamp: {
11546 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11547 return SDValue();
11548
11549 return emitRemovedIntrinsicError(DAG, DL, VT);
11550 }
11551 case Intrinsic::amdgcn_fract:
11552 return DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: Op.getOperand(i: 1));
11553
11554 case Intrinsic::amdgcn_class: {
11555 SDValue Src = Op.getOperand(i: 1);
11556 EVT SrcVT = Src.getValueType();
11557 bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 ||
11558 (SrcVT == MVT::f16 && Subtarget->has16BitInsts());
11559 if (!IsLegal) {
11560 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11561 DAG.getMachineFunction().getFunction(),
11562 "llvm.amdgcn.class only supports f16, f32, and f64",
11563 DL.getDebugLoc()));
11564 return DAG.getPOISON(VT);
11565 }
11566 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT, N1: Src, N2: Op.getOperand(i: 2));
11567 }
11568 case Intrinsic::amdgcn_div_fmas:
11569 return DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL, VT, N1: Op.getOperand(i: 1),
11570 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11571
11572 case Intrinsic::amdgcn_div_fixup:
11573 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL, VT, N1: Op.getOperand(i: 1),
11574 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11575
11576 case Intrinsic::amdgcn_div_scale: {
11577 const ConstantSDNode *Param = cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11578
11579 // Translate to the operands expected by the machine instruction. The
11580 // first parameter must be the same as the first instruction.
11581 SDValue Numerator = Op.getOperand(i: 1);
11582 SDValue Denominator = Op.getOperand(i: 2);
11583
11584 // Note this order is opposite of the machine instruction's operations,
11585 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The
11586 // intrinsic has the numerator as the first operand to match a normal
11587 // division operation.
11588
11589 SDValue Src0 = Param->isAllOnes() ? Numerator : Denominator;
11590
11591 return DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL, VTList: Op->getVTList(), N1: Src0,
11592 N2: Denominator, N3: Numerator);
11593 }
11594 case Intrinsic::amdgcn_ballot:
11595 return lowerBALLOTIntrinsic(TLI: *this, N: Op.getNode(), DAG);
11596 case Intrinsic::amdgcn_fmed3:
11597 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL, VT, N1: Op.getOperand(i: 1),
11598 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), Flags: Op->getFlags());
11599 case Intrinsic::amdgcn_fdot2:
11600 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL, VT, N1: Op.getOperand(i: 1),
11601 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11602 case Intrinsic::amdgcn_fmul_legacy:
11603 return DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL, VT, N1: Op.getOperand(i: 1),
11604 N2: Op.getOperand(i: 2));
11605 case Intrinsic::amdgcn_sbfe:
11606 case Intrinsic::amdgcn_ubfe:
11607 return lowerBFEIntrinsic(Op, DAG, IntrinsicID);
11608 case Intrinsic::amdgcn_cvt_pkrtz:
11609 case Intrinsic::amdgcn_cvt_pknorm_i16:
11610 case Intrinsic::amdgcn_cvt_pknorm_u16:
11611 case Intrinsic::amdgcn_cvt_pk_i16:
11612 case Intrinsic::amdgcn_cvt_pk_u16: {
11613 // FIXME: Stop adding cast if v2f16/v2i16 are legal.
11614 EVT VT = Op.getValueType();
11615 unsigned Opcode;
11616
11617 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
11618 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32;
11619 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
11620 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
11621 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
11622 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
11623 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
11624 Opcode = AMDGPUISD::CVT_PK_I16_I32;
11625 else
11626 Opcode = AMDGPUISD::CVT_PK_U16_U32;
11627
11628 if (isTypeLegal(VT))
11629 return DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11630
11631 SDValue Node =
11632 DAG.getNode(Opcode, DL, VT: MVT::i32, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11633 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Node);
11634 }
11635 case Intrinsic::amdgcn_fmad_ftz:
11636 return DAG.getNode(Opcode: AMDGPUISD::FMAD_FTZ, DL, VT, N1: Op.getOperand(i: 1),
11637 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11638
11639 case Intrinsic::amdgcn_if_break:
11640 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_IF_BREAK, dl: DL, VT,
11641 Op1: Op->getOperand(Num: 1), Op2: Op->getOperand(Num: 2)),
11642 0);
11643
11644 case Intrinsic::amdgcn_groupstaticsize: {
11645 Triple::OSType OS = getTargetMachine().getTargetTriple().getOS();
11646 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL)
11647 return Op;
11648
11649 const Module *M = MF.getFunction().getParent();
11650 const GlobalValue *GV =
11651 Intrinsic::getDeclarationIfExists(M, id: Intrinsic::amdgcn_groupstaticsize);
11652 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: 0,
11653 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11654 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11655 }
11656 case Intrinsic::amdgcn_is_shared:
11657 case Intrinsic::amdgcn_is_private: {
11658 SDLoc SL(Op);
11659 SDValue SrcVec =
11660 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
11661 SDValue SrcHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: SrcVec,
11662 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
11663
11664 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
11665 ? AMDGPUAS::LOCAL_ADDRESS
11666 : AMDGPUAS::PRIVATE_ADDRESS;
11667 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
11668 Subtarget->hasGloballyAddressableScratch()) {
11669 SDValue FlatScratchBaseHi(
11670 DAG.getMachineNode(
11671 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
11672 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, VT: MVT::i32)),
11673 0);
11674 // Test bits 63..58 against the aperture address.
11675 return DAG.getSetCC(
11676 DL: SL, VT: MVT::i1,
11677 LHS: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: SrcHi, N2: FlatScratchBaseHi),
11678 RHS: DAG.getConstant(Val: 1u << 26, DL: SL, VT: MVT::i32), Cond: ISD::SETULT);
11679 }
11680
11681 SDValue Aperture = getSegmentAperture(AS, DL: SL, DAG);
11682 return DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SrcHi, RHS: Aperture, Cond: ISD::SETEQ);
11683 }
11684 case Intrinsic::amdgcn_perm:
11685 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op.getOperand(i: 1),
11686 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11687 case Intrinsic::amdgcn_reloc_constant: {
11688 Module *M = MF.getFunction().getParent();
11689 const MDNode *Metadata = cast<MDNodeSDNode>(Val: Op.getOperand(i: 1))->getMD();
11690 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
11691 auto *RelocSymbol = cast<GlobalVariable>(
11692 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
11693 SDValue GA = DAG.getTargetGlobalAddress(GV: RelocSymbol, DL, VT: MVT::i32, offset: 0,
11694 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11695 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11696 }
11697 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
11698 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
11699 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
11700 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
11701 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
11702 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
11703 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
11704 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
11705 if (Op.getOperand(i: 4).getValueType() == MVT::i32)
11706 return SDValue();
11707
11708 SDLoc SL(Op);
11709 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i32);
11710 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11711 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2),
11712 N4: Op.getOperand(i: 3), N5: IndexKeyi32);
11713 }
11714 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
11715 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
11716 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
11717 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
11718 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
11719 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
11720 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
11721 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
11722 if (Op.getOperand(i: 4).getValueType() == MVT::i64)
11723 return SDValue();
11724
11725 SDLoc SL(Op);
11726 auto IndexKeyi64 =
11727 Op.getOperand(i: 4).getValueType() == MVT::v2i32
11728 ? DAG.getBitcast(VT: MVT::i64, V: Op.getOperand(i: 4))
11729 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i64);
11730 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11731 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11732 Op.getOperand(i: 3), IndexKeyi64, Op.getOperand(i: 5),
11733 Op.getOperand(i: 6)});
11734 }
11735 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
11736 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
11737 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
11738 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
11739 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
11740 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8: {
11741 EVT IndexKeyTy = IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
11742 ? MVT::i64
11743 : MVT::i32;
11744 if (Op.getOperand(i: 6).getValueType() == IndexKeyTy)
11745 return SDValue();
11746
11747 SDLoc SL(Op);
11748 auto IndexKey =
11749 Op.getOperand(i: 6).getValueType().isVector()
11750 ? DAG.getBitcast(VT: IndexKeyTy, V: Op.getOperand(i: 6))
11751 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: IndexKeyTy);
11752 SmallVector<SDValue> Args{
11753 Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11754 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11755 IndexKey, Op.getOperand(i: 7), Op.getOperand(i: 8)};
11756 if (IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8)
11757 Args.push_back(Elt: Op.getOperand(i: 9));
11758 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(), Ops: Args);
11759 }
11760 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
11761 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
11762 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
11763 if (Op.getOperand(i: 6).getValueType() == MVT::i32)
11764 return SDValue();
11765
11766 SDLoc SL(Op);
11767 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: MVT::i32);
11768 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11769 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11770 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11771 IndexKeyi32, Op.getOperand(i: 7)});
11772 }
11773 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
11774 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
11775 unsigned AFmt = (unsigned)Op.getConstantOperandVal(i: 1);
11776 unsigned BFmt = (unsigned)Op.getConstantOperandVal(i: 3);
11777 unsigned AScaleFmt = (unsigned)Op.getConstantOperandVal(i: 8);
11778 unsigned BScaleFmt = (unsigned)Op.getConstantOperandVal(i: 11);
11779 if (!AMDGPU::isValidWMMAScaleFmtCombination(AFmt, AScale: AScaleFmt, BFmt,
11780 BScale: BScaleFmt)) {
11781 DAG.getMachineFunction().getFunction().getContext().emitError(
11782 ErrorStr: "invalid matrix and scale format combination in wmma call");
11783 Op->print(OS&: errs());
11784 errs() << '\n';
11785 }
11786 return SDValue();
11787 }
11788 case Intrinsic::amdgcn_readlane:
11789 case Intrinsic::amdgcn_readfirstlane:
11790 case Intrinsic::amdgcn_writelane:
11791 case Intrinsic::amdgcn_permlane16:
11792 case Intrinsic::amdgcn_permlanex16:
11793 case Intrinsic::amdgcn_permlane64:
11794 case Intrinsic::amdgcn_set_inactive:
11795 case Intrinsic::amdgcn_set_inactive_chain_arg:
11796 case Intrinsic::amdgcn_mov_dpp8:
11797 case Intrinsic::amdgcn_update_dpp:
11798 case Intrinsic::amdgcn_permlane_bcast:
11799 case Intrinsic::amdgcn_permlane_up:
11800 case Intrinsic::amdgcn_permlane_down:
11801 case Intrinsic::amdgcn_permlane_xor:
11802 return lowerLaneOp(TLI: *this, N: Op.getNode(), DAG);
11803 case Intrinsic::amdgcn_dead: {
11804 SmallVector<SDValue, 8> Poisons;
11805 for (const EVT ValTy : Op.getNode()->values())
11806 Poisons.push_back(Elt: DAG.getPOISON(VT: ValTy));
11807 return DAG.getMergeValues(Ops: Poisons, dl: SDLoc(Op));
11808 }
11809 case Intrinsic::amdgcn_wave_shuffle:
11810 return lowerWaveShuffle(TLI: *this, N: Op.getNode(), DAG);
11811 default:
11812 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
11813 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
11814 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: false);
11815
11816 return Op;
11817 }
11818}
11819
11820// On targets not supporting constant in soffset field, turn zero to
11821// SGPR_NULL to avoid generating an extra s_mov with zero.
11822static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG,
11823 const GCNSubtarget *Subtarget) {
11824 if (Subtarget->hasRestrictedSOffset() && isNullConstant(V: SOffset))
11825 return DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32);
11826 return SOffset;
11827}
11828
11829SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
11830 SelectionDAG &DAG,
11831 unsigned NewOpcode) const {
11832 SDLoc DL(Op);
11833
11834 SDValue VData = Op.getOperand(i: 2);
11835 if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
11836 SmallVector<EVT, 2> ResultTypes(Op->values());
11837 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
11838 Msg: "unsupported buffer atomic data type");
11839 }
11840 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11841 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11842 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11843 SDValue Ops[] = {
11844 Op.getOperand(i: 0), // Chain
11845 VData, // vdata
11846 Rsrc, // rsrc
11847 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11848 VOffset, // voffset
11849 SOffset, // soffset
11850 Offset, // offset
11851 Op.getOperand(i: 6), // cachepolicy
11852 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11853 };
11854
11855 auto *M = cast<MemSDNode>(Val&: Op);
11856
11857 EVT MemVT = VData.getValueType();
11858 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11859 MMO: M->getMemOperand());
11860}
11861
11862SDValue
11863SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
11864 unsigned NewOpcode) const {
11865 SDLoc DL(Op);
11866
11867 SDValue VData = Op.getOperand(i: 2);
11868 if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
11869 SmallVector<EVT, 2> ResultTypes(Op->values());
11870 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
11871 Msg: "unsupported buffer atomic data type");
11872 }
11873 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11874 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
11875 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
11876 SDValue Ops[] = {
11877 Op.getOperand(i: 0), // Chain
11878 VData, // vdata
11879 Rsrc, // rsrc
11880 Op.getOperand(i: 4), // vindex
11881 VOffset, // voffset
11882 SOffset, // soffset
11883 Offset, // offset
11884 Op.getOperand(i: 7), // cachepolicy
11885 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11886 };
11887
11888 auto *M = cast<MemSDNode>(Val&: Op);
11889
11890 EVT MemVT = VData.getValueType();
11891 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11892 MMO: M->getMemOperand());
11893}
11894
11895static void initializeM0ToZeroForClusterLoad(SDValue Op, SelectionDAG &DAG,
11896 SDLoc DL) {
11897 SDNode *N = Op.getNode();
11898 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
11899 unsigned NumOperands = N->getNumOperands();
11900 if (N->getOperand(Num: NumOperands - 1) == Zero)
11901 return;
11902 SmallVector<SDValue, 7> Ops(N->ops());
11903 Ops[NumOperands - 1] = Zero; // M0 = 0
11904 DAG.UpdateNodeOperands(N, Ops);
11905}
11906
11907SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
11908 SelectionDAG &DAG) const {
11909 unsigned IntrID = Op.getConstantOperandVal(i: 1);
11910 SDLoc DL(Op);
11911
11912 switch (IntrID) {
11913 case Intrinsic::amdgcn_cluster_load_b32:
11914 case Intrinsic::amdgcn_cluster_load_b64:
11915 case Intrinsic::amdgcn_cluster_load_b128: {
11916 if (Subtarget->hasGFX1250_STRICT())
11917 initializeM0ToZeroForClusterLoad(Op, DAG, DL);
11918 return SDValue();
11919 }
11920 case Intrinsic::amdgcn_ds_ordered_add:
11921 case Intrinsic::amdgcn_ds_ordered_swap: {
11922 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11923 SDValue Chain = M->getOperand(Num: 0);
11924 SDValue M0 = M->getOperand(Num: 2);
11925 SDValue Value = M->getOperand(Num: 3);
11926 unsigned IndexOperand = M->getConstantOperandVal(Num: 7);
11927 unsigned WaveRelease = M->getConstantOperandVal(Num: 8);
11928 unsigned WaveDone = M->getConstantOperandVal(Num: 9);
11929
11930 unsigned OrderedCountIndex = IndexOperand & 0x3f;
11931 IndexOperand &= ~0x3f;
11932 unsigned CountDw = 0;
11933
11934 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) {
11935 CountDw = (IndexOperand >> 24) & 0xf;
11936 IndexOperand &= ~(0xf << 24);
11937
11938 if (CountDw < 1 || CountDw > 4) {
11939 const Function &Fn = DAG.getMachineFunction().getFunction();
11940 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11941 Fn, "ds_ordered_count: dword count must be between 1 and 4",
11942 DL.getDebugLoc()));
11943 CountDw = 1;
11944 }
11945 }
11946
11947 if (IndexOperand) {
11948 const Function &Fn = DAG.getMachineFunction().getFunction();
11949 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11950 Fn, "ds_ordered_count: bad index operand", DL.getDebugLoc()));
11951 }
11952
11953 if (WaveDone && !WaveRelease) {
11954 // TODO: Move this to IR verifier
11955 const Function &Fn = DAG.getMachineFunction().getFunction();
11956 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11957 Fn, "ds_ordered_count: wave_done requires wave_release",
11958 DL.getDebugLoc()));
11959 }
11960
11961 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
11962 unsigned ShaderType =
11963 SIInstrInfo::getDSShaderTypeValue(MF: DAG.getMachineFunction());
11964 unsigned Offset0 = OrderedCountIndex << 2;
11965 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
11966
11967 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
11968 Offset1 |= (CountDw - 1) << 6;
11969
11970 if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX11)
11971 Offset1 |= ShaderType << 2;
11972
11973 unsigned Offset = Offset0 | (Offset1 << 8);
11974
11975 SDValue Ops[] = {
11976 Chain, Value, DAG.getTargetConstant(Val: Offset, DL, VT: MVT::i16),
11977 copyToM0(DAG, Chain, DL, V: M0).getValue(R: 1), // Glue
11978 };
11979 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::DS_ORDERED_COUNT, dl: DL,
11980 VTList: M->getVTList(), Ops, MemVT: M->getMemoryVT(),
11981 MMO: M->getMemOperand());
11982 }
11983 case Intrinsic::amdgcn_ptr_s_buffer_load: {
11984 unsigned CPol = Op.getConstantOperandVal(i: 4);
11985 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11986 ? AMDGPU::CPol::ALL
11987 : AMDGPU::CPol::ALL_pregfx12))
11988 return Op;
11989
11990 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11991 return lowerSBuffer(
11992 VT: Op.getValueType(), MemVT: M->getMemoryVT(), DL, Chain: Op.getOperand(i: 0),
11993 Rsrc: bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG), Offset: Op.getOperand(i: 3),
11994 CachePolicy: Op.getOperand(i: 4), DAG, MMO: M->getMemOperand());
11995 }
11996 case Intrinsic::amdgcn_raw_buffer_load:
11997 case Intrinsic::amdgcn_raw_ptr_buffer_load:
11998 case Intrinsic::amdgcn_raw_atomic_buffer_load:
11999 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
12000 case Intrinsic::amdgcn_raw_buffer_load_format:
12001 case Intrinsic::amdgcn_raw_ptr_buffer_load_format: {
12002 const bool IsFormat =
12003 IntrID == Intrinsic::amdgcn_raw_buffer_load_format ||
12004 IntrID == Intrinsic::amdgcn_raw_ptr_buffer_load_format;
12005
12006 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12007 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
12008 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
12009 SDValue Ops[] = {
12010 Op.getOperand(i: 0), // Chain
12011 Rsrc, // rsrc
12012 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12013 VOffset, // voffset
12014 SOffset, // soffset
12015 Offset, // offset
12016 Op.getOperand(i: 5), // cachepolicy, swizzled buffer
12017 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12018 };
12019
12020 auto *M = cast<MemSDNode>(Val&: Op);
12021 return lowerIntrinsicLoad(M, IsFormat, DAG, Ops);
12022 }
12023 case Intrinsic::amdgcn_struct_buffer_load:
12024 case Intrinsic::amdgcn_struct_ptr_buffer_load:
12025 case Intrinsic::amdgcn_struct_buffer_load_format:
12026 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
12027 case Intrinsic::amdgcn_struct_atomic_buffer_load:
12028 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
12029 const bool IsFormat =
12030 IntrID == Intrinsic::amdgcn_struct_buffer_load_format ||
12031 IntrID == Intrinsic::amdgcn_struct_ptr_buffer_load_format;
12032
12033 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12034 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12035 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12036 SDValue Ops[] = {
12037 Op.getOperand(i: 0), // Chain
12038 Rsrc, // rsrc
12039 Op.getOperand(i: 3), // vindex
12040 VOffset, // voffset
12041 SOffset, // soffset
12042 Offset, // offset
12043 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
12044 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12045 };
12046
12047 return lowerIntrinsicLoad(M: cast<MemSDNode>(Val&: Op), IsFormat, DAG, Ops);
12048 }
12049 case Intrinsic::amdgcn_raw_tbuffer_load:
12050 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
12051 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12052 EVT LoadVT = Op.getValueType();
12053 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12054 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
12055 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
12056
12057 SDValue Ops[] = {
12058 Op.getOperand(i: 0), // Chain
12059 Rsrc, // rsrc
12060 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12061 VOffset, // voffset
12062 SOffset, // soffset
12063 Offset, // offset
12064 Op.getOperand(i: 5), // format
12065 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
12066 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12067 };
12068
12069 if (LoadVT.getScalarSizeInBits() == 16)
12070 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
12071 Ops);
12072 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
12073 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
12074 DAG);
12075 }
12076 case Intrinsic::amdgcn_struct_tbuffer_load:
12077 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: {
12078 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12079 EVT LoadVT = Op.getValueType();
12080 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12081 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12082 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12083
12084 SDValue Ops[] = {
12085 Op.getOperand(i: 0), // Chain
12086 Rsrc, // rsrc
12087 Op.getOperand(i: 3), // vindex
12088 VOffset, // voffset
12089 SOffset, // soffset
12090 Offset, // offset
12091 Op.getOperand(i: 6), // format
12092 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12093 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12094 };
12095
12096 if (LoadVT.getScalarSizeInBits() == 16)
12097 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
12098 Ops);
12099 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
12100 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
12101 DAG);
12102 }
12103 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
12104 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
12105 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
12106 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
12107 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
12108 return lowerStructBufferAtomicIntrin(Op, DAG,
12109 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
12110 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
12111 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
12112 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
12113 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
12114 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
12115 return lowerStructBufferAtomicIntrin(Op, DAG,
12116 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
12117 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
12118 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
12119 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
12120 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
12121 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
12122 return lowerStructBufferAtomicIntrin(Op, DAG,
12123 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
12124 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
12125 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
12126 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
12127 case Intrinsic::amdgcn_raw_buffer_atomic_add:
12128 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
12129 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
12130 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
12131 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
12132 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
12133 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
12134 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
12135 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
12136 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
12137 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
12138 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
12139 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
12140 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
12141 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
12142 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
12143 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
12144 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
12145 case Intrinsic::amdgcn_raw_buffer_atomic_and:
12146 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
12147 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
12148 case Intrinsic::amdgcn_raw_buffer_atomic_or:
12149 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
12150 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
12151 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
12152 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
12153 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
12154 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
12155 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
12156 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
12157 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
12158 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
12159 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
12160 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
12161 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
12162 return lowerStructBufferAtomicIntrin(Op, DAG,
12163 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
12164 case Intrinsic::amdgcn_struct_buffer_atomic_add:
12165 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
12166 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
12167 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
12168 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
12169 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
12170 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
12171 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
12172 return lowerStructBufferAtomicIntrin(Op, DAG,
12173 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
12174 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
12175 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
12176 return lowerStructBufferAtomicIntrin(Op, DAG,
12177 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
12178 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
12179 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
12180 return lowerStructBufferAtomicIntrin(Op, DAG,
12181 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
12182 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
12183 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
12184 return lowerStructBufferAtomicIntrin(Op, DAG,
12185 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
12186 case Intrinsic::amdgcn_struct_buffer_atomic_and:
12187 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
12188 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
12189 case Intrinsic::amdgcn_struct_buffer_atomic_or:
12190 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
12191 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
12192 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
12193 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
12194 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
12195 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
12196 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
12197 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
12198 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
12199 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
12200 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
12201 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
12202 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
12203 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
12204 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
12205 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
12206 return lowerStructBufferAtomicIntrin(Op, DAG,
12207 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
12208 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
12209 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
12210 return lowerRawBufferAtomicIntrin(Op, DAG,
12211 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12212 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
12213 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
12214 return lowerStructBufferAtomicIntrin(Op, DAG,
12215 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12216 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
12217 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
12218 SDValue Src = Op.getOperand(i: 2);
12219 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12220 SmallVector<EVT, 2> ResultTypes(Op->values());
12221 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
12222 Msg: "unsupported buffer atomic data type");
12223 }
12224 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 4), DAG);
12225 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12226 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12227 SDValue Ops[] = {
12228 Op.getOperand(i: 0), // Chain
12229 Op.getOperand(i: 2), // src
12230 Op.getOperand(i: 3), // cmp
12231 Rsrc, // rsrc
12232 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12233 VOffset, // voffset
12234 SOffset, // soffset
12235 Offset, // offset
12236 Op.getOperand(i: 7), // cachepolicy
12237 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12238 };
12239 EVT VT = Op.getValueType();
12240 auto *M = cast<MemSDNode>(Val&: Op);
12241
12242 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
12243 VTList: Op->getVTList(), Ops, MemVT: VT,
12244 MMO: M->getMemOperand());
12245 }
12246 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
12247 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
12248 SDValue Src = Op.getOperand(i: 2);
12249 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12250 SmallVector<EVT, 2> ResultTypes(Op->values());
12251 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
12252 Msg: "unsupported buffer atomic data type");
12253 }
12254 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op->getOperand(Num: 4), DAG);
12255 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 6), DAG);
12256 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 7), DAG, Subtarget);
12257 SDValue Ops[] = {
12258 Op.getOperand(i: 0), // Chain
12259 Op.getOperand(i: 2), // src
12260 Op.getOperand(i: 3), // cmp
12261 Rsrc, // rsrc
12262 Op.getOperand(i: 5), // vindex
12263 VOffset, // voffset
12264 SOffset, // soffset
12265 Offset, // offset
12266 Op.getOperand(i: 8), // cachepolicy
12267 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12268 };
12269 EVT VT = Op.getValueType();
12270 auto *M = cast<MemSDNode>(Val&: Op);
12271
12272 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
12273 VTList: Op->getVTList(), Ops, MemVT: VT,
12274 MMO: M->getMemOperand());
12275 }
12276 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
12277 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
12278 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12279 SDValue NodePtr = M->getOperand(Num: 2);
12280 SDValue RayExtent = M->getOperand(Num: 3);
12281 SDValue InstanceMask = M->getOperand(Num: 4);
12282 SDValue RayOrigin = M->getOperand(Num: 5);
12283 SDValue RayDir = M->getOperand(Num: 6);
12284 SDValue Offsets = M->getOperand(Num: 7);
12285 SDValue TDescr = M->getOperand(Num: 8);
12286
12287 assert(NodePtr.getValueType() == MVT::i64);
12288 assert(RayDir.getValueType() == MVT::v3f32);
12289
12290 bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
12291 const unsigned NumVDataDwords = 10;
12292 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
12293 int Opcode = AMDGPU::getMIMGOpcode(
12294 BaseOpcode: IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
12295 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
12296 MIMGEncoding: AMDGPU::MIMGEncGfx12, VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12297 assert(Opcode != -1);
12298
12299 SmallVector<SDValue, 7> Ops;
12300 Ops.push_back(Elt: NodePtr);
12301 Ops.push_back(Elt: DAG.getBuildVector(
12302 VT: MVT::v2i32, DL,
12303 Ops: {DAG.getBitcast(VT: MVT::i32, V: RayExtent),
12304 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: InstanceMask)}));
12305 Ops.push_back(Elt: RayOrigin);
12306 Ops.push_back(Elt: RayDir);
12307 Ops.push_back(Elt: Offsets);
12308 Ops.push_back(Elt: TDescr);
12309 Ops.push_back(Elt: M->getChain());
12310
12311 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12312 MachineMemOperand *MemRef = M->getMemOperand();
12313 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12314 return SDValue(NewNode, 0);
12315 }
12316 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
12317 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12318 SDValue NodePtr = M->getOperand(Num: 2);
12319 SDValue RayExtent = M->getOperand(Num: 3);
12320 SDValue RayOrigin = M->getOperand(Num: 4);
12321 SDValue RayDir = M->getOperand(Num: 5);
12322 SDValue RayInvDir = M->getOperand(Num: 6);
12323 SDValue TDescr = M->getOperand(Num: 7);
12324
12325 assert(NodePtr.getValueType() == MVT::i32 ||
12326 NodePtr.getValueType() == MVT::i64);
12327 assert(RayDir.getValueType() == MVT::v3f16 ||
12328 RayDir.getValueType() == MVT::v3f32);
12329
12330 const bool IsGFX11 = AMDGPU::isGFX11(STI: *Subtarget);
12331 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
12332 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12333 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16;
12334 const bool Is64 = NodePtr.getValueType() == MVT::i64;
12335 const unsigned NumVDataDwords = 4;
12336 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
12337 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
12338 const bool UseNSA = (Subtarget->hasNSAEncoding() &&
12339 NumVAddrs <= Subtarget->getNSAMaxSize()) ||
12340 IsGFX12Plus;
12341 const unsigned BaseOpcodes[2][2] = {
12342 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
12343 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
12344 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
12345 int Opcode;
12346 if (UseNSA) {
12347 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
12348 MIMGEncoding: IsGFX12Plus ? AMDGPU::MIMGEncGfx12
12349 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
12350 : AMDGPU::MIMGEncGfx10NSA,
12351 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12352 } else {
12353 assert(!IsGFX12Plus);
12354 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
12355 MIMGEncoding: IsGFX11 ? AMDGPU::MIMGEncGfx11Default
12356 : AMDGPU::MIMGEncGfx10Default,
12357 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12358 }
12359 assert(Opcode != -1);
12360
12361 SmallVector<SDValue, 16> Ops;
12362
12363 auto packLanes = [&DAG, &Ops, &DL](SDValue Op, bool IsAligned) {
12364 SmallVector<SDValue, 3> Lanes;
12365 DAG.ExtractVectorElements(Op, Args&: Lanes, Start: 0, Count: 3);
12366 if (Lanes[0].getValueSizeInBits() == 32) {
12367 for (unsigned I = 0; I < 3; ++I)
12368 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: Lanes[I]));
12369 } else {
12370 if (IsAligned) {
12371 Ops.push_back(Elt: DAG.getBitcast(
12372 VT: MVT::i32,
12373 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[0], Lanes[1]})));
12374 Ops.push_back(Elt: Lanes[2]);
12375 } else {
12376 SDValue Elt0 = Ops.pop_back_val();
12377 Ops.push_back(Elt: DAG.getBitcast(
12378 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Elt0, Lanes[0]})));
12379 Ops.push_back(Elt: DAG.getBitcast(
12380 VT: MVT::i32,
12381 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[1], Lanes[2]})));
12382 }
12383 }
12384 };
12385
12386 if (UseNSA && IsGFX11Plus) {
12387 Ops.push_back(Elt: NodePtr);
12388 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
12389 Ops.push_back(Elt: RayOrigin);
12390 if (IsA16) {
12391 SmallVector<SDValue, 3> DirLanes, InvDirLanes, MergedLanes;
12392 DAG.ExtractVectorElements(Op: RayDir, Args&: DirLanes, Start: 0, Count: 3);
12393 DAG.ExtractVectorElements(Op: RayInvDir, Args&: InvDirLanes, Start: 0, Count: 3);
12394 for (unsigned I = 0; I < 3; ++I) {
12395 MergedLanes.push_back(Elt: DAG.getBitcast(
12396 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL,
12397 Ops: {DirLanes[I], InvDirLanes[I]})));
12398 }
12399 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v3i32, DL, Ops: MergedLanes));
12400 } else {
12401 Ops.push_back(Elt: RayDir);
12402 Ops.push_back(Elt: RayInvDir);
12403 }
12404 } else {
12405 if (Is64)
12406 DAG.ExtractVectorElements(Op: DAG.getBitcast(VT: MVT::v2i32, V: NodePtr), Args&: Ops, Start: 0,
12407 Count: 2);
12408 else
12409 Ops.push_back(Elt: NodePtr);
12410
12411 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
12412 packLanes(RayOrigin, true);
12413 packLanes(RayDir, true);
12414 packLanes(RayInvDir, false);
12415 }
12416
12417 if (!UseNSA) {
12418 // Build a single vector containing all the operands so far prepared.
12419 if (NumVAddrDwords > 12) {
12420 SDValue Undef = DAG.getPOISON(VT: MVT::i32);
12421 Ops.append(NumInputs: 16 - Ops.size(), Elt: Undef);
12422 }
12423 assert(Ops.size() >= 8 && Ops.size() <= 12);
12424 SDValue MergedOps =
12425 DAG.getBuildVector(VT: MVT::getVectorVT(VT: MVT::i32, NumElements: Ops.size()), DL, Ops);
12426 Ops.clear();
12427 Ops.push_back(Elt: MergedOps);
12428 }
12429
12430 Ops.push_back(Elt: TDescr);
12431 Ops.push_back(Elt: DAG.getTargetConstant(Val: IsA16, DL, VT: MVT::i1));
12432 Ops.push_back(Elt: M->getChain());
12433
12434 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12435 MachineMemOperand *MemRef = M->getMemOperand();
12436 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12437 return SDValue(NewNode, 0);
12438 }
12439 case Intrinsic::amdgcn_global_atomic_fmin_num:
12440 case Intrinsic::amdgcn_global_atomic_fmax_num:
12441 case Intrinsic::amdgcn_flat_atomic_fmin_num:
12442 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12443 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12444 SDValue Ops[] = {
12445 M->getOperand(Num: 0), // Chain
12446 M->getOperand(Num: 2), // Ptr
12447 M->getOperand(Num: 3) // Value
12448 };
12449 unsigned Opcode = 0;
12450 switch (IntrID) {
12451 case Intrinsic::amdgcn_global_atomic_fmin_num:
12452 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
12453 Opcode = ISD::ATOMIC_LOAD_FMIN;
12454 break;
12455 }
12456 case Intrinsic::amdgcn_global_atomic_fmax_num:
12457 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12458 Opcode = ISD::ATOMIC_LOAD_FMAX;
12459 break;
12460 }
12461 default:
12462 llvm_unreachable("unhandled atomic opcode");
12463 }
12464 return DAG.getAtomic(Opcode, dl: SDLoc(Op), MemVT: M->getMemoryVT(), VTList: M->getVTList(),
12465 Ops, MMO: M->getMemOperand());
12466 }
12467 case Intrinsic::amdgcn_s_alloc_vgpr: {
12468 SDValue NumVGPRs = Op.getOperand(i: 2);
12469 if (!NumVGPRs->isDivergent())
12470 return Op;
12471
12472 SDValue ReadFirstLaneID =
12473 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
12474 NumVGPRs = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
12475 N1: ReadFirstLaneID, N2: NumVGPRs);
12476
12477 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, VTList: Op->getVTList(),
12478 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: NumVGPRs);
12479 }
12480 case Intrinsic::amdgcn_s_get_barrier_state:
12481 case Intrinsic::amdgcn_s_get_named_barrier_state: {
12482 SDValue Chain = Op->getOperand(Num: 0);
12483 SmallVector<SDValue, 2> Ops;
12484 unsigned Opc;
12485
12486 if (isa<ConstantSDNode>(Val: Op->getOperand(Num: 2))) {
12487 uint64_t BarID = cast<ConstantSDNode>(Val: Op->getOperand(Num: 2))->getZExtValue();
12488 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
12489 BarID = BarID & 0x3F;
12490 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
12491 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
12492 Ops.push_back(Elt: K);
12493 Ops.push_back(Elt: Chain);
12494 } else {
12495 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
12496 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
12497 SDValue M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op->getOperand(Num: 2),
12498 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12499 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12500 } else
12501 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: Op->getOperand(Num: 2)).getValue(R: 0));
12502 }
12503
12504 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12505 return SDValue(NewMI, 0);
12506 }
12507 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
12508 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
12509 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
12510 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12511 SDValue Chain = Op->getOperand(Num: 0);
12512 SDValue Ptr = Op->getOperand(Num: 2);
12513 EVT VT = Op->getValueType(ResNo: 0);
12514 return DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl: DL, MemVT: MII->getMemoryVT(), VT,
12515 Chain, Ptr, MMO: MII->getMemOperand());
12516 }
12517 case Intrinsic::amdgcn_av_load_b128: {
12518 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12519 SDValue Chain = Op->getOperand(Num: 0);
12520 SDValue Ptr = Op->getOperand(Num: 2);
12521 EVT VT = Op->getValueType(ResNo: 0);
12522 // Lower to a regular ISD::LOAD. The MachineMemOperand carries Monotonic
12523 // ordering and syncscope so that SIMemoryLegalizer sets cache policy bits.
12524 // Address space filtering in the load_global/load_flat PatFrags selects
12525 // the correct GLOBAL vs FLAT instruction.
12526 return DAG.getLoad(VT, dl: DL, Chain, Ptr, MMO: MII->getMemOperand());
12527 }
12528 case Intrinsic::amdgcn_flat_load_monitor_b32:
12529 case Intrinsic::amdgcn_flat_load_monitor_b64:
12530 case Intrinsic::amdgcn_flat_load_monitor_b128: {
12531 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12532 SDValue Chain = Op->getOperand(Num: 0);
12533 SDValue Ptr = Op->getOperand(Num: 2);
12534 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::FLAT_LOAD_MONITOR, dl: DL,
12535 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12536 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12537 }
12538 case Intrinsic::amdgcn_global_load_monitor_b32:
12539 case Intrinsic::amdgcn_global_load_monitor_b64:
12540 case Intrinsic::amdgcn_global_load_monitor_b128: {
12541 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12542 SDValue Chain = Op->getOperand(Num: 0);
12543 SDValue Ptr = Op->getOperand(Num: 2);
12544 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::GLOBAL_LOAD_MONITOR, dl: DL,
12545 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12546 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12547 }
12548 default:
12549
12550 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12551 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID))
12552 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
12553
12554 return SDValue();
12555 }
12556}
12557
12558// Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
12559// dwordx4 if on SI and handle TFE loads.
12560SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
12561 SDVTList VTList,
12562 ArrayRef<SDValue> Ops, EVT MemVT,
12563 MachineMemOperand *MMO,
12564 SelectionDAG &DAG) const {
12565 LLVMContext &C = *DAG.getContext();
12566 MachineFunction &MF = DAG.getMachineFunction();
12567 EVT VT = VTList.VTs[0];
12568
12569 assert(VTList.NumVTs == 2 || VTList.NumVTs == 3);
12570 bool IsTFE = VTList.NumVTs == 3;
12571 if (IsTFE) {
12572 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
12573 unsigned NumOpDWords = NumValueDWords + 1;
12574 EVT OpDWordsVT = EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumOpDWords);
12575 SDVTList OpDWordsVTList = DAG.getVTList(VT1: OpDWordsVT, VT2: VTList.VTs[2]);
12576 MachineMemOperand *OpDWordsMMO =
12577 MF.getMachineMemOperand(MMO, Offset: 0, Size: NumOpDWords * 4);
12578 SDValue Op = getMemIntrinsicNode(Opcode, DL, VTList: OpDWordsVTList, Ops,
12579 MemVT: OpDWordsVT, MMO: OpDWordsMMO, DAG);
12580 auto [Value, Status] = splitTFEValueAndStatus(Op, VT, DL, DAG);
12581 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
12582 }
12583
12584 if (!Subtarget->hasDwordx3LoadStores() &&
12585 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
12586 EVT WidenedVT = EVT::getVectorVT(Context&: C, VT: VT.getVectorElementType(), NumElements: 4);
12587 EVT WidenedMemVT = EVT::getVectorVT(Context&: C, VT: MemVT.getVectorElementType(), NumElements: 4);
12588 MachineMemOperand *WidenedMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 16);
12589 SDVTList WidenedVTList = DAG.getVTList(VT1: WidenedVT, VT2: VTList.VTs[1]);
12590 SDValue Op = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: WidenedVTList, Ops,
12591 MemVT: WidenedMemVT, MMO: WidenedMMO);
12592 SDValue Value = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Op,
12593 N2: DAG.getVectorIdxConstant(Val: 0, DL));
12594 return DAG.getMergeValues(Ops: {Value, SDValue(Op.getNode(), 1)}, dl: DL);
12595 }
12596
12597 return DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops, MemVT, MMO);
12598}
12599
12600SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG,
12601 bool ImageStore) const {
12602 EVT StoreVT = VData.getValueType();
12603
12604 // No change for f16 and legal vector D16 types.
12605 if (!StoreVT.isVector())
12606 return VData;
12607
12608 SDLoc DL(VData);
12609 unsigned NumElements = StoreVT.getVectorNumElements();
12610
12611 if (Subtarget->hasUnpackedD16VMem()) {
12612 // We need to unpack the packed data to store.
12613 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12614 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12615
12616 EVT EquivStoreVT =
12617 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements);
12618 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EquivStoreVT, Operand: IntVData);
12619 return DAG.UnrollVectorOp(N: ZExt.getNode());
12620 }
12621
12622 // The sq block of gfx8.1 does not estimate register use correctly for d16
12623 // image store instructions. The data operand is computed as if it were not a
12624 // d16 image instruction.
12625 if (ImageStore && Subtarget->hasImageStoreD16Bug()) {
12626 // Bitcast to i16
12627 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12628 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12629
12630 // Decompose into scalars
12631 SmallVector<SDValue, 4> Elts;
12632 DAG.ExtractVectorElements(Op: IntVData, Args&: Elts);
12633
12634 // Group pairs of i16 into v2i16 and bitcast to i32
12635 SmallVector<SDValue, 4> PackedElts;
12636 for (unsigned I = 0; I < Elts.size() / 2; I += 1) {
12637 SDValue Pair =
12638 DAG.getBuildVector(VT: MVT::v2i16, DL, Ops: {Elts[I * 2], Elts[I * 2 + 1]});
12639 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12640 PackedElts.push_back(Elt: IntPair);
12641 }
12642 if ((NumElements % 2) == 1) {
12643 // Handle v3i16
12644 unsigned I = Elts.size() / 2;
12645 SDValue Pair = DAG.getBuildVector(VT: MVT::v2i16, DL,
12646 Ops: {Elts[I * 2], DAG.getPOISON(VT: MVT::i16)});
12647 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12648 PackedElts.push_back(Elt: IntPair);
12649 }
12650
12651 // Pad using UNDEF
12652 PackedElts.resize(N: Elts.size(), NV: DAG.getPOISON(VT: MVT::i32));
12653
12654 // Build final vector
12655 EVT VecVT =
12656 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: PackedElts.size());
12657 return DAG.getBuildVector(VT: VecVT, DL, Ops: PackedElts);
12658 }
12659
12660 if (NumElements == 3) {
12661 EVT IntStoreVT =
12662 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreVT.getStoreSizeInBits());
12663 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12664
12665 EVT WidenedStoreVT = EVT::getVectorVT(
12666 Context&: *DAG.getContext(), VT: StoreVT.getVectorElementType(), NumElements: NumElements + 1);
12667 EVT WidenedIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12668 BitWidth: WidenedStoreVT.getStoreSizeInBits());
12669 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenedIntVT, Operand: IntVData);
12670 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WidenedStoreVT, Operand: ZExt);
12671 }
12672
12673 assert(isTypeLegal(StoreVT));
12674 return VData;
12675}
12676
12677static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
12678 switch (Intr) {
12679 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
12680 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
12681 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
12682 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
12683 case Intrinsic::amdgcn_load_async_to_lds:
12684 case Intrinsic::amdgcn_global_load_async_lds:
12685 return true;
12686 }
12687 return false;
12688}
12689
12690SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12691 SelectionDAG &DAG) const {
12692 SDLoc DL(Op);
12693 SDValue Chain = Op.getOperand(i: 0);
12694 unsigned IntrinsicID = Op.getConstantOperandVal(i: 1);
12695
12696 switch (IntrinsicID) {
12697 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
12698 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
12699 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
12700 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
12701 if (Subtarget->hasGFX1250_STRICT())
12702 initializeM0ToZeroForClusterLoad(Op, DAG, DL);
12703 return SDValue();
12704 }
12705 case Intrinsic::amdgcn_exp_compr: {
12706 SDValue Src0 = Op.getOperand(i: 4);
12707 SDValue Src1 = Op.getOperand(i: 5);
12708 // Hack around illegal type on SI by directly selecting it.
12709 if (isTypeLegal(VT: Src0.getValueType()))
12710 return SDValue();
12711
12712 const ConstantSDNode *Done = cast<ConstantSDNode>(Val: Op.getOperand(i: 6));
12713 SDValue Undef = DAG.getPOISON(VT: MVT::f32);
12714 const SDValue Ops[] = {
12715 Op.getOperand(i: 2), // tgt
12716 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src0), // src0
12717 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src1), // src1
12718 Undef, // src2
12719 Undef, // src3
12720 Op.getOperand(i: 7), // vm
12721 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // compr
12722 Op.getOperand(i: 3), // en
12723 Op.getOperand(i: 0) // Chain
12724 };
12725
12726 unsigned Opc = Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
12727 return SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops), 0);
12728 }
12729
12730 case Intrinsic::amdgcn_struct_tbuffer_store:
12731 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
12732 SDValue VData = Op.getOperand(i: 2);
12733 bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
12734 if (IsD16)
12735 VData = handleD16VData(VData, DAG);
12736 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12737 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12738 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12739 SDValue Ops[] = {
12740 Chain,
12741 VData, // vdata
12742 Rsrc, // rsrc
12743 Op.getOperand(i: 4), // vindex
12744 VOffset, // voffset
12745 SOffset, // soffset
12746 Offset, // offset
12747 Op.getOperand(i: 7), // format
12748 Op.getOperand(i: 8), // cachepolicy, swizzled buffer
12749 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12750 };
12751 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12752 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12753 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12754 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12755 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12756 }
12757
12758 case Intrinsic::amdgcn_raw_tbuffer_store:
12759 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
12760 SDValue VData = Op.getOperand(i: 2);
12761 bool IsD16 = (VData.getValueType().getScalarSizeInBits() == 16);
12762 if (IsD16)
12763 VData = handleD16VData(VData, DAG);
12764 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12765 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12766 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12767 SDValue Ops[] = {
12768 Chain,
12769 VData, // vdata
12770 Rsrc, // rsrc
12771 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12772 VOffset, // voffset
12773 SOffset, // soffset
12774 Offset, // offset
12775 Op.getOperand(i: 6), // format
12776 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12777 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12778 };
12779 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12780 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12781 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12782 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12783 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12784 }
12785
12786 case Intrinsic::amdgcn_raw_buffer_store:
12787 case Intrinsic::amdgcn_raw_ptr_buffer_store:
12788 case Intrinsic::amdgcn_raw_buffer_store_format:
12789 case Intrinsic::amdgcn_raw_ptr_buffer_store_format: {
12790 const bool IsFormat =
12791 IntrinsicID == Intrinsic::amdgcn_raw_buffer_store_format ||
12792 IntrinsicID == Intrinsic::amdgcn_raw_ptr_buffer_store_format;
12793
12794 SDValue VData = Op.getOperand(i: 2);
12795 EVT VDataVT = VData.getValueType();
12796 EVT EltType = VDataVT.getScalarType();
12797 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12798
12799 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12800 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12801 DAG.getMachineFunction().getFunction(),
12802 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12803 return Chain;
12804 }
12805
12806 if (IsD16) {
12807 VData = handleD16VData(VData, DAG);
12808 VDataVT = VData.getValueType();
12809 }
12810
12811 if (!isTypeLegal(VT: VDataVT)) {
12812 VData =
12813 DAG.getNode(Opcode: ISD::BITCAST, DL,
12814 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12815 }
12816
12817 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12818 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12819 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12820 SDValue Ops[] = {
12821 Chain,
12822 VData,
12823 Rsrc,
12824 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12825 VOffset, // voffset
12826 SOffset, // soffset
12827 Offset, // offset
12828 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
12829 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12830 };
12831 unsigned Opc =
12832 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
12833 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12834 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12835
12836 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12837 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12838 return handleByteShortBufferStores(DAG, VDataType: VDataVT, DL, Ops, M);
12839
12840 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12841 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12842 }
12843
12844 case Intrinsic::amdgcn_struct_buffer_store:
12845 case Intrinsic::amdgcn_struct_ptr_buffer_store:
12846 case Intrinsic::amdgcn_struct_buffer_store_format:
12847 case Intrinsic::amdgcn_struct_ptr_buffer_store_format: {
12848 const bool IsFormat =
12849 IntrinsicID == Intrinsic::amdgcn_struct_buffer_store_format ||
12850 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_store_format;
12851
12852 SDValue VData = Op.getOperand(i: 2);
12853 EVT VDataVT = VData.getValueType();
12854 EVT EltType = VDataVT.getScalarType();
12855 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12856
12857 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12858 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12859 DAG.getMachineFunction().getFunction(),
12860 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12861 return Chain;
12862 }
12863
12864 if (IsD16) {
12865 VData = handleD16VData(VData, DAG);
12866 VDataVT = VData.getValueType();
12867 }
12868
12869 if (!isTypeLegal(VT: VDataVT)) {
12870 VData =
12871 DAG.getNode(Opcode: ISD::BITCAST, DL,
12872 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12873 }
12874
12875 auto Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12876 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12877 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12878 SDValue Ops[] = {
12879 Chain,
12880 VData,
12881 Rsrc,
12882 Op.getOperand(i: 4), // vindex
12883 VOffset, // voffset
12884 SOffset, // soffset
12885 Offset, // offset
12886 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12887 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12888 };
12889 unsigned Opc =
12890 !IsFormat ? AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT;
12891 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12892 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12893
12894 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12895 EVT VDataType = VData.getValueType().getScalarType();
12896 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12897 return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M);
12898
12899 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12900 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12901 }
12902 case Intrinsic::amdgcn_raw_buffer_load_lds:
12903 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
12904 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
12905 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
12906 case Intrinsic::amdgcn_struct_buffer_load_lds:
12907 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
12908 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
12909 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
12910 unsigned Opc;
12911 bool HasVIndex =
12912 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds ||
12913 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_async_lds ||
12914 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_lds ||
12915 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds;
12916 unsigned OpOffset = HasVIndex ? 1 : 0;
12917 SDValue VOffset = Op.getOperand(i: 5 + OpOffset);
12918 bool HasVOffset = !isNullConstant(V: VOffset);
12919 unsigned Size = Op->getConstantOperandVal(Num: 4);
12920
12921 switch (Size) {
12922 default:
12923 return SDValue();
12924 case 1:
12925 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
12926 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
12927 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
12928 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
12929 break;
12930 case 2:
12931 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
12932 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
12933 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
12934 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
12935 break;
12936 case 4:
12937 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
12938 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
12939 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
12940 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
12941 break;
12942 case 12:
12943 if (!Subtarget->hasLDSLoadB96_B128())
12944 return SDValue();
12945 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
12946 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
12947 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
12948 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
12949 break;
12950 case 16:
12951 if (!Subtarget->hasLDSLoadB96_B128())
12952 return SDValue();
12953 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
12954 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
12955 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
12956 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
12957 break;
12958 }
12959
12960 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12961
12962 SmallVector<SDValue, 8> Ops;
12963
12964 if (HasVIndex && HasVOffset)
12965 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v2i32, DL,
12966 Ops: {Op.getOperand(i: 5), // VIndex
12967 VOffset}));
12968 else if (HasVIndex)
12969 Ops.push_back(Elt: Op.getOperand(i: 5));
12970 else if (HasVOffset)
12971 Ops.push_back(Elt: VOffset);
12972
12973 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12974 Ops.push_back(Elt: Rsrc);
12975 Ops.push_back(Elt: Op.getOperand(i: 6 + OpOffset)); // soffset
12976 Ops.push_back(Elt: Op.getOperand(i: 7 + OpOffset)); // imm offset
12977 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12978 unsigned Aux = Op.getConstantOperandVal(i: 8 + OpOffset);
12979 Ops.push_back(Elt: DAG.getTargetConstant(
12980 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12),
12981 DL, VT: MVT::i8)); // cpol
12982 Ops.push_back(Elt: DAG.getTargetConstant(
12983 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
12984 ? 1
12985 : 0,
12986 DL, VT: MVT::i8)); // swz
12987 Ops.push_back(
12988 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
12989 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12990 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12991
12992 auto *M = cast<MemSDNode>(Val&: Op);
12993 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: M->getVTList(), Ops);
12994 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12995
12996 return SDValue(Load, 0);
12997 }
12998 // Buffers are handled by LowerBufferFatPointers, and we're going to go
12999 // for "trust me" that the remaining cases are global pointers until
13000 // such time as we can put two mem operands on an intrinsic.
13001 case Intrinsic::amdgcn_load_to_lds:
13002 case Intrinsic::amdgcn_load_async_to_lds:
13003 case Intrinsic::amdgcn_global_load_lds:
13004 case Intrinsic::amdgcn_global_load_async_lds: {
13005 if (!Subtarget->hasVMemToLDSLoad())
13006 return SDValue();
13007
13008 unsigned Opc;
13009 unsigned Size = Op->getConstantOperandVal(Num: 4);
13010 switch (Size) {
13011 default:
13012 return SDValue();
13013 case 1:
13014 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
13015 break;
13016 case 2:
13017 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
13018 break;
13019 case 4:
13020 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
13021 break;
13022 case 12:
13023 if (!Subtarget->hasLDSLoadB96_B128())
13024 return SDValue();
13025 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
13026 break;
13027 case 16:
13028 if (!Subtarget->hasLDSLoadB96_B128())
13029 return SDValue();
13030 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
13031 break;
13032 }
13033
13034 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
13035
13036 SmallVector<SDValue, 6> Ops;
13037
13038 SDValue Addr = Op.getOperand(i: 2); // Global ptr
13039 SDValue VOffset;
13040 // Try to split SAddr and VOffset. Global and LDS pointers share the same
13041 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
13042 if (Addr->isDivergent() && Addr->isAnyAdd()) {
13043 SDValue LHS = Addr.getOperand(i: 0);
13044 SDValue RHS = Addr.getOperand(i: 1);
13045
13046 if (LHS->isDivergent())
13047 std::swap(a&: LHS, b&: RHS);
13048
13049 if (!LHS->isDivergent() && RHS.getOpcode() == ISD::ZERO_EXTEND &&
13050 RHS.getOperand(i: 0).getValueType() == MVT::i32) {
13051 // add (i64 sgpr), (zero_extend (i32 vgpr))
13052 Addr = LHS;
13053 VOffset = RHS.getOperand(i: 0);
13054 }
13055 }
13056
13057 Ops.push_back(Elt: Addr);
13058 if (!Addr->isDivergent()) {
13059 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
13060 if (!VOffset)
13061 VOffset =
13062 SDValue(DAG.getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32, dl: DL, VT: MVT::i32,
13063 Op1: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)),
13064 0);
13065 Ops.push_back(Elt: VOffset);
13066 }
13067
13068 Ops.push_back(Elt: Op.getOperand(i: 5)); // Offset
13069
13070 unsigned Aux = Op.getConstantOperandVal(i: 6);
13071 Ops.push_back(Elt: DAG.getTargetConstant(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS, DL,
13072 VT: MVT::i32)); // CPol
13073 Ops.push_back(
13074 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
13075
13076 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
13077 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
13078
13079 auto *M = cast<MemSDNode>(Val&: Op);
13080 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13081 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
13082
13083 return SDValue(Load, 0);
13084 }
13085 case Intrinsic::amdgcn_end_cf:
13086 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_END_CF, dl: DL, VT: MVT::Other,
13087 Op1: Op->getOperand(Num: 2), Op2: Chain),
13088 0);
13089 case Intrinsic::amdgcn_s_barrier_signal_var: {
13090 // Member count of 0 means to re-use a previous member count,
13091 // which, if the named barrier is statically chosen, means we can use
13092 // the immarg form. Otherwisee, fall through to constructiong M0 as for
13093 // s_barrier_init.
13094 SDValue CntOp = Op->getOperand(Num: 3);
13095 auto *CntC = dyn_cast<ConstantSDNode>(Val&: CntOp);
13096 if (CntC && CntC->isZero()) {
13097 SDValue Chain = Op->getOperand(Num: 0);
13098 SDValue BarOp = Op->getOperand(Num: 2);
13099 SmallVector<SDValue, 2> Ops;
13100
13101 std::optional<uint64_t> BarVal;
13102 if (auto *C = dyn_cast<ConstantSDNode>(Val&: BarOp))
13103 BarVal = C->getZExtValue();
13104 else if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: BarOp))
13105 if (auto Addr = AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(
13106 GV: *GA->getGlobal(), AS: AMDGPUAS::BARRIER))
13107 BarVal = *Addr + GA->getOffset();
13108
13109 if (BarVal) {
13110 unsigned BarID = *BarVal & 0x3F;
13111 Ops.push_back(Elt: DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32));
13112 Ops.push_back(Elt: Chain);
13113 auto *NewMI = DAG.getMachineNode(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM, dl: DL,
13114 VTs: Op->getVTList(), Ops);
13115 return SDValue(NewMI, 0);
13116 }
13117 }
13118 [[fallthrough]];
13119 }
13120 case Intrinsic::amdgcn_s_barrier_init: {
13121 // these two intrinsics have two operands: barrier pointer and member count
13122 SDValue Chain = Op->getOperand(Num: 0);
13123 SmallVector<SDValue, 2> Ops;
13124 SDValue BarOp = Op->getOperand(Num: 2);
13125 SDValue CntOp = Op->getOperand(Num: 3);
13126 SDValue M0Val;
13127 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
13128 ? AMDGPU::S_BARRIER_INIT_M0
13129 : AMDGPU::S_BARRIER_SIGNAL_M0;
13130 // extract the BarrierID from bits 0-5 of BarOp
13131 SDValue BarID = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: BarOp,
13132 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
13133 // Member count should be put into M0[ShAmt:+CntMask width]
13134 // Barrier ID should be put into M0[5:0]
13135 unsigned CntMask = AMDGPU::getBarrierMemberCountBitMask(
13136 Version: AMDGPU::getIsaVersion(GPU: Subtarget->getCPU()));
13137 SDValue MemberCnt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: CntOp,
13138 N2: DAG.getConstant(Val: CntMask, DL, VT: MVT::i32));
13139 constexpr unsigned ShAmt = 16;
13140 M0Val = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: MemberCnt,
13141 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
13142
13143 M0Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M0Val, N2: BarID);
13144
13145 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
13146
13147 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13148 return SDValue(NewMI, 0);
13149 }
13150 case Intrinsic::amdgcn_s_wakeup_barrier: {
13151 if (!Subtarget->hasSWakeupBarrier())
13152 return SDValue();
13153 [[fallthrough]];
13154 }
13155 case Intrinsic::amdgcn_s_barrier_join: {
13156 // these three intrinsics have one operand: barrier pointer
13157 SDValue Chain = Op->getOperand(Num: 0);
13158 SmallVector<SDValue, 2> Ops;
13159 SDValue BarOp = Op->getOperand(Num: 2);
13160 unsigned Opc;
13161
13162 if (isa<ConstantSDNode>(Val: BarOp)) {
13163 uint64_t BarVal = cast<ConstantSDNode>(Val&: BarOp)->getZExtValue();
13164 switch (IntrinsicID) {
13165 default:
13166 return SDValue();
13167 case Intrinsic::amdgcn_s_barrier_join:
13168 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
13169 break;
13170 case Intrinsic::amdgcn_s_wakeup_barrier:
13171 Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
13172 break;
13173 }
13174 // extract the BarrierID from bits 0-5 of the immediate
13175 unsigned BarID = BarVal & 0x3F;
13176 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
13177 Ops.push_back(Elt: K);
13178 Ops.push_back(Elt: Chain);
13179 } else {
13180 switch (IntrinsicID) {
13181 default:
13182 return SDValue();
13183 case Intrinsic::amdgcn_s_barrier_join:
13184 Opc = AMDGPU::S_BARRIER_JOIN_M0;
13185 break;
13186 case Intrinsic::amdgcn_s_wakeup_barrier:
13187 Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
13188 break;
13189 }
13190 // extract the BarrierID from bits 0-5 of BarOp, copy to M0[5:0]
13191 SDValue M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: BarOp,
13192 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
13193 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
13194 }
13195
13196 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13197 return SDValue(NewMI, 0);
13198 }
13199 case Intrinsic::amdgcn_s_prefetch_data:
13200 case Intrinsic::amdgcn_s_prefetch_inst: {
13201 // For non-global address space preserve the chain and remove the call.
13202 if (!AMDGPU::isFlatGlobalAddrSpace(AS: cast<MemSDNode>(Val&: Op)->getAddressSpace()))
13203 return Op.getOperand(i: 0);
13204 return Op;
13205 }
13206 case Intrinsic::amdgcn_s_buffer_prefetch_data: {
13207 SDValue Ops[] = {
13208 Chain, bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG),
13209 Op.getOperand(i: 3), // offset
13210 Op.getOperand(i: 4), // length
13211 };
13212
13213 MemSDNode *M = cast<MemSDNode>(Val&: Op);
13214 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_PREFETCH_DATA, dl: DL,
13215 VTList: Op->getVTList(), Ops, MemVT: M->getMemoryVT(),
13216 MMO: M->getMemOperand());
13217 }
13218 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
13219 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
13220 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
13221 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
13222 SDValue Chain = Op->getOperand(Num: 0);
13223 SDValue Ptr = Op->getOperand(Num: 2);
13224 SDValue Val = Op->getOperand(Num: 3);
13225 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MII->getMemoryVT(), Chain, Ptr: Val,
13226 Val: Ptr, MMO: MII->getMemOperand());
13227 }
13228 case Intrinsic::amdgcn_av_store_b128: {
13229 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
13230 SDValue Chain = Op->getOperand(Num: 0);
13231 SDValue Ptr = Op->getOperand(Num: 2);
13232 SDValue Val = Op->getOperand(Num: 3);
13233 return DAG.getStore(Chain, dl: DL, Val, Ptr, MMO: MII->getMemOperand());
13234 }
13235 default: {
13236 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
13237 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
13238 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
13239
13240 return Op;
13241 }
13242 }
13243}
13244
13245// Return whether the operation has NoUnsignedWrap property.
13246static bool isNoUnsignedWrap(SDValue Addr) {
13247 return (Addr.getOpcode() == ISD::ADD &&
13248 Addr->getFlags().hasNoUnsignedWrap()) ||
13249 Addr->getOpcode() == ISD::OR;
13250}
13251
13252bool SITargetLowering::shouldPreservePtrArith(const Function &F,
13253 EVT PtrVT) const {
13254 return PtrVT == MVT::i64;
13255}
13256
13257bool SITargetLowering::canTransformPtrArithOutOfBounds(const Function &F,
13258 EVT PtrVT) const {
13259 return true;
13260}
13261
13262// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
13263// offset (the offset that is included in bounds checking and swizzling, to be
13264// split between the instruction's voffset and immoffset fields) and soffset
13265// (the offset that is excluded from bounds checking and swizzling, to go in
13266// the instruction's soffset field). This function takes the first kind of
13267// offset and figures out how to split it between voffset and immoffset.
13268std::pair<SDValue, SDValue>
13269SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
13270 SDLoc DL(Offset);
13271 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
13272 SDValue N0 = Offset;
13273 ConstantSDNode *C1 = nullptr;
13274
13275 if ((C1 = dyn_cast<ConstantSDNode>(Val&: N0)))
13276 N0 = SDValue();
13277 else if (DAG.isBaseWithConstantOffset(Op: N0)) {
13278 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13279 // being added, so we can only safely match a 32-bit addition with no
13280 // unsigned overflow.
13281 bool CheckNUW = Subtarget->hasGFX1250Insts();
13282 if (!CheckNUW || isNoUnsignedWrap(Addr: N0)) {
13283 C1 = cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
13284 N0 = N0.getOperand(i: 0);
13285 }
13286 }
13287
13288 if (C1) {
13289 unsigned ImmOffset = C1->getZExtValue();
13290 // If the immediate value is too big for the immoffset field, put only bits
13291 // that would normally fit in the immoffset field. The remaining value that
13292 // is copied/added for the voffset field is a large power of 2, and it
13293 // stands more chance of being CSEd with the copy/add for another similar
13294 // load/store.
13295 // However, do not do that rounding down if that is a negative
13296 // number, as it appears to be illegal to have a negative offset in the
13297 // vgpr, even if adding the immediate offset makes it positive.
13298 unsigned Overflow = ImmOffset & ~MaxImm;
13299 ImmOffset -= Overflow;
13300 if ((int32_t)Overflow < 0) {
13301 Overflow += ImmOffset;
13302 ImmOffset = 0;
13303 }
13304 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32));
13305 if (Overflow) {
13306 auto OverflowVal = DAG.getConstant(Val: Overflow, DL, VT: MVT::i32);
13307 if (!N0)
13308 N0 = OverflowVal;
13309 else {
13310 SDValue Ops[] = {N0, OverflowVal};
13311 N0 = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, Ops);
13312 }
13313 }
13314 }
13315 if (!N0)
13316 N0 = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13317 if (!C1)
13318 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
13319 return {N0, SDValue(C1, 0)};
13320}
13321
13322// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
13323// the three offsets (voffset, soffset and instoffset) into the SDValue[3] array
13324// pointed to by Offsets.
13325void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
13326 SelectionDAG &DAG, SDValue *Offsets,
13327 Align Alignment) const {
13328 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
13329 SDLoc DL(CombinedOffset);
13330 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CombinedOffset)) {
13331 uint32_t Imm = C->getZExtValue();
13332 uint32_t SOffset, ImmOffset;
13333 if (TII->splitMUBUFOffset(Imm, SOffset, ImmOffset, Alignment)) {
13334 Offsets[0] = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13335 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
13336 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
13337 return;
13338 }
13339 }
13340 if (DAG.isBaseWithConstantOffset(Op: CombinedOffset)) {
13341 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13342 // being added, so we can only safely match a 32-bit addition with no
13343 // unsigned overflow.
13344 bool CheckNUW = Subtarget->hasGFX1250Insts();
13345 SDValue N0 = CombinedOffset.getOperand(i: 0);
13346 SDValue N1 = CombinedOffset.getOperand(i: 1);
13347 uint32_t SOffset, ImmOffset;
13348 int Offset = cast<ConstantSDNode>(Val&: N1)->getSExtValue();
13349 if (Offset >= 0 && (!CheckNUW || isNoUnsignedWrap(Addr: CombinedOffset)) &&
13350 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
13351 Offsets[0] = N0;
13352 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
13353 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
13354 return;
13355 }
13356 }
13357
13358 SDValue SOffsetZero = Subtarget->hasRestrictedSOffset()
13359 ? DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32)
13360 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13361
13362 Offsets[0] = CombinedOffset;
13363 Offsets[1] = SOffsetZero;
13364 Offsets[2] = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
13365}
13366
13367SDValue SITargetLowering::bufferRsrcPtrToVector(SDValue MaybePointer,
13368 SelectionDAG &DAG) const {
13369 if (!MaybePointer.getValueType().isScalarInteger())
13370 return MaybePointer;
13371
13372 SDValue Rsrc = DAG.getBitcast(VT: MVT::v4i32, V: MaybePointer);
13373 return Rsrc;
13374}
13375
13376// Wrap a global or flat pointer into a buffer intrinsic using the flags
13377// specified in the intrinsic.
13378SDValue SITargetLowering::lowerPointerAsRsrcIntrin(SDNode *Op,
13379 SelectionDAG &DAG) const {
13380 SDLoc Loc(Op);
13381
13382 SDValue Pointer = Op->getOperand(Num: 1);
13383 SDValue Stride = Op->getOperand(Num: 2);
13384 SDValue NumRecords = Op->getOperand(Num: 3);
13385 SDValue Flags = Op->getOperand(Num: 4);
13386
13387 SDValue ExtStride = DAG.getAnyExtOrTrunc(Op: Stride, DL: Loc, VT: MVT::i32);
13388 SDValue Rsrc;
13389
13390 if (Subtarget->getBufferResourceNumRecordsWidth() == 45) {
13391 NumRecords = DAG.getZExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i64);
13392 NumRecords = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i64, N1: NumRecords,
13393 N2: DAG.getConstant(Val: (1ULL << 45) - 1, DL: Loc, VT: MVT::i64));
13394 SDValue Zero = DAG.getConstant(Val: 0, DL: Loc, VT: MVT::i32);
13395 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
13396 // num_records.
13397 SDValue ExtPointer = DAG.getAnyExtOrTrunc(Op: Pointer, DL: Loc, VT: MVT::i64);
13398 SDValue NumRecordsLHS =
13399 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i64, N1: NumRecords,
13400 N2: DAG.getShiftAmountConstant(Val: 57, VT: MVT::i32, DL: Loc));
13401 SDValue LowHalf =
13402 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: ExtPointer, N2: NumRecordsLHS);
13403
13404 // Build the higher 64-bit value, which has the higher 38-bit num_records,
13405 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
13406 SDValue NumRecordsRHS =
13407 DAG.getNode(Opcode: ISD::SRL, DL: Loc, VT: MVT::i64, N1: NumRecords,
13408 N2: DAG.getShiftAmountConstant(Val: 7, VT: MVT::i32, DL: Loc));
13409 SDValue ShiftedStride =
13410 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13411 N2: DAG.getShiftAmountConstant(Val: 12, VT: MVT::i32, DL: Loc));
13412 SDValue ExtShiftedStrideVec =
13413 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedStride);
13414 SDValue ExtShiftedStride =
13415 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedStrideVec);
13416 SDValue ShiftedFlags =
13417 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: Flags,
13418 N2: DAG.getShiftAmountConstant(Val: 28, VT: MVT::i32, DL: Loc));
13419 SDValue ExtShiftedFlagsVec =
13420 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedFlags);
13421 SDValue ExtShiftedFlags =
13422 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedFlagsVec);
13423 SDValue CombinedFields =
13424 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: NumRecordsRHS, N2: ExtShiftedStride);
13425 SDValue HighHalf =
13426 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: CombinedFields, N2: ExtShiftedFlags);
13427
13428 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i64, N1: LowHalf, N2: HighHalf);
13429 } else {
13430 NumRecords = DAG.getZExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i32);
13431 auto [LowHalf, HighHalf] =
13432 DAG.SplitScalar(N: Pointer, DL: Loc, LoVT: MVT::i32, HiVT: MVT::i32);
13433 SDValue Mask = DAG.getConstant(Val: 0x0000ffff, DL: Loc, VT: MVT::i32);
13434 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i32, N1: HighHalf, N2: Mask);
13435 SDValue ShiftedStride =
13436 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13437 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: Loc));
13438 SDValue NewHighHalf =
13439 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i32, N1: Masked, N2: ShiftedStride);
13440
13441 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v4i32, N1: LowHalf, N2: NewHighHalf,
13442 N3: NumRecords, N4: Flags);
13443 }
13444
13445 SDValue RsrcPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i128, Operand: Rsrc);
13446 return RsrcPtr;
13447}
13448
13449// Handle 8 bit and 16 bit buffer loads
13450SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
13451 EVT LoadVT, SDLoc DL,
13452 ArrayRef<SDValue> Ops,
13453 MachineMemOperand *MMO,
13454 bool IsTFE) const {
13455 EVT IntVT = LoadVT.changeTypeToInteger();
13456
13457 if (IsTFE) {
13458 unsigned Opc = (LoadVT.getScalarType() == MVT::i8)
13459 ? AMDGPUISD::BUFFER_LOAD_UBYTE_TFE
13460 : AMDGPUISD::BUFFER_LOAD_USHORT_TFE;
13461 MachineFunction &MF = DAG.getMachineFunction();
13462 MachineMemOperand *OpMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 8);
13463 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i32, VT2: MVT::Other);
13464 SDValue Op = getMemIntrinsicNode(Opcode: Opc, DL, VTList: VTs, Ops, MemVT: MVT::v2i32, MMO: OpMMO, DAG);
13465 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13466 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
13467 SDValue Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13468 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
13469 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Data);
13470 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: Trunc);
13471 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
13472 }
13473
13474 unsigned Opc = LoadVT.getScalarType() == MVT::i8
13475 ? AMDGPUISD::BUFFER_LOAD_UBYTE
13476 : AMDGPUISD::BUFFER_LOAD_USHORT;
13477
13478 SDVTList ResList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
13479 SDValue BufferLoad =
13480 DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: IntVT, MMO);
13481 SDValue LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: BufferLoad);
13482 LoadVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: LoadVal);
13483
13484 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
13485}
13486
13487// Handle 8 bit and 16 bit buffer stores
13488SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
13489 EVT VDataType, SDLoc DL,
13490 SDValue Ops[],
13491 MemSDNode *M) const {
13492 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
13493 Ops[1] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Ops[1]);
13494
13495 SDValue BufferStoreExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Ops[1]);
13496 Ops[1] = BufferStoreExt;
13497 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE
13498 : AMDGPUISD::BUFFER_STORE_SHORT;
13499 ArrayRef<SDValue> OpsRef = ArrayRef(&Ops[0], 9);
13500 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: M->getVTList(), Ops: OpsRef, MemVT: VDataType,
13501 MMO: M->getMemOperand());
13502}
13503
13504static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType,
13505 SDValue Op, const SDLoc &SL, EVT VT) {
13506 if (VT.bitsLT(VT: Op.getValueType()))
13507 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
13508
13509 switch (ExtType) {
13510 case ISD::SEXTLOAD:
13511 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SL, VT, Operand: Op);
13512 case ISD::ZEXTLOAD:
13513 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT, Operand: Op);
13514 case ISD::EXTLOAD:
13515 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT, Operand: Op);
13516 case ISD::NON_EXTLOAD:
13517 return Op;
13518 }
13519
13520 llvm_unreachable("invalid ext type");
13521}
13522
13523// Try to turn 8 and 16-bit scalar loads into SMEM eligible 32-bit loads.
13524// TODO: Skip this on GFX12 which does have scalar sub-dword loads.
13525SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
13526 DAGCombinerInfo &DCI) const {
13527 SelectionDAG &DAG = DCI.DAG;
13528 if (Ld->getAlign() < Align(4) || Ld->isDivergent())
13529 return SDValue();
13530
13531 // FIXME: Constant loads should all be marked invariant.
13532 unsigned AS = Ld->getAddressSpace();
13533 if (AS != AMDGPUAS::CONSTANT_ADDRESS &&
13534 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
13535 (AS != AMDGPUAS::GLOBAL_ADDRESS || !Ld->isInvariant()))
13536 return SDValue();
13537
13538 // Don't do this early, since it may interfere with adjacent load merging for
13539 // illegal types. We can avoid losing alignment information for exotic types
13540 // pre-legalize.
13541 EVT MemVT = Ld->getMemoryVT();
13542 if ((MemVT.isSimple() && !DCI.isAfterLegalizeDAG()) ||
13543 MemVT.getSizeInBits() >= 32)
13544 return SDValue();
13545
13546 SDLoc SL(Ld);
13547
13548 assert((!MemVT.isVector() || Ld->getExtensionType() == ISD::NON_EXTLOAD) &&
13549 "unexpected vector extload");
13550
13551 // TODO: Drop only high part of range.
13552 SDValue Ptr = Ld->getBasePtr();
13553 SDValue NewLoad = DAG.getLoad(
13554 AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT: MVT::i32, dl: SL, Chain: Ld->getChain(), Ptr,
13555 Offset: Ld->getOffset(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i32, Alignment: Ld->getAlign(),
13556 MMOFlags: Ld->getMemOperand()->getFlags(), Metadata: Ld->getAAInfo()); // Drop ranges
13557
13558 EVT TruncVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
13559 if (MemVT.isFloatingPoint()) {
13560 assert(Ld->getExtensionType() == ISD::NON_EXTLOAD &&
13561 "unexpected fp extload");
13562 TruncVT = MemVT.changeTypeToInteger();
13563 }
13564
13565 SDValue Cvt = NewLoad;
13566 if (Ld->getExtensionType() == ISD::SEXTLOAD) {
13567 Cvt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SL, VT: MVT::i32, N1: NewLoad,
13568 N2: DAG.getValueType(TruncVT));
13569 } else if (Ld->getExtensionType() == ISD::ZEXTLOAD ||
13570 Ld->getExtensionType() == ISD::NON_EXTLOAD) {
13571 Cvt = DAG.getZeroExtendInReg(Op: NewLoad, DL: SL, VT: TruncVT);
13572 } else {
13573 assert(Ld->getExtensionType() == ISD::EXTLOAD);
13574 }
13575
13576 EVT VT = Ld->getValueType(ResNo: 0);
13577 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
13578
13579 DCI.AddToWorklist(N: Cvt.getNode());
13580
13581 // We may need to handle exotic cases, such as i16->i64 extloads, so insert
13582 // the appropriate extension from the 32-bit load.
13583 Cvt = getLoadExtOrTrunc(DAG, ExtType: Ld->getExtensionType(), Op: Cvt, SL, VT: IntVT);
13584 DCI.AddToWorklist(N: Cvt.getNode());
13585
13586 // Handle conversion back to floating point if necessary.
13587 Cvt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Cvt);
13588
13589 return DAG.getMergeValues(Ops: {Cvt, NewLoad.getValue(R: 1)}, dl: SL);
13590}
13591
13592static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO,
13593 const SIMachineFunctionInfo &Info) {
13594 // TODO: Should check if the address can definitely not access stack.
13595 if (Info.isEntryFunction())
13596 return Info.getUserSGPRInfo().hasFlatScratchInit();
13597 return true;
13598}
13599
13600SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
13601 SDLoc DL(Op);
13602 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
13603 ISD::LoadExtType ExtType = Load->getExtensionType();
13604 EVT MemVT = Load->getMemoryVT();
13605 MachineMemOperand *MMO = Load->getMemOperand();
13606
13607 if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
13608 // Legalize uniform 16-bit loads to i16 = trunc (zextload i16->i32)
13609 // to match subword load patterns.
13610 // Only do this for loads that can use scalar subword load instructions.
13611 if (!MemVT.isVector() && MemVT.getSizeInBits() == 16 &&
13612 isTypeLegal(VT: MemVT) && Subtarget->hasScalarSubwordLoads() &&
13613 isUniformLoad(Load)) {
13614 SDValue Chain = Load->getChain();
13615 SDValue BasePtr = Load->getBasePtr();
13616
13617 // Load as i16 and zero-extend to i32 (matches S_LOAD_U16 behavior)
13618 SDValue NewLD = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: DL, VT: MVT::i32, Chain,
13619 Ptr: BasePtr, MemVT: MVT::i16, MMO);
13620
13621 // Truncate back to i16
13622 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: NewLD);
13623
13624 // For f16/bf16, bitcast from i16 to the original fp type
13625 SDValue Result = (MemVT == MVT::i16)
13626 ? Trunc
13627 : DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MemVT, Operand: Trunc);
13628
13629 SDValue Ops[] = {Result, NewLD.getValue(R: 1)};
13630 return DAG.getMergeValues(Ops, dl: DL);
13631 }
13632
13633 if (MemVT == MVT::i16 && isTypeLegal(VT: MVT::i16))
13634 return SDValue();
13635
13636 // FIXME: Copied from PPC
13637 // First, load into 32 bits, then truncate to 1 bit.
13638
13639 SDValue Chain = Load->getChain();
13640 SDValue BasePtr = Load->getBasePtr();
13641
13642 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
13643
13644 SDValue NewLD = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i32, Chain, Ptr: BasePtr,
13645 MemVT: RealMemVT, MMO);
13646
13647 if (!MemVT.isVector()) {
13648 SDValue Ops[] = {DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: NewLD),
13649 NewLD.getValue(R: 1)};
13650
13651 return DAG.getMergeValues(Ops, dl: DL);
13652 }
13653
13654 SmallVector<SDValue, 3> Elts;
13655 for (unsigned I = 0, N = MemVT.getVectorNumElements(); I != N; ++I) {
13656 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: NewLD,
13657 N2: DAG.getConstant(Val: I, DL, VT: MVT::i32));
13658
13659 Elts.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Elt));
13660 }
13661
13662 SDValue Ops[] = {DAG.getBuildVector(VT: MemVT, DL, Ops: Elts), NewLD.getValue(R: 1)};
13663
13664 return DAG.getMergeValues(Ops, dl: DL);
13665 }
13666
13667 if (!MemVT.isVector())
13668 return SDValue();
13669
13670 assert(Op.getValueType().getVectorElementType() == MVT::i32 &&
13671 "Custom lowering for non-i32 vectors hasn't been implemented.");
13672
13673 Align Alignment = Load->getAlign();
13674 unsigned AS = Load->getAddressSpace();
13675 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13676 AS == AMDGPUAS::FLAT_ADDRESS &&
13677 Alignment.value() < MemVT.getStoreSize() && MemVT.getSizeInBits() > 32) {
13678 return SplitVectorLoad(Op, DAG);
13679 }
13680
13681 MachineFunction &MF = DAG.getMachineFunction();
13682 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
13683 // If there is a possibility that flat instruction access scratch memory
13684 // then we need to use the same legalization rules we use for private.
13685 if (AS == AMDGPUAS::FLAT_ADDRESS &&
13686 !Subtarget->hasMultiDwordFlatScratchAddressing())
13687 AS = addressMayBeAccessedAsPrivate(MMO: Load->getMemOperand(), Info: *MFI)
13688 ? AMDGPUAS::PRIVATE_ADDRESS
13689 : AMDGPUAS::GLOBAL_ADDRESS;
13690
13691 unsigned NumElements = MemVT.getVectorNumElements();
13692
13693 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13694 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13695 (AS == AMDGPUAS::GLOBAL_ADDRESS && Load->isSimple() &&
13696 (Load->isInvariant() || isMemOpHasNoClobberedMemOperand(N: Load)))) {
13697 if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) &&
13698 Alignment >= Align(4) && NumElements < 32) {
13699 if (MemVT.isPow2VectorType() ||
13700 (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
13701 return SDValue();
13702 return WidenOrSplitVectorLoad(Op, DAG);
13703 }
13704 // Non-uniform loads will be selected to MUBUF instructions, so they
13705 // have the same legalization requirements as global and private
13706 // loads.
13707 //
13708 }
13709 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13710 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13711 AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
13712 if (NumElements > 4)
13713 return SplitVectorLoad(Op, DAG);
13714 // v3 loads not supported on SI.
13715 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13716 return WidenOrSplitVectorLoad(Op, DAG);
13717
13718 // v3 and v4 loads are supported for private and global memory.
13719 return SDValue();
13720 }
13721 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
13722 // Depending on the setting of the private_element_size field in the
13723 // resource descriptor, we can only make private accesses up to a certain
13724 // size.
13725 switch (Subtarget->getMaxPrivateElementSize()) {
13726 case 4: {
13727 auto [Op0, Op1] = scalarizeVectorLoad(LD: Load, DAG);
13728 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13729 }
13730 case 8:
13731 if (NumElements > 2)
13732 return SplitVectorLoad(Op, DAG);
13733 return SDValue();
13734 case 16:
13735 // Same as global/flat
13736 if (NumElements > 4)
13737 return SplitVectorLoad(Op, DAG);
13738 // v3 loads not supported on SI.
13739 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13740 return WidenOrSplitVectorLoad(Op, DAG);
13741
13742 return SDValue();
13743 default:
13744 llvm_unreachable("unsupported private_element_size");
13745 }
13746 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
13747 unsigned Fast = 0;
13748 auto Flags = Load->getMemOperand()->getFlags();
13749 if (allowsMisalignedMemoryAccessesImpl(Size: MemVT.getSizeInBits(), AddrSpace: AS,
13750 Alignment: Load->getAlign(), Flags, IsFast: &Fast) &&
13751 Fast > 1)
13752 return SDValue();
13753
13754 if (MemVT.isVector())
13755 return SplitVectorLoad(Op, DAG);
13756 }
13757
13758 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
13759 VT: MemVT, MMO: *Load->getMemOperand())) {
13760 auto [Op0, Op1] = expandUnalignedLoad(LD: Load, DAG);
13761 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13762 }
13763
13764 return SDValue();
13765}
13766
13767SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
13768 EVT VT = Op.getValueType();
13769 if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256 ||
13770 VT.getSizeInBits() == 512)
13771 return splitTernaryVectorOp(Op, DAG);
13772
13773 assert(VT.getSizeInBits() == 64);
13774
13775 SDLoc DL(Op);
13776 SDValue Cond = DAG.getFreeze(V: Op.getOperand(i: 0));
13777
13778 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13779 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
13780
13781 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
13782 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 2));
13783
13784 SDValue Lo0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: Zero);
13785 SDValue Lo1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: Zero);
13786
13787 SDValue Lo = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Lo0, RHS: Lo1);
13788
13789 SDValue Hi0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: One);
13790 SDValue Hi1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: One);
13791
13792 SDValue Hi = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Hi0, RHS: Hi1);
13793
13794 SDValue Res = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Lo, Hi});
13795 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
13796}
13797
13798// Catch division cases where we can use shortcuts with rcp and rsq
13799// instructions.
13800SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
13801 SelectionDAG &DAG) const {
13802 SDLoc SL(Op);
13803 SDValue LHS = Op.getOperand(i: 0);
13804 SDValue RHS = Op.getOperand(i: 1);
13805 EVT VT = Op.getValueType();
13806 const SDNodeFlags Flags = Op->getFlags();
13807
13808 bool AllowInaccurateRcp = Flags.hasApproximateFuncs();
13809
13810 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
13811 // Without !fpmath accuracy information, we can't do more because we don't
13812 // know exactly whether rcp is accurate enough to meet !fpmath requirement.
13813 // f16 is always accurate enough
13814 if (!AllowInaccurateRcp && VT != MVT::f16 && VT != MVT::bf16)
13815 return SDValue();
13816
13817 if (CLHS->isOne()) {
13818 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
13819 // the CI documentation has a worst case error of 1 ulp.
13820 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
13821 // use it as long as we aren't trying to use denormals.
13822 //
13823 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
13824
13825 // 1.0 / sqrt(x) -> rsq(x)
13826
13827 // XXX - Is afn sufficient to do this for f64? The maximum ULP
13828 // error seems really high at 2^29 ULP.
13829 // 1.0 / x -> rcp(x)
13830 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13831 }
13832
13833 // Same as for 1.0, but expand the sign out of the constant.
13834 if (CLHS->isMinusOne()) {
13835 // -1.0 / x -> rcp (fneg x)
13836 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
13837 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: FNegRHS);
13838 }
13839 }
13840
13841 // For f16 and bf16 require afn or arcp.
13842 // For f32 require afn.
13843 if (!AllowInaccurateRcp &&
13844 ((VT != MVT::f16 && VT != MVT::bf16) || !Flags.hasAllowReciprocal()))
13845 return SDValue();
13846
13847 // Turn into multiply by the reciprocal.
13848 // x / y -> x * (1.0 / y)
13849 SDValue Recip = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13850 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LHS, N2: Recip, Flags);
13851}
13852
13853SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op,
13854 SelectionDAG &DAG) const {
13855 SDLoc SL(Op);
13856 SDValue X = Op.getOperand(i: 0);
13857 SDValue Y = Op.getOperand(i: 1);
13858 EVT VT = Op.getValueType();
13859 const SDNodeFlags Flags = Op->getFlags();
13860
13861 bool AllowInaccurateDiv = Flags.hasApproximateFuncs();
13862 if (!AllowInaccurateDiv)
13863 return SDValue();
13864
13865 const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: X);
13866 bool IsNegRcp = CLHS && CLHS->isMinusOne();
13867
13868 // Pull out the negation so it folds for free into the source modifiers.
13869 if (IsNegRcp)
13870 X = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13871
13872 SDValue NegY = IsNegRcp ? Y : DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y);
13873 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13874
13875 SDValue R = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: Y);
13876 if (IsNegRcp)
13877 R = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: R);
13878
13879 SDValue Tmp0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13880
13881 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp0, N2: R, N3: R);
13882 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13883 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp1, N2: R, N3: R);
13884
13885 // Skip the last 2 correction terms for reciprocal.
13886 if (IsNegRcp || (CLHS && CLHS->isOne()))
13887 return R;
13888
13889 SDValue Ret = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: R);
13890 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: Ret, N3: X);
13891 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp2, N2: R, N3: Ret);
13892}
13893
13894static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13895 EVT VT, SDValue A, SDValue B, SDValue GlueChain,
13896 SDNodeFlags Flags) {
13897 if (GlueChain->getNumValues() <= 1) {
13898 return DAG.getNode(Opcode, DL: SL, VT, N1: A, N2: B, Flags);
13899 }
13900
13901 assert(GlueChain->getNumValues() == 3);
13902
13903 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13904 switch (Opcode) {
13905 default:
13906 llvm_unreachable("no chain equivalent for opcode");
13907 case ISD::FMUL:
13908 Opcode = AMDGPUISD::FMUL_W_CHAIN;
13909 break;
13910 }
13911
13912 return DAG.getNode(Opcode, DL: SL, VTList,
13913 Ops: {GlueChain.getValue(R: 1), A, B, GlueChain.getValue(R: 2)},
13914 Flags);
13915}
13916
13917static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13918 EVT VT, SDValue A, SDValue B, SDValue C,
13919 SDValue GlueChain, SDNodeFlags Flags) {
13920 if (GlueChain->getNumValues() <= 1) {
13921 return DAG.getNode(Opcode, DL: SL, VT, Ops: {A, B, C}, Flags);
13922 }
13923
13924 assert(GlueChain->getNumValues() == 3);
13925
13926 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13927 switch (Opcode) {
13928 default:
13929 llvm_unreachable("no chain equivalent for opcode");
13930 case ISD::FMA:
13931 Opcode = AMDGPUISD::FMA_W_CHAIN;
13932 break;
13933 }
13934
13935 return DAG.getNode(Opcode, DL: SL, VTList,
13936 Ops: {GlueChain.getValue(R: 1), A, B, C, GlueChain.getValue(R: 2)},
13937 Flags);
13938}
13939
13940SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
13941 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13942 return FastLowered;
13943
13944 SDLoc SL(Op);
13945 EVT VT = Op.getValueType();
13946 SDValue LHS = Op.getOperand(i: 0);
13947 SDValue RHS = Op.getOperand(i: 1);
13948
13949 SDValue LHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: LHS);
13950 SDValue RHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: RHS);
13951
13952 if (VT == MVT::bf16) {
13953 SDValue ExtDiv =
13954 DAG.getNode(Opcode: ISD::FDIV, DL: SL, VT: MVT::f32, N1: LHSExt, N2: RHSExt, Flags: Op->getFlags());
13955 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::bf16, N1: ExtDiv,
13956 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13957 }
13958
13959 assert(VT == MVT::f16);
13960
13961 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
13962 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
13963 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
13964 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
13965 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13966 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
13967 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13968 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
13969 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
13970 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
13971 // q16.u = opx(V_CVT_F16_F32, q32.u);
13972 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
13973
13974 // We will use ISD::FMA on targets that don't support ISD::FMAD.
13975 unsigned FMADOpCode =
13976 isOperationLegal(Op: ISD::FMAD, VT: MVT::f32) ? ISD::FMAD : ISD::FMA;
13977 SDValue NegRHSExt = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHSExt);
13978 SDValue Rcp =
13979 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: RHSExt, Flags: Op->getFlags());
13980 SDValue Quot =
13981 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHSExt, N2: Rcp, Flags: Op->getFlags());
13982 SDValue Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13983 Flags: Op->getFlags());
13984 Quot = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, N3: Quot, Flags: Op->getFlags());
13985 Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13986 Flags: Op->getFlags());
13987 SDValue Tmp = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, Flags: Op->getFlags());
13988 SDValue TmpCast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Tmp);
13989 TmpCast = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TmpCast,
13990 N2: DAG.getConstant(Val: 0xff800000, DL: SL, VT: MVT::i32));
13991 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: TmpCast);
13992 Quot = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f32, N1: Tmp, N2: Quot, Flags: Op->getFlags());
13993 SDValue RDst = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Quot,
13994 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13995 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f16, N1: RDst, N2: RHS, N3: LHS,
13996 Flags: Op->getFlags());
13997}
13998
13999// Faster 2.5 ULP division that does not support denormals.
14000SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
14001 SDNodeFlags Flags = Op->getFlags();
14002 SDLoc SL(Op);
14003 SDValue LHS = Op.getOperand(i: 1);
14004 SDValue RHS = Op.getOperand(i: 2);
14005
14006 // TODO: The combiner should probably handle elimination of redundant fabs.
14007 SDValue r1 = DAG.SignBitIsZeroFP(Op: RHS)
14008 ? RHS
14009 : DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f32, Operand: RHS, Flags);
14010
14011 const APFloat K0Val(0x1p+96f);
14012 const SDValue K0 = DAG.getConstantFP(Val: K0Val, DL: SL, VT: MVT::f32);
14013
14014 const APFloat K1Val(0x1p-32f);
14015 const SDValue K1 = DAG.getConstantFP(Val: K1Val, DL: SL, VT: MVT::f32);
14016
14017 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
14018
14019 EVT SetCCVT =
14020 getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT: MVT::f32);
14021
14022 SDValue r2 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: r1, RHS: K0, Cond: ISD::SETOGT);
14023
14024 SDValue r3 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: r2, N2: K1, N3: One, Flags);
14025
14026 r1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: RHS, N2: r3, Flags);
14027
14028 // rcp does not support denormals.
14029 SDValue r0 = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: r1, Flags);
14030
14031 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHS, N2: r0, Flags);
14032
14033 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: r3, N2: Mul, Flags);
14034}
14035
14036// Returns immediate value for setting the F32 denorm mode when using the
14037// S_DENORM_MODE instruction.
14038static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG,
14039 const SIMachineFunctionInfo *Info,
14040 const GCNSubtarget *ST) {
14041 assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE");
14042 uint32_t DPDenormModeDefault = Info->getMode().fpDenormModeDPValue();
14043 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
14044 return DAG.getTargetConstant(Val: Mode, DL: SDLoc(), VT: MVT::i32);
14045}
14046
14047SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
14048 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
14049 return FastLowered;
14050
14051 // The selection matcher assumes anything with a chain selecting to a
14052 // mayRaiseFPException machine instruction. Since we're introducing a chain
14053 // here, we need to explicitly report nofpexcept for the regular fdiv
14054 // lowering.
14055 SDNodeFlags Flags = Op->getFlags();
14056 Flags.setNoFPExcept(true);
14057
14058 SDLoc SL(Op);
14059 SDValue LHS = Op.getOperand(i: 0);
14060 SDValue RHS = Op.getOperand(i: 1);
14061
14062 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
14063
14064 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f32, VT2: MVT::i1);
14065
14066 SDValue DenominatorScaled =
14067 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {RHS, RHS, LHS}, Flags);
14068 SDValue NumeratorScaled =
14069 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {LHS, RHS, LHS}, Flags);
14070
14071 // Denominator is scaled to not be denormal, so using rcp is ok.
14072 SDValue ApproxRcp =
14073 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
14074 SDValue NegDivScale0 =
14075 DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
14076
14077 using namespace AMDGPU::Hwreg;
14078 const unsigned Denorm32Reg = HwregEncoding::encode(Values: ID_MODE, Values: 4, Values: 2);
14079 const SDValue BitField = DAG.getTargetConstant(Val: Denorm32Reg, DL: SL, VT: MVT::i32);
14080
14081 const MachineFunction &MF = DAG.getMachineFunction();
14082 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
14083 const DenormalMode DenormMode = Info->getMode().FP32Denormals;
14084
14085 const bool PreservesDenormals = DenormMode == DenormalMode::getIEEE();
14086 const bool HasDynamicDenormals =
14087 (DenormMode.Input == DenormalMode::Dynamic) ||
14088 (DenormMode.Output == DenormalMode::Dynamic);
14089
14090 SDValue SavedDenormMode;
14091
14092 if (!PreservesDenormals) {
14093 // Note we can't use the STRICT_FMA/STRICT_FMUL for the non-strict FDIV
14094 // lowering. The chain dependence is insufficient, and we need glue. We do
14095 // not need the glue variants in a strictfp function.
14096
14097 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
14098
14099 SDValue Glue = DAG.getEntryNode();
14100 if (HasDynamicDenormals) {
14101 SDNode *GetReg = DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32, dl: SL,
14102 VTs: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Glue),
14103 Ops: {BitField, Glue});
14104 SavedDenormMode = SDValue(GetReg, 0);
14105
14106 Glue = DAG.getMergeValues(
14107 Ops: {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, dl: SL);
14108 }
14109
14110 SDNode *EnableDenorm;
14111 if (Subtarget->hasDenormModeInst()) {
14112 const SDValue EnableDenormValue =
14113 getSPDenormModeValue(FP_DENORM_FLUSH_NONE, DAG, Info, ST: Subtarget);
14114
14115 EnableDenorm = DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs, N1: Glue,
14116 N2: EnableDenormValue)
14117 .getNode();
14118 } else {
14119 const SDValue EnableDenormValue =
14120 DAG.getConstant(FP_DENORM_FLUSH_NONE, DL: SL, VT: MVT::i32);
14121 EnableDenorm = DAG.getMachineNode(Opcode: AMDGPU::S_SETREG_B32, dl: SL, VTs: BindParamVTs,
14122 Ops: {EnableDenormValue, BitField, Glue});
14123 }
14124
14125 SDValue Ops[3] = {NegDivScale0, SDValue(EnableDenorm, 0),
14126 SDValue(EnableDenorm, 1)};
14127
14128 NegDivScale0 = DAG.getMergeValues(Ops, dl: SL);
14129 }
14130
14131 SDValue Fma0 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0,
14132 B: ApproxRcp, C: One, GlueChain: NegDivScale0, Flags);
14133
14134 SDValue Fma1 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma0, B: ApproxRcp,
14135 C: ApproxRcp, GlueChain: Fma0, Flags);
14136
14137 SDValue Mul = getFPBinOp(DAG, Opcode: ISD::FMUL, SL, VT: MVT::f32, A: NumeratorScaled, B: Fma1,
14138 GlueChain: Fma1, Flags);
14139
14140 SDValue Fma2 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Mul,
14141 C: NumeratorScaled, GlueChain: Mul, Flags);
14142
14143 SDValue Fma3 =
14144 getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma2, B: Fma1, C: Mul, GlueChain: Fma2, Flags);
14145
14146 SDValue Fma4 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Fma3,
14147 C: NumeratorScaled, GlueChain: Fma3, Flags);
14148
14149 if (!PreservesDenormals) {
14150 SDNode *DisableDenorm;
14151 if (!HasDynamicDenormals && Subtarget->hasDenormModeInst()) {
14152 const SDValue DisableDenormValue = getSPDenormModeValue(
14153 FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, Info, ST: Subtarget);
14154
14155 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
14156 DisableDenorm =
14157 DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs,
14158 N1: Fma4.getValue(R: 1), N2: DisableDenormValue, N3: Fma4.getValue(R: 2))
14159 .getNode();
14160 } else {
14161 assert(HasDynamicDenormals == (bool)SavedDenormMode);
14162 const SDValue DisableDenormValue =
14163 HasDynamicDenormals
14164 ? SavedDenormMode
14165 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, DL: SL, VT: MVT::i32);
14166
14167 DisableDenorm = DAG.getMachineNode(
14168 Opcode: AMDGPU::S_SETREG_B32, dl: SL, VT: MVT::Other,
14169 Ops: {DisableDenormValue, BitField, Fma4.getValue(R: 1), Fma4.getValue(R: 2)});
14170 }
14171
14172 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
14173 N1: SDValue(DisableDenorm, 0), N2: DAG.getRoot());
14174 DAG.setRoot(OutputChain);
14175 }
14176
14177 SDValue Scale = NumeratorScaled.getValue(R: 1);
14178 SDValue Fmas = DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f32,
14179 Ops: {Fma4, Fma1, Fma3, Scale}, Flags);
14180
14181 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f32, N1: Fmas, N2: RHS, N3: LHS, Flags);
14182}
14183
14184SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
14185 if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG))
14186 return FastLowered;
14187
14188 SDLoc SL(Op);
14189 SDValue X = Op.getOperand(i: 0);
14190 SDValue Y = Op.getOperand(i: 1);
14191
14192 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
14193
14194 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f64, VT2: MVT::i1);
14195
14196 SDValue DivScale0 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: Y, N2: Y, N3: X);
14197
14198 SDValue NegDivScale0 = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f64, Operand: DivScale0);
14199
14200 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f64, Operand: DivScale0);
14201
14202 SDValue Fma0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Rcp, N3: One);
14203
14204 SDValue Fma1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Rcp, N2: Fma0, N3: Rcp);
14205
14206 SDValue Fma2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Fma1, N3: One);
14207
14208 SDValue DivScale1 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: X, N2: Y, N3: X);
14209
14210 SDValue Fma3 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Fma1, N2: Fma2, N3: Fma1);
14211 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f64, N1: DivScale1, N2: Fma3);
14212
14213 SDValue Fma4 =
14214 DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Mul, N3: DivScale1);
14215
14216 SDValue Scale;
14217
14218 if (!Subtarget->hasUsableDivScaleConditionOutput()) {
14219 // Workaround a hardware bug on SI where the condition output from div_scale
14220 // is not usable.
14221
14222 const SDValue Hi = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
14223
14224 // Figure out if the scale to use for div_fmas.
14225 SDValue NumBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: X);
14226 SDValue DenBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Y);
14227 SDValue Scale0BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale0);
14228 SDValue Scale1BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale1);
14229
14230 SDValue NumHi =
14231 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: NumBC, N2: Hi);
14232 SDValue DenHi =
14233 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: DenBC, N2: Hi);
14234
14235 SDValue Scale0Hi =
14236 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale0BC, N2: Hi);
14237 SDValue Scale1Hi =
14238 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale1BC, N2: Hi);
14239
14240 SDValue CmpDen = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: DenHi, RHS: Scale0Hi, Cond: ISD::SETEQ);
14241 SDValue CmpNum = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: NumHi, RHS: Scale1Hi, Cond: ISD::SETEQ);
14242 Scale = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: CmpNum, N2: CmpDen);
14243 } else {
14244 Scale = DivScale1.getValue(R: 1);
14245 }
14246
14247 SDValue Fmas =
14248 DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f64, N1: Fma4, N2: Fma3, N3: Mul, N4: Scale);
14249
14250 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f64, N1: Fmas, N2: Y, N3: X);
14251}
14252
14253SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
14254 EVT VT = Op.getValueType();
14255
14256 if (VT == MVT::f32)
14257 return LowerFDIV32(Op, DAG);
14258
14259 if (VT == MVT::f64)
14260 return LowerFDIV64(Op, DAG);
14261
14262 if (VT == MVT::f16 || VT == MVT::bf16)
14263 return LowerFDIV16(Op, DAG);
14264
14265 llvm_unreachable("Unexpected type for fdiv");
14266}
14267
14268SDValue SITargetLowering::LowerFFREXP(SDValue Op, SelectionDAG &DAG) const {
14269 SDLoc dl(Op);
14270 SDValue Val = Op.getOperand(i: 0);
14271 EVT VT = Val.getValueType();
14272 EVT ResultExpVT = Op->getValueType(ResNo: 1);
14273 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
14274
14275 SDValue Mant = DAG.getNode(
14276 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT,
14277 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_mant, DL: dl, VT: MVT::i32), N2: Val);
14278
14279 SDValue Exp = DAG.getNode(
14280 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: InstrExpVT,
14281 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_exp, DL: dl, VT: MVT::i32), N2: Val);
14282
14283 if (Subtarget->hasFractBug()) {
14284 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val);
14285 SDValue Inf =
14286 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
14287
14288 SDValue IsFinite = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Fabs, RHS: Inf, Cond: ISD::SETOLT);
14289 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: InstrExpVT);
14290 Exp = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InstrExpVT, N1: IsFinite, N2: Exp, N3: Zero);
14291 Mant = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: IsFinite, N2: Mant, N3: Val);
14292 }
14293
14294 SDValue CastExp = DAG.getSExtOrTrunc(Op: Exp, DL: dl, VT: ResultExpVT);
14295 return DAG.getMergeValues(Ops: {Mant, CastExp}, dl);
14296}
14297
14298SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
14299 SDLoc DL(Op);
14300 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
14301 EVT VT = Store->getMemoryVT();
14302
14303 if (VT == MVT::i1) {
14304 return DAG.getTruncStore(
14305 Chain: Store->getChain(), dl: DL,
14306 Val: DAG.getSExtOrTrunc(Op: Store->getValue(), DL, VT: MVT::i32),
14307 Ptr: Store->getBasePtr(), SVT: MVT::i1, MMO: Store->getMemOperand());
14308 }
14309
14310 assert(VT.isVector() &&
14311 Store->getValue().getValueType().getScalarType() == MVT::i32);
14312
14313 unsigned AS = Store->getAddressSpace();
14314 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
14315 AS == AMDGPUAS::FLAT_ADDRESS &&
14316 Store->getAlign().value() < VT.getStoreSize() &&
14317 VT.getSizeInBits() > 32) {
14318 return SplitVectorStore(Op, DAG);
14319 }
14320
14321 MachineFunction &MF = DAG.getMachineFunction();
14322 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
14323 // If there is a possibility that flat instruction access scratch memory
14324 // then we need to use the same legalization rules we use for private.
14325 if (AS == AMDGPUAS::FLAT_ADDRESS &&
14326 !Subtarget->hasMultiDwordFlatScratchAddressing())
14327 AS = addressMayBeAccessedAsPrivate(MMO: Store->getMemOperand(), Info: *MFI)
14328 ? AMDGPUAS::PRIVATE_ADDRESS
14329 : AMDGPUAS::GLOBAL_ADDRESS;
14330
14331 unsigned NumElements = VT.getVectorNumElements();
14332 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
14333 if (NumElements > 4)
14334 return SplitVectorStore(Op, DAG);
14335 // v3 stores not supported on SI.
14336 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
14337 return SplitVectorStore(Op, DAG);
14338
14339 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
14340 VT, MMO: *Store->getMemOperand()))
14341 return expandUnalignedStore(ST: Store, DAG);
14342
14343 return SDValue();
14344 }
14345 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
14346 switch (Subtarget->getMaxPrivateElementSize()) {
14347 case 4:
14348 return scalarizeVectorStore(ST: Store, DAG);
14349 case 8:
14350 if (NumElements > 2)
14351 return SplitVectorStore(Op, DAG);
14352 return SDValue();
14353 case 16:
14354 if (NumElements > 4 ||
14355 (NumElements == 3 && !Subtarget->hasFlatScratchEnabled()))
14356 return SplitVectorStore(Op, DAG);
14357 return SDValue();
14358 default:
14359 llvm_unreachable("unsupported private_element_size");
14360 }
14361 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
14362 unsigned Fast = 0;
14363 auto Flags = Store->getMemOperand()->getFlags();
14364 if (allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace: AS,
14365 Alignment: Store->getAlign(), Flags, IsFast: &Fast) &&
14366 Fast > 1)
14367 return SDValue();
14368
14369 if (VT.isVector())
14370 return SplitVectorStore(Op, DAG);
14371
14372 return expandUnalignedStore(ST: Store, DAG);
14373 }
14374
14375 // Probably an invalid store. If so we'll end up emitting a selection error.
14376 return SDValue();
14377}
14378
14379// Avoid the full correct expansion for f32 sqrt when promoting from f16.
14380SDValue SITargetLowering::lowerFSQRTF16(SDValue Op, SelectionDAG &DAG) const {
14381 SDLoc SL(Op);
14382 assert(!Subtarget->has16BitInsts());
14383 SDNodeFlags Flags = Op->getFlags();
14384 SDValue Ext =
14385 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op.getOperand(i: 0), Flags);
14386
14387 SDValue SqrtID = DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL: SL, VT: MVT::i32);
14388 SDValue Sqrt =
14389 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::f32, N1: SqrtID, N2: Ext, Flags);
14390
14391 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Sqrt,
14392 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
14393}
14394
14395SDValue SITargetLowering::lowerFSQRTF32(SDValue Op, SelectionDAG &DAG) const {
14396 SDLoc DL(Op);
14397 SDNodeFlags Flags = Op->getFlags();
14398 MVT VT = Op.getValueType().getSimpleVT();
14399 const SDValue X = Op.getOperand(i: 0);
14400
14401 if (allowApproxFunc(DAG, Flags)) {
14402 // Instruction is 1ulp but ignores denormals.
14403 return DAG.getNode(
14404 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
14405 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32), N2: X, Flags);
14406 }
14407
14408 SDValue ScaleThreshold = DAG.getConstantFP(Val: 0x1.0p-96f, DL, VT);
14409 SDValue NeedScale = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleThreshold, Cond: ISD::SETOLT);
14410
14411 SDValue ScaleUpFactor = DAG.getConstantFP(Val: 0x1.0p+32f, DL, VT);
14412
14413 SDValue ScaledX = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: ScaleUpFactor, Flags);
14414
14415 SDValue SqrtX =
14416 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledX, N3: X, Flags);
14417
14418 SDValue SqrtS;
14419 if (needsDenormHandlingF32(DAG, Src: X, Flags)) {
14420 SDValue SqrtID =
14421 DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32);
14422 SqrtS = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: SqrtID, N2: SqrtX, Flags);
14423
14424 SDValue SqrtSAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: SqrtS);
14425 SDValue SqrtSNextDownInt =
14426 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14427 N2: DAG.getAllOnesConstant(DL, VT: MVT::i32));
14428 SDValue SqrtSNextDown = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextDownInt);
14429
14430 SDValue NegSqrtSNextDown =
14431 DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextDown, Flags);
14432
14433 SDValue SqrtVP =
14434 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextDown, N2: SqrtS, N3: SqrtX, Flags);
14435
14436 SDValue SqrtSNextUpInt = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14437 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
14438 SDValue SqrtSNextUp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextUpInt);
14439
14440 SDValue NegSqrtSNextUp = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextUp, Flags);
14441 SDValue SqrtVS =
14442 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextUp, N2: SqrtS, N3: SqrtX, Flags);
14443
14444 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
14445 SDValue SqrtVPLE0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVP, RHS: Zero, Cond: ISD::SETOLE);
14446
14447 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPLE0, N2: SqrtSNextDown, N3: SqrtS,
14448 Flags);
14449
14450 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVS, RHS: Zero, Cond: ISD::SETOGT);
14451 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPVSGT0, N2: SqrtSNextUp, N3: SqrtS,
14452 Flags);
14453 } else {
14454 SDValue SqrtR = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: SqrtX, Flags);
14455
14456 SqrtS = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtX, N2: SqrtR, Flags);
14457
14458 SDValue Half = DAG.getConstantFP(Val: 0.5f, DL, VT);
14459 SDValue SqrtH = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtR, N2: Half, Flags);
14460 SDValue NegSqrtH = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtH, Flags);
14461
14462 SDValue SqrtE = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtH, N2: SqrtS, N3: Half, Flags);
14463 SqrtH = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtH, N2: SqrtE, N3: SqrtH, Flags);
14464 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtS, N2: SqrtE, N3: SqrtS, Flags);
14465
14466 SDValue NegSqrtS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtS, Flags);
14467 SDValue SqrtD =
14468 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtS, N2: SqrtS, N3: SqrtX, Flags);
14469 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtD, N2: SqrtH, N3: SqrtS, Flags);
14470 }
14471
14472 SDValue ScaleDownFactor = DAG.getConstantFP(Val: 0x1.0p-16f, DL, VT);
14473
14474 SDValue ScaledDown =
14475 DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtS, N2: ScaleDownFactor, Flags);
14476
14477 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledDown, N3: SqrtS, Flags);
14478 SDValue IsZeroOrInf =
14479 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14480 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14481
14482 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtS, Flags);
14483}
14484
14485SDValue SITargetLowering::lowerFSQRTF64(SDValue Op, SelectionDAG &DAG) const {
14486 // For double type, the SQRT and RSQ instructions don't have required
14487 // precision, we apply Goldschmidt's algorithm to improve the result:
14488 //
14489 // y0 = rsq(x)
14490 // g0 = x * y0
14491 // h0 = 0.5 * y0
14492 //
14493 // r0 = 0.5 - h0 * g0
14494 // g1 = g0 * r0 + g0
14495 // h1 = h0 * r0 + h0
14496 //
14497 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
14498 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
14499 // h2 = h1 * r1 + h1
14500 //
14501 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
14502 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
14503 //
14504 // sqrt(x) = g3
14505
14506 SDNodeFlags Flags = Op->getFlags();
14507
14508 SDLoc DL(Op);
14509
14510 SDValue X = Op.getOperand(i: 0);
14511 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
14512
14513 SDValue SqrtX = X;
14514 SDValue Scaling;
14515 if (!Flags.hasApproximateFuncs()) {
14516 SDValue ScaleConstant = DAG.getConstantFP(Val: 0x1.0p-767, DL, VT: MVT::f64);
14517 Scaling = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleConstant, Cond: ISD::SETOLT);
14518
14519 // Scale up input if it is too small.
14520 SDValue ScaleUpFactor = DAG.getConstant(Val: 256, DL, VT: MVT::i32);
14521 SDValue ScaleUp =
14522 DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling, N2: ScaleUpFactor, N3: ZeroInt);
14523 SqrtX = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: X, N2: ScaleUp, Flags);
14524 }
14525
14526 SDValue SqrtY = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT: MVT::f64, Operand: SqrtX);
14527
14528 SDValue SqrtS0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtX, N2: SqrtY);
14529
14530 SDValue Half = DAG.getConstantFP(Val: 0.5, DL, VT: MVT::f64);
14531 SDValue SqrtH0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtY, N2: Half);
14532
14533 SDValue NegSqrtH0 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtH0);
14534 SDValue SqrtR0 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtH0, N2: SqrtS0, N3: Half);
14535
14536 SDValue SqrtH1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtH0, N2: SqrtR0, N3: SqrtH0);
14537
14538 SDValue SqrtS1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtS0, N2: SqrtR0, N3: SqrtS0);
14539
14540 SDValue NegSqrtS1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS1);
14541 SDValue SqrtD0 =
14542 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS1, N2: SqrtS1, N3: SqrtX);
14543
14544 SDValue SqrtS2 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD0, N2: SqrtH1, N3: SqrtS1);
14545
14546 SDValue SqrtRet = SqrtS2;
14547 if (!Flags.hasApproximateFuncs()) {
14548 SDValue NegSqrtS2 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS2);
14549 SDValue SqrtD1 =
14550 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS2, N2: SqrtS2, N3: SqrtX);
14551
14552 SqrtRet = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD1, N2: SqrtH1, N3: SqrtS2);
14553
14554 SDValue ScaleDownFactor = DAG.getSignedConstant(Val: -128, DL, VT: MVT::i32);
14555 SDValue ScaleDown = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling,
14556 N2: ScaleDownFactor, N3: ZeroInt);
14557 SqrtRet = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: SqrtRet, N2: ScaleDown, Flags);
14558 }
14559
14560 // TODO: Check for DAZ and expand to subnormals
14561
14562 SDValue IsZeroOrInf;
14563 if (Flags.hasNoInfs()) {
14564 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
14565 IsZeroOrInf = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtX, RHS: Zero, Cond: ISD::SETOEQ);
14566 } else {
14567 IsZeroOrInf =
14568 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14569 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14570 }
14571
14572 // If x is +INF, +0, or -0, use its original value
14573 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f64, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtRet,
14574 Flags);
14575}
14576
14577SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
14578 SDLoc DL(Op);
14579 EVT VT = Op.getValueType();
14580 SDValue Arg = Op.getOperand(i: 0);
14581 SDValue TrigVal;
14582
14583 // Propagate fast-math flags so that the multiply we introduce can be folded
14584 // if Arg is already the result of a multiply by constant.
14585 auto Flags = Op->getFlags();
14586
14587 // AMDGPUISD nodes of vector type must be unrolled here since
14588 // they will not be expanded elsewhere.
14589 auto UnrollIfVec = [&DAG](SDValue V) -> SDValue {
14590 if (!V.getValueType().isVector())
14591 return V;
14592
14593 return DAG.UnrollVectorOp(N: cast<SDNode>(Val&: V));
14594 };
14595
14596 SDValue OneOver2Pi = DAG.getConstantFP(Val: 0.5 * numbers::inv_pi, DL, VT);
14597
14598 if (Subtarget->hasTrigReducedRange()) {
14599 SDValue MulVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14600 TrigVal = UnrollIfVec(DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: MulVal, Flags));
14601 } else {
14602 TrigVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14603 }
14604
14605 switch (Op.getOpcode()) {
14606 case ISD::FCOS:
14607 TrigVal = DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14608 break;
14609 case ISD::FSIN:
14610 TrigVal = DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14611 break;
14612 default:
14613 llvm_unreachable("Wrong trig opcode");
14614 }
14615
14616 return UnrollIfVec(TrigVal);
14617}
14618
14619SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
14620 SelectionDAG &DAG) const {
14621 AtomicSDNode *AtomicNode = cast<AtomicSDNode>(Val&: Op);
14622 assert(AtomicNode->isCompareAndSwap());
14623 unsigned AS = AtomicNode->getAddressSpace();
14624
14625 // No custom lowering required for local address space
14626 if (!AMDGPU::isFlatGlobalAddrSpace(AS))
14627 return Op;
14628
14629 // Non-local address space requires custom lowering for atomic compare
14630 // and swap; cmp and swap should be in a v2i32 or v2i64 in case of _X2
14631 SDLoc DL(Op);
14632 SDValue ChainIn = Op.getOperand(i: 0);
14633 SDValue Addr = Op.getOperand(i: 1);
14634 SDValue Old = Op.getOperand(i: 2);
14635 SDValue New = Op.getOperand(i: 3);
14636 EVT VT = Op.getValueType();
14637 MVT SimpleVT = VT.getSimpleVT();
14638 MVT VecType = MVT::getVectorVT(VT: SimpleVT, NumElements: 2);
14639
14640 SDValue NewOld = DAG.getBuildVector(VT: VecType, DL, Ops: {New, Old});
14641 SDValue Ops[] = {ChainIn, Addr, NewOld};
14642
14643 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::ATOMIC_CMP_SWAP, dl: DL,
14644 VTList: Op->getVTList(), Ops, MemVT: VT,
14645 MMO: AtomicNode->getMemOperand());
14646}
14647
14648//===----------------------------------------------------------------------===//
14649// Custom DAG optimizations
14650//===----------------------------------------------------------------------===//
14651
14652SDValue
14653SITargetLowering::performUCharToFloatCombine(SDNode *N,
14654 DAGCombinerInfo &DCI) const {
14655 EVT VT = N->getValueType(ResNo: 0);
14656 EVT ScalarVT = VT.getScalarType();
14657 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
14658 return SDValue();
14659
14660 SelectionDAG &DAG = DCI.DAG;
14661 SDLoc DL(N);
14662
14663 SDValue Src = N->getOperand(Num: 0);
14664 EVT SrcVT = Src.getValueType();
14665
14666 // TODO: We could try to match extracting the higher bytes, which would be
14667 // easier if i8 vectors weren't promoted to i32 vectors, particularly after
14668 // types are legalized. v4i8 -> v4f32 is probably the only case to worry
14669 // about in practice.
14670 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
14671 if (DAG.MaskedValueIsZero(Op: Src, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 24))) {
14672 SDValue Cvt = DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0, DL, VT: MVT::f32, Operand: Src);
14673 DCI.AddToWorklist(N: Cvt.getNode());
14674
14675 // For the f16 case, fold to a cast to f32 and then cast back to f16.
14676 if (ScalarVT != MVT::f32) {
14677 Cvt = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Cvt,
14678 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
14679 }
14680 return Cvt;
14681 }
14682 }
14683
14684 return SDValue();
14685}
14686
14687SDValue SITargetLowering::performFCopySignCombine(SDNode *N,
14688 DAGCombinerInfo &DCI) const {
14689 SDValue MagnitudeOp = N->getOperand(Num: 0);
14690 SDValue SignOp = N->getOperand(Num: 1);
14691
14692 // The generic combine for fcopysign + fp cast is too conservative with
14693 // vectors, and also gets confused by the splitting we will perform here, so
14694 // peek through FP casts.
14695 if (SignOp.getOpcode() == ISD::FP_EXTEND ||
14696 SignOp.getOpcode() == ISD::FP_ROUND)
14697 SignOp = SignOp.getOperand(i: 0);
14698
14699 SelectionDAG &DAG = DCI.DAG;
14700 SDLoc DL(N);
14701 EVT SignVT = SignOp.getValueType();
14702
14703 // f64 fcopysign is really an f32 copysign on the high bits, so replace the
14704 // lower half with a copy.
14705 // fcopysign f64:x, _:y -> x.lo32, (fcopysign (f32 x.hi32), _:y)
14706 EVT MagVT = MagnitudeOp.getValueType();
14707
14708 unsigned NumElts = MagVT.isVector() ? MagVT.getVectorNumElements() : 1;
14709
14710 if (MagVT.getScalarType() == MVT::f64) {
14711 EVT F32VT = MagVT.isVector()
14712 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14713 : MVT::v2f32;
14714
14715 SDValue MagAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: MagnitudeOp);
14716
14717 SmallVector<SDValue, 8> NewElts;
14718 for (unsigned I = 0; I != NumElts; ++I) {
14719 SDValue MagLo =
14720 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14721 N2: DAG.getConstant(Val: 2 * I, DL, VT: MVT::i32));
14722 SDValue MagHi =
14723 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14724 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14725
14726 SDValue SignOpElt =
14727 MagVT.isVector()
14728 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SignVT.getScalarType(),
14729 N1: SignOp, N2: DAG.getConstant(Val: I, DL, VT: MVT::i32))
14730 : SignOp;
14731
14732 SDValue HiOp =
14733 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: MVT::f32, N1: MagHi, N2: SignOpElt);
14734
14735 SDValue Vector =
14736 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2f32, N1: MagLo, N2: HiOp);
14737
14738 SDValue NewElt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Vector);
14739 NewElts.push_back(Elt: NewElt);
14740 }
14741
14742 if (NewElts.size() == 1)
14743 return NewElts[0];
14744
14745 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MagVT, Ops: NewElts);
14746 }
14747
14748 if (SignVT.getScalarType() != MVT::f64)
14749 return SDValue();
14750
14751 // Reduce width of sign operand, we only need the highest bit.
14752 //
14753 // fcopysign f64:x, f64:y ->
14754 // fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1)
14755 // TODO: In some cases it might make sense to go all the way to f16.
14756
14757 EVT F32VT = MagVT.isVector()
14758 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14759 : MVT::v2f32;
14760
14761 SDValue SignAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: SignOp);
14762
14763 SmallVector<SDValue, 8> F32Signs;
14764 for (unsigned I = 0; I != NumElts; ++I) {
14765 // Take sign from odd elements of cast vector
14766 SDValue SignAsF32 =
14767 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: SignAsVector,
14768 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14769 F32Signs.push_back(Elt: SignAsF32);
14770 }
14771
14772 SDValue NewSign =
14773 NumElts == 1
14774 ? F32Signs.back()
14775 : DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL,
14776 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: NumElts),
14777 Ops: F32Signs);
14778
14779 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
14780 N2: NewSign);
14781}
14782
14783// (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2)
14784// (shl (or x, c1), c2) -> add (shl x, c2), (shl c1, c2) iff x and c1 share no
14785// bits
14786
14787// This is a variant of
14788// (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2),
14789//
14790// The normal DAG combiner will do this, but only if the add has one use since
14791// that would increase the number of instructions.
14792//
14793// This prevents us from seeing a constant offset that can be folded into a
14794// memory instruction's addressing mode. If we know the resulting add offset of
14795// a pointer can be folded into an addressing offset, we can replace the pointer
14796// operand with the add of new constant offset. This eliminates one of the uses,
14797// and may allow the remaining use to also be simplified.
14798//
14799SDValue SITargetLowering::performSHLPtrCombine(SDNode *N, unsigned AddrSpace,
14800 EVT MemVT,
14801 DAGCombinerInfo &DCI) const {
14802 SDValue N0 = N->getOperand(Num: 0);
14803 SDValue N1 = N->getOperand(Num: 1);
14804
14805 // We only do this to handle cases where it's profitable when there are
14806 // multiple uses of the add, so defer to the standard combine.
14807 if ((!N0->isAnyAdd() && N0.getOpcode() != ISD::OR) || N0->hasOneUse())
14808 return SDValue();
14809
14810 const ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val&: N1);
14811 if (!CN1)
14812 return SDValue();
14813
14814 const ConstantSDNode *CAdd = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
14815 if (!CAdd)
14816 return SDValue();
14817
14818 SelectionDAG &DAG = DCI.DAG;
14819
14820 if (N0->getOpcode() == ISD::OR &&
14821 !DAG.haveNoCommonBitsSet(A: N0.getOperand(i: 0), B: N0.getOperand(i: 1)))
14822 return SDValue();
14823
14824 // If the resulting offset is too large, we can't fold it into the
14825 // addressing mode offset.
14826 APInt Offset = CAdd->getAPIntValue() << CN1->getAPIntValue();
14827 Type *Ty = MemVT.getTypeForEVT(Context&: *DCI.DAG.getContext());
14828
14829 AddrMode AM;
14830 AM.HasBaseReg = true;
14831 AM.BaseOffs = Offset.getSExtValue();
14832 if (!isLegalAddressingMode(DL: DCI.DAG.getDataLayout(), AM, Ty, AS: AddrSpace))
14833 return SDValue();
14834
14835 SDLoc SL(N);
14836 EVT VT = N->getValueType(ResNo: 0);
14837
14838 SDValue ShlX = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: N0.getOperand(i: 0), N2: N1);
14839 SDValue COffset = DAG.getConstant(Val: Offset, DL: SL, VT);
14840
14841 SDNodeFlags Flags;
14842 Flags.setNoUnsignedWrap(
14843 N->getFlags().hasNoUnsignedWrap() &&
14844 (N0.getOpcode() == ISD::OR || N0->getFlags().hasNoUnsignedWrap()));
14845
14846 // Use ISD::ADD even if the original operation was ISD::PTRADD, since we can't
14847 // be sure that the new left operand is a proper base pointer.
14848 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ShlX, N2: COffset, Flags);
14849}
14850
14851/// MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset
14852/// by the chain and intrinsic ID. Theoretically we would also need to check the
14853/// specific intrinsic, but they all place the pointer operand first.
14854static unsigned getBasePtrIndex(const MemSDNode *N) {
14855 switch (N->getOpcode()) {
14856 case ISD::STORE:
14857 case ISD::INTRINSIC_W_CHAIN:
14858 case ISD::INTRINSIC_VOID:
14859 return 2;
14860 default:
14861 return 1;
14862 }
14863}
14864
14865SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
14866 DAGCombinerInfo &DCI) const {
14867 SelectionDAG &DAG = DCI.DAG;
14868
14869 unsigned PtrIdx = getBasePtrIndex(N);
14870 SDValue Ptr = N->getOperand(Num: PtrIdx);
14871
14872 // TODO: We could also do this for multiplies.
14873 if (Ptr.getOpcode() == ISD::SHL) {
14874 SDValue NewPtr = performSHLPtrCombine(N: Ptr.getNode(), AddrSpace: N->getAddressSpace(),
14875 MemVT: N->getMemoryVT(), DCI);
14876 if (NewPtr) {
14877 SmallVector<SDValue, 8> NewOps(N->ops());
14878
14879 NewOps[PtrIdx] = NewPtr;
14880 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
14881 }
14882 }
14883
14884 return SDValue();
14885}
14886
14887static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val) {
14888 return (Opc == ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
14889 (Opc == ISD::OR && (Val == 0xffffffff || Val == 0)) ||
14890 (Opc == ISD::XOR && Val == 0);
14891}
14892
14893// Break up 64-bit bit operation of a constant into two 32-bit and/or/xor. This
14894// will typically happen anyway for a VALU 64-bit and. This exposes other 32-bit
14895// integer combine opportunities since most 64-bit operations are decomposed
14896// this way. TODO: We won't want this for SALU especially if it is an inline
14897// immediate.
14898SDValue SITargetLowering::splitBinaryBitConstantOp(
14899 DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
14900 const ConstantSDNode *CRHS) const {
14901 uint64_t Val = CRHS->getZExtValue();
14902 uint32_t ValLo = Lo_32(Value: Val);
14903 uint32_t ValHi = Hi_32(Value: Val);
14904 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14905
14906 if ((bitOpWithConstantIsReducible(Opc, Val: ValLo) ||
14907 bitOpWithConstantIsReducible(Opc, Val: ValHi)) ||
14908 (CRHS->hasOneUse() && !TII->isInlineConstant(Imm: CRHS->getAPIntValue()))) {
14909 // We have 64-bit scalar and/or/xor, but do not have vector forms.
14910 if (Subtarget->has64BitLiterals() && CRHS->hasOneUse() &&
14911 !CRHS->user_begin()->isDivergent())
14912 return SDValue();
14913
14914 // If we need to materialize a 64-bit immediate, it will be split up later
14915 // anyway. Avoid creating the harder to understand 64-bit immediate
14916 // materialization.
14917 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
14918 }
14919
14920 return SDValue();
14921}
14922
14923bool llvm::isBoolSGPR(SDValue V) {
14924 if (V.getValueType() != MVT::i1)
14925 return false;
14926 switch (V.getOpcode()) {
14927 default:
14928 break;
14929 case ISD::SETCC:
14930 case ISD::IS_FPCLASS:
14931 case AMDGPUISD::FP_CLASS:
14932 return true;
14933 case ISD::AND:
14934 case ISD::OR:
14935 case ISD::XOR:
14936 return isBoolSGPR(V: V.getOperand(i: 0)) && isBoolSGPR(V: V.getOperand(i: 1));
14937 case ISD::SADDO:
14938 case ISD::UADDO:
14939 case ISD::SSUBO:
14940 case ISD::USUBO:
14941 case ISD::SMULO:
14942 case ISD::UMULO:
14943 return V.getResNo() == 1;
14944 case ISD::INTRINSIC_WO_CHAIN: {
14945 unsigned IntrinsicID = V.getConstantOperandVal(i: 0);
14946 switch (IntrinsicID) {
14947 case Intrinsic::amdgcn_is_shared:
14948 case Intrinsic::amdgcn_is_private:
14949 return true;
14950 default:
14951 return false;
14952 }
14953
14954 return false;
14955 }
14956 }
14957 return false;
14958}
14959
14960// If a constant has all zeroes or all ones within each byte return it.
14961// Otherwise return 0.
14962static uint32_t getConstantPermuteMask(uint32_t C) {
14963 // 0xff for any zero byte in the mask
14964 uint32_t ZeroByteMask = 0;
14965 if (!(C & 0x000000ff))
14966 ZeroByteMask |= 0x000000ff;
14967 if (!(C & 0x0000ff00))
14968 ZeroByteMask |= 0x0000ff00;
14969 if (!(C & 0x00ff0000))
14970 ZeroByteMask |= 0x00ff0000;
14971 if (!(C & 0xff000000))
14972 ZeroByteMask |= 0xff000000;
14973 uint32_t NonZeroByteMask = ~ZeroByteMask; // 0xff for any non-zero byte
14974 if ((NonZeroByteMask & C) != NonZeroByteMask)
14975 return 0; // Partial bytes selected.
14976 return C;
14977}
14978
14979// Check if a node selects whole bytes from its operand 0 starting at a byte
14980// boundary while masking the rest. Returns select mask as in the v_perm_b32
14981// or -1 if not succeeded.
14982// Note byte select encoding:
14983// value 0-3 selects corresponding source byte;
14984// value 0xc selects zero;
14985// value 0xff selects 0xff.
14986static uint32_t getPermuteMask(SDValue V) {
14987 assert(V.getValueSizeInBits() == 32);
14988
14989 if (V.getNumOperands() != 2)
14990 return ~0;
14991
14992 ConstantSDNode *N1 = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
14993 if (!N1)
14994 return ~0;
14995
14996 uint32_t C = N1->getZExtValue();
14997
14998 switch (V.getOpcode()) {
14999 default:
15000 break;
15001 case ISD::AND:
15002 if (uint32_t ConstMask = getConstantPermuteMask(C))
15003 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
15004 break;
15005
15006 case ISD::OR:
15007 if (uint32_t ConstMask = getConstantPermuteMask(C))
15008 return (0x03020100 & ~ConstMask) | ConstMask;
15009 break;
15010
15011 case ISD::SHL:
15012 if (C % 8)
15013 return ~0;
15014
15015 return uint32_t((0x030201000c0c0c0cull << C) >> 32);
15016
15017 case ISD::SRL:
15018 if (C % 8)
15019 return ~0;
15020
15021 return uint32_t(0x0c0c0c0c03020100ull >> C);
15022 }
15023
15024 return ~0;
15025}
15026
15027SDValue SITargetLowering::performAndCombine(SDNode *N,
15028 DAGCombinerInfo &DCI) const {
15029 if (DCI.isBeforeLegalize())
15030 return SDValue();
15031
15032 SelectionDAG &DAG = DCI.DAG;
15033 EVT VT = N->getValueType(ResNo: 0);
15034 SDValue LHS = N->getOperand(Num: 0);
15035 SDValue RHS = N->getOperand(Num: 1);
15036
15037 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
15038 if (VT == MVT::i64 && CRHS) {
15039 if (SDValue Split =
15040 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::AND, LHS, CRHS))
15041 return Split;
15042 }
15043
15044 if (CRHS && VT == MVT::i32) {
15045 // and (srl x, c), mask => shl (bfe x, nb + c, mask >> nb), nb
15046 // nb = number of trailing zeroes in mask
15047 // It can be optimized out using SDWA for GFX8+ in the SDWA peephole pass,
15048 // given that we are selecting 8 or 16 bit fields starting at byte boundary.
15049 uint64_t Mask = CRHS->getZExtValue();
15050 unsigned Bits = llvm::popcount(Value: Mask);
15051 if (getSubtarget()->hasSDWA() && LHS->getOpcode() == ISD::SRL &&
15052 (Bits == 8 || Bits == 16) && isShiftedMask_64(Value: Mask) && !(Mask & 1)) {
15053 if (auto *CShift = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1))) {
15054 unsigned Shift = CShift->getZExtValue();
15055 unsigned NB = CRHS->getAPIntValue().countr_zero();
15056 unsigned Offset = NB + Shift;
15057 if ((Offset & (Bits - 1)) == 0) { // Starts at a byte or word boundary.
15058 SDLoc SL(N);
15059 SDValue BFE =
15060 DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32, N1: LHS->getOperand(Num: 0),
15061 N2: DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32),
15062 N3: DAG.getConstant(Val: Bits, DL: SL, VT: MVT::i32));
15063 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
15064 SDValue Ext = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT, N1: BFE,
15065 N2: DAG.getValueType(NarrowVT));
15066 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(LHS), VT, N1: Ext,
15067 N2: DAG.getConstant(Val: NB, DL: SDLoc(CRHS), VT: MVT::i32));
15068 return Shl;
15069 }
15070 }
15071 }
15072
15073 // and (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15074 if (LHS.hasOneUse() && LHS.getOpcode() == AMDGPUISD::PERM &&
15075 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
15076 uint32_t Sel = getConstantPermuteMask(C: Mask);
15077 if (!Sel)
15078 return SDValue();
15079
15080 // Select 0xc for all zero bytes
15081 Sel = (LHS.getConstantOperandVal(i: 2) & Sel) | (~Sel & 0x0c0c0c0c);
15082 SDLoc DL(N);
15083 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15084 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15085 }
15086 }
15087
15088 // and (trunc (srl x, c)), mask => srl (trunc x), c
15089 // if mask keeps exactly the bits that srl leaves in the low half
15090 if (CRHS && VT == MVT::i16 && LHS.getOpcode() == ISD::TRUNCATE &&
15091 LHS.hasOneUse() && LHS.getOperand(i: 0).getOpcode() == ISD::SRL) {
15092 auto *Shift = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 0).getOperand(i: 1));
15093 if (Shift && Shift->getZExtValue() < 16 &&
15094 CRHS->getAPIntValue().isMask(numBits: 16 - Shift->getZExtValue())) {
15095 SDLoc SL(N);
15096 SDValue Trunc =
15097 DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: LHS.getOperand(i: 0).getOperand(i: 0));
15098 return DAG.getNode(
15099 Opcode: ISD::SRL, DL: SL, VT, N1: Trunc,
15100 N2: DAG.getShiftAmountConstant(Val: Shift->getZExtValue(), VT, DL: SL));
15101 }
15102 }
15103
15104 // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
15105 // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
15106 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
15107 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
15108 ISD::CondCode RCC = cast<CondCodeSDNode>(Val: RHS.getOperand(i: 2))->get();
15109
15110 SDValue X = LHS.getOperand(i: 0);
15111 SDValue Y = RHS.getOperand(i: 0);
15112 if (Y.getOpcode() != ISD::FABS || Y.getOperand(i: 0) != X ||
15113 !isTypeLegal(VT: X.getValueType()))
15114 return SDValue();
15115
15116 if (LCC == ISD::SETO) {
15117 if (X != LHS.getOperand(i: 1))
15118 return SDValue();
15119
15120 if (RCC == ISD::SETUNE) {
15121 const ConstantFPSDNode *C1 =
15122 dyn_cast<ConstantFPSDNode>(Val: RHS.getOperand(i: 1));
15123 if (!C1 || !C1->isInfinity() || C1->isNegative())
15124 return SDValue();
15125
15126 const uint32_t Mask = SIInstrFlags::N_NORMAL |
15127 SIInstrFlags::N_SUBNORMAL | SIInstrFlags::N_ZERO |
15128 SIInstrFlags::P_ZERO | SIInstrFlags::P_SUBNORMAL |
15129 SIInstrFlags::P_NORMAL;
15130
15131 static_assert(
15132 ((~(SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN |
15133 SIInstrFlags::N_INFINITY | SIInstrFlags::P_INFINITY)) &
15134 0x3ff) == Mask,
15135 "mask not equal");
15136
15137 SDLoc DL(N);
15138 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: X,
15139 N2: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
15140 }
15141 }
15142 }
15143
15144 if (RHS.getOpcode() == ISD::SETCC && LHS.getOpcode() == AMDGPUISD::FP_CLASS)
15145 std::swap(a&: LHS, b&: RHS);
15146
15147 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15148 RHS.hasOneUse()) {
15149 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
15150 // and (fcmp seto), (fp_class x, mask) -> fp_class x, mask & ~(p_nan |
15151 // n_nan) and (fcmp setuo), (fp_class x, mask) -> fp_class x, mask & (p_nan
15152 // | n_nan)
15153 const ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
15154 if ((LCC == ISD::SETO || LCC == ISD::SETUO) && Mask &&
15155 (RHS.getOperand(i: 0) == LHS.getOperand(i: 0) &&
15156 LHS.getOperand(i: 0) == LHS.getOperand(i: 1))) {
15157 const unsigned OrdMask = SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN;
15158 unsigned NewMask = LCC == ISD::SETO ? Mask->getZExtValue() & ~OrdMask
15159 : Mask->getZExtValue() & OrdMask;
15160
15161 SDLoc DL(N);
15162 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: RHS.getOperand(i: 0),
15163 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
15164 }
15165 }
15166
15167 if (VT == MVT::i32 && (RHS.getOpcode() == ISD::SIGN_EXTEND ||
15168 LHS.getOpcode() == ISD::SIGN_EXTEND)) {
15169 // and x, (sext cc from i1) => select cc, x, 0
15170 if (RHS.getOpcode() != ISD::SIGN_EXTEND)
15171 std::swap(a&: LHS, b&: RHS);
15172 if (isBoolSGPR(V: RHS.getOperand(i: 0)))
15173 return DAG.getSelect(DL: SDLoc(N), VT: MVT::i32, Cond: RHS.getOperand(i: 0), LHS,
15174 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32));
15175 }
15176
15177 // and (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15178 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15179 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15180 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15181 uint32_t LHSMask = getPermuteMask(V: LHS);
15182 uint32_t RHSMask = getPermuteMask(V: RHS);
15183 if (LHSMask != ~0u && RHSMask != ~0u) {
15184 // Canonicalize the expression in an attempt to have fewer unique masks
15185 // and therefore fewer registers used to hold the masks.
15186 if (LHSMask > RHSMask) {
15187 std::swap(a&: LHSMask, b&: RHSMask);
15188 std::swap(a&: LHS, b&: RHS);
15189 }
15190
15191 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15192 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15193 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15194 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15195
15196 // Check of we need to combine values from two sources within a byte.
15197 if (!(LHSUsedLanes & RHSUsedLanes) &&
15198 // If we select high and lower word keep it for SDWA.
15199 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15200 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15201 // Each byte in each mask is either selector mask 0-3, or has higher
15202 // bits set in either of masks, which can be 0xff for 0xff or 0x0c for
15203 // zero. If 0x0c is in either mask it shall always be 0x0c. Otherwise
15204 // mask which is not 0xff wins. By anding both masks we have a correct
15205 // result except that 0x0c shall be corrected to give 0x0c only.
15206 uint32_t Mask = LHSMask & RHSMask;
15207 for (unsigned I = 0; I < 32; I += 8) {
15208 uint32_t ByteSel = 0xff << I;
15209 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
15210 Mask &= (0x0c << I) & 0xffffffff;
15211 }
15212
15213 // Add 4 to each active LHS lane. It will not affect any existing 0xff
15214 // or 0x0c.
15215 uint32_t Sel = Mask | (LHSUsedLanes & 0x04040404);
15216 SDLoc DL(N);
15217
15218 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15219 N2: RHS.getOperand(i: 0),
15220 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15221 }
15222 }
15223 }
15224
15225 return SDValue();
15226}
15227
15228// A key component of v_perm is a mapping between byte position of the src
15229// operands, and the byte position of the dest. To provide such, we need: 1. the
15230// node that provides x byte of the dest of the OR, and 2. the byte of the node
15231// used to provide that x byte. calculateByteProvider finds which node provides
15232// a certain byte of the dest of the OR, and calculateSrcByte takes that node,
15233// and finds an ultimate src and byte position For example: The supported
15234// LoadCombine pattern for vector loads is as follows
15235// t1
15236// or
15237// / \
15238// t2 t3
15239// zext shl
15240// | | \
15241// t4 t5 16
15242// or anyext
15243// / \ |
15244// t6 t7 t8
15245// srl shl or
15246// / | / \ / \
15247// t9 t10 t11 t12 t13 t14
15248// trunc* 8 trunc* 8 and and
15249// | | / | | \
15250// t15 t16 t17 t18 t19 t20
15251// trunc* 255 srl -256
15252// | / \
15253// t15 t15 16
15254//
15255// *In this example, the truncs are from i32->i16
15256//
15257// calculateByteProvider would find t6, t7, t13, and t14 for bytes 0-3
15258// respectively. calculateSrcByte would find (given node) -> ultimate src &
15259// byteposition: t6 -> t15 & 1, t7 -> t16 & 0, t13 -> t15 & 0, t14 -> t15 & 3.
15260// After finding the mapping, we can combine the tree into vperm t15, t16,
15261// 0x05000407
15262
15263// Find the source and byte position from a node.
15264// \p DestByte is the byte position of the dest of the or that the src
15265// ultimately provides. \p SrcIndex is the byte of the src that maps to this
15266// dest of the or byte. \p Depth tracks how many recursive iterations we have
15267// performed.
15268static const std::optional<ByteProvider<SDValue>>
15269calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
15270 unsigned Depth = 0) {
15271 // We may need to recursively traverse a series of SRLs
15272 if (Depth >= 6)
15273 return std::nullopt;
15274
15275 if (Op.getValueSizeInBits() < 8)
15276 return std::nullopt;
15277
15278 if (Op.getValueType().isVector())
15279 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
15280
15281 switch (Op->getOpcode()) {
15282 case ISD::TRUNCATE: {
15283 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
15284 }
15285
15286 case ISD::ANY_EXTEND:
15287 case ISD::SIGN_EXTEND:
15288 case ISD::ZERO_EXTEND:
15289 case ISD::SIGN_EXTEND_INREG: {
15290 SDValue NarrowOp = Op->getOperand(Num: 0);
15291 auto NarrowVT = NarrowOp.getValueType();
15292 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG) {
15293 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15294 NarrowVT = VTSign->getVT();
15295 }
15296 if (!NarrowVT.isByteSized())
15297 return std::nullopt;
15298 uint64_t NarrowByteWidth = NarrowVT.getStoreSize();
15299
15300 if (SrcIndex >= NarrowByteWidth)
15301 return std::nullopt;
15302 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
15303 }
15304
15305 case ISD::SRA:
15306 case ISD::SRL: {
15307 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15308 if (!ShiftOp)
15309 return std::nullopt;
15310
15311 uint64_t BitShift = ShiftOp->getZExtValue();
15312
15313 if (BitShift % 8 != 0)
15314 return std::nullopt;
15315
15316 uint64_t NewSrcIndex = SrcIndex + BitShift / 8;
15317 if (NewSrcIndex >= Op.getScalarValueSizeInBits() / 8)
15318 return std::nullopt;
15319
15320 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex: NewSrcIndex,
15321 Depth: Depth + 1);
15322 }
15323
15324 default: {
15325 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
15326 }
15327 }
15328 llvm_unreachable("fully handled switch");
15329}
15330
15331// For a byte position in the result of an Or, traverse the tree and find the
15332// node (and the byte of the node) which ultimately provides this {Or,
15333// BytePosition}. \p Op is the operand we are currently examining. \p Index is
15334// the byte position of the Op that corresponds with the originally requested
15335// byte of the Or \p Depth tracks how many recursive iterations we have
15336// performed. \p StartingIndex is the originally requested byte of the Or
15337static const std::optional<ByteProvider<SDValue>>
15338calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
15339 unsigned StartingIndex = 0) {
15340 // Finding Src tree of RHS of or typically requires at least 1 additional
15341 // depth
15342 if (Depth > 6)
15343 return std::nullopt;
15344
15345 unsigned BitWidth = Op.getScalarValueSizeInBits();
15346 if (BitWidth % 8 != 0)
15347 return std::nullopt;
15348 if (Index > BitWidth / 8 - 1)
15349 return std::nullopt;
15350
15351 bool IsVec = Op.getValueType().isVector();
15352 switch (Op.getOpcode()) {
15353 case ISD::OR: {
15354 if (IsVec)
15355 return std::nullopt;
15356
15357 auto RHS = calculateByteProvider(Op: Op.getOperand(i: 1), Index, Depth: Depth + 1,
15358 StartingIndex);
15359 if (!RHS)
15360 return std::nullopt;
15361 auto LHS = calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15362 StartingIndex);
15363 if (!LHS)
15364 return std::nullopt;
15365 // A well formed Or will have two ByteProviders for each byte, one of which
15366 // is constant zero
15367 if (!LHS->isConstantZero() && !RHS->isConstantZero())
15368 return std::nullopt;
15369 if (!LHS || LHS->isConstantZero())
15370 return RHS;
15371 if (!RHS || RHS->isConstantZero())
15372 return LHS;
15373 return std::nullopt;
15374 }
15375
15376 case ISD::AND: {
15377 if (IsVec)
15378 return std::nullopt;
15379
15380 auto *BitMaskOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15381 if (!BitMaskOp)
15382 return std::nullopt;
15383
15384 uint32_t BitMask = BitMaskOp->getZExtValue();
15385 // Bits we expect for our StartingIndex
15386 uint32_t IndexMask = 0xFF << (Index * 8);
15387
15388 if ((IndexMask & BitMask) != IndexMask) {
15389 // If the result of the and partially provides the byte, then it
15390 // is not well formatted
15391 if (IndexMask & BitMask)
15392 return std::nullopt;
15393 return ByteProvider<SDValue>::getConstantZero();
15394 }
15395
15396 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex, SrcIndex: Index);
15397 }
15398
15399 case ISD::FSHR: {
15400 if (IsVec)
15401 return std::nullopt;
15402
15403 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
15404 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15405 if (!ShiftOp || Op.getValueType().isVector())
15406 return std::nullopt;
15407
15408 uint64_t BitsProvided = Op.getValueSizeInBits();
15409 if (BitsProvided % 8 != 0)
15410 return std::nullopt;
15411
15412 uint64_t BitShift = ShiftOp->getAPIntValue().urem(RHS: BitsProvided);
15413 if (BitShift % 8)
15414 return std::nullopt;
15415
15416 uint64_t ConcatSizeInBytes = BitsProvided / 4;
15417 uint64_t ByteShift = BitShift / 8;
15418
15419 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
15420 uint64_t BytesProvided = BitsProvided / 8;
15421 SDValue NextOp = Op.getOperand(i: NewIndex >= BytesProvided ? 0 : 1);
15422 NewIndex %= BytesProvided;
15423 return calculateByteProvider(Op: NextOp, Index: NewIndex, Depth: Depth + 1, StartingIndex);
15424 }
15425
15426 case ISD::SRA:
15427 case ISD::SRL: {
15428 if (IsVec)
15429 return std::nullopt;
15430
15431 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15432 if (!ShiftOp)
15433 return std::nullopt;
15434
15435 uint64_t BitShift = ShiftOp->getZExtValue();
15436 if (BitShift % 8)
15437 return std::nullopt;
15438
15439 auto BitsProvided = Op.getScalarValueSizeInBits();
15440 if (BitsProvided % 8 != 0)
15441 return std::nullopt;
15442
15443 uint64_t BytesProvided = BitsProvided / 8;
15444 uint64_t ByteShift = BitShift / 8;
15445 if (Index + ByteShift < BytesProvided)
15446 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex,
15447 SrcIndex: Index + ByteShift);
15448 // SRA's out-of-range bytes are sign bits, not constant zero.
15449 if (Op.getOpcode() == ISD::SRA)
15450 return std::nullopt;
15451 return ByteProvider<SDValue>::getConstantZero();
15452 }
15453
15454 case ISD::SHL: {
15455 if (IsVec)
15456 return std::nullopt;
15457
15458 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15459 if (!ShiftOp)
15460 return std::nullopt;
15461
15462 uint64_t BitShift = ShiftOp->getZExtValue();
15463 if (BitShift % 8 != 0)
15464 return std::nullopt;
15465 uint64_t ByteShift = BitShift / 8;
15466
15467 // If we are shifting by an amount greater than (or equal to)
15468 // the index we are trying to provide, then it provides 0s. If not,
15469 // then this bytes are not definitively 0s, and the corresponding byte
15470 // of interest is Index - ByteShift of the src
15471 return Index < ByteShift
15472 ? ByteProvider<SDValue>::getConstantZero()
15473 : calculateByteProvider(Op: Op.getOperand(i: 0), Index: Index - ByteShift,
15474 Depth: Depth + 1, StartingIndex);
15475 }
15476 case ISD::ANY_EXTEND:
15477 case ISD::SIGN_EXTEND:
15478 case ISD::ZERO_EXTEND:
15479 case ISD::SIGN_EXTEND_INREG:
15480 case ISD::AssertZext:
15481 case ISD::AssertSext: {
15482 if (IsVec)
15483 return std::nullopt;
15484
15485 SDValue NarrowOp = Op->getOperand(Num: 0);
15486 unsigned NarrowBitWidth = NarrowOp.getValueSizeInBits();
15487 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG ||
15488 Op->getOpcode() == ISD::AssertZext ||
15489 Op->getOpcode() == ISD::AssertSext) {
15490 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15491 NarrowBitWidth = VTSign->getVT().getSizeInBits();
15492 }
15493 if (NarrowBitWidth % 8 != 0)
15494 return std::nullopt;
15495 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15496
15497 if (Index >= NarrowByteWidth)
15498 return Op.getOpcode() == ISD::ZERO_EXTEND
15499 ? std::optional<ByteProvider<SDValue>>(
15500 ByteProvider<SDValue>::getConstantZero())
15501 : std::nullopt;
15502 return calculateByteProvider(Op: NarrowOp, Index, Depth: Depth + 1, StartingIndex);
15503 }
15504
15505 case ISD::TRUNCATE: {
15506 if (IsVec)
15507 return std::nullopt;
15508
15509 uint64_t NarrowByteWidth = BitWidth / 8;
15510
15511 if (NarrowByteWidth >= Index) {
15512 return calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15513 StartingIndex);
15514 }
15515
15516 return std::nullopt;
15517 }
15518
15519 case ISD::CopyFromReg: {
15520 if (BitWidth / 8 > Index)
15521 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15522
15523 return std::nullopt;
15524 }
15525
15526 case ISD::LOAD: {
15527 auto *L = cast<LoadSDNode>(Val: Op.getNode());
15528
15529 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
15530 if (NarrowBitWidth % 8 != 0)
15531 return std::nullopt;
15532 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15533
15534 // If the width of the load does not reach byte we are trying to provide for
15535 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
15536 // question
15537 if (Index >= NarrowByteWidth) {
15538 return L->getExtensionType() == ISD::ZEXTLOAD
15539 ? std::optional<ByteProvider<SDValue>>(
15540 ByteProvider<SDValue>::getConstantZero())
15541 : std::nullopt;
15542 }
15543
15544 if (NarrowByteWidth > Index) {
15545 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15546 }
15547
15548 return std::nullopt;
15549 }
15550
15551 case ISD::BSWAP: {
15552 if (IsVec)
15553 return std::nullopt;
15554
15555 return calculateByteProvider(Op: Op->getOperand(Num: 0), Index: BitWidth / 8 - Index - 1,
15556 Depth: Depth + 1, StartingIndex);
15557 }
15558
15559 case ISD::EXTRACT_VECTOR_ELT: {
15560 auto *IdxOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15561 if (!IdxOp)
15562 return std::nullopt;
15563 auto VecIdx = IdxOp->getZExtValue();
15564 auto ScalarSize = Op.getScalarValueSizeInBits();
15565 if (ScalarSize < 32)
15566 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
15567 return calculateSrcByte(Op: ScalarSize >= 32 ? Op : Op.getOperand(i: 0),
15568 DestByte: StartingIndex, SrcIndex: Index);
15569 }
15570
15571 case AMDGPUISD::PERM: {
15572 if (IsVec)
15573 return std::nullopt;
15574
15575 auto *PermMask = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15576 if (!PermMask)
15577 return std::nullopt;
15578
15579 auto IdxMask =
15580 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
15581 if (IdxMask > 0x07 && IdxMask != 0x0c)
15582 return std::nullopt;
15583
15584 auto NextOp = Op.getOperand(i: IdxMask > 0x03 ? 0 : 1);
15585 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
15586
15587 return IdxMask != 0x0c ? calculateSrcByte(Op: NextOp, DestByte: StartingIndex, SrcIndex: NextIndex)
15588 : ByteProvider<SDValue>(
15589 ByteProvider<SDValue>::getConstantZero());
15590 }
15591
15592 default: {
15593 return std::nullopt;
15594 }
15595 }
15596
15597 llvm_unreachable("fully handled switch");
15598}
15599
15600// Returns true if the Operand is a scalar and is 16 bits
15601static bool isExtendedFrom16Bits(SDValue &Operand) {
15602
15603 switch (Operand.getOpcode()) {
15604 case ISD::ANY_EXTEND:
15605 case ISD::SIGN_EXTEND:
15606 case ISD::ZERO_EXTEND: {
15607 auto OpVT = Operand.getOperand(i: 0).getValueType();
15608 return !OpVT.isVector() && OpVT.getSizeInBits() == 16;
15609 }
15610 case ISD::LOAD: {
15611 LoadSDNode *L = cast<LoadSDNode>(Val: Operand.getNode());
15612 auto ExtType = cast<LoadSDNode>(Val: L)->getExtensionType();
15613 if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::SEXTLOAD ||
15614 ExtType == ISD::EXTLOAD) {
15615 auto MemVT = L->getMemoryVT();
15616 return !MemVT.isVector() && MemVT.getSizeInBits() == 16;
15617 }
15618 return L->getMemoryVT().getSizeInBits() == 16;
15619 }
15620 default:
15621 return false;
15622 }
15623}
15624
15625// Returns true if the mask matches consecutive bytes, and the first byte
15626// begins at a power of 2 byte offset from 0th byte
15627static bool addresses16Bits(int Mask) {
15628 int Low8 = Mask & 0xff;
15629 int Hi8 = (Mask & 0xff00) >> 8;
15630
15631 assert(Low8 < 8 && Hi8 < 8);
15632 // Are the bytes contiguous in the order of increasing addresses.
15633 bool IsConsecutive = (Hi8 - Low8 == 1);
15634 // Is the first byte at location that is aligned for 16 bit instructions.
15635 // A counter example is taking 2 consecutive bytes starting at the 8th bit.
15636 // In this case, we still need code to extract the 16 bit operand, so it
15637 // is better to use i8 v_perm
15638 bool Is16Aligned = !(Low8 % 2);
15639
15640 return IsConsecutive && Is16Aligned;
15641}
15642
15643// Do not lower into v_perm if the operands are actually 16 bit
15644// and the selected bits (based on PermMask) correspond with two
15645// easily addressable 16 bit operands.
15646static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op,
15647 SDValue &OtherOp) {
15648 int Low16 = PermMask & 0xffff;
15649 int Hi16 = (PermMask & 0xffff0000) >> 16;
15650
15651 auto TempOp = peekThroughBitcasts(V: Op);
15652 auto TempOtherOp = peekThroughBitcasts(V: OtherOp);
15653
15654 auto OpIs16Bit =
15655 TempOp.getValueSizeInBits() == 16 || isExtendedFrom16Bits(Operand&: TempOp);
15656 if (!OpIs16Bit)
15657 return true;
15658
15659 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
15660 isExtendedFrom16Bits(Operand&: TempOtherOp);
15661 if (!OtherOpIs16Bit)
15662 return true;
15663
15664 // Do we cleanly address both
15665 return !addresses16Bits(Mask: Low16) || !addresses16Bits(Mask: Hi16);
15666}
15667
15668static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src,
15669 unsigned DWordOffset) {
15670 SDValue Ret;
15671
15672 auto TypeSize = Src.getValueSizeInBits().getFixedValue();
15673 // ByteProvider must be at least 8 bits
15674 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
15675
15676 if (TypeSize <= 32)
15677 return DAG.getBitcastedAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
15678
15679 if (Src.getValueType().isVector()) {
15680 auto ScalarTySize = Src.getScalarValueSizeInBits();
15681 auto ScalarTy = Src.getValueType().getScalarType();
15682 if (ScalarTySize == 32) {
15683 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Src,
15684 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15685 }
15686 if (ScalarTySize > 32) {
15687 Ret = DAG.getNode(
15688 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ScalarTy, N1: Src,
15689 N2: DAG.getConstant(Val: DWordOffset / (ScalarTySize / 32), DL: SL, VT: MVT::i32));
15690 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
15691 if (ShiftVal)
15692 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Ret.getValueType(), N1: Ret,
15693 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15694 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15695 }
15696
15697 assert(ScalarTySize < 32);
15698 if (TypeSize % 32 == 0) {
15699 assert(DWordOffset < TypeSize / 32);
15700 SDValue Cast = DAG.getBitcast(
15701 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: TypeSize / 32), V: Src);
15702 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
15703 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15704 }
15705
15706 auto NumElements = TypeSize / ScalarTySize;
15707 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
15708 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
15709 auto NumElementsIn32 = 32 / ScalarTySize;
15710 auto NumAvailElements = DWordOffset < Trunc32Elements
15711 ? NumElementsIn32
15712 : NumElements - NormalizedTrunc;
15713
15714 SmallVector<SDValue, 4> VecSrcs;
15715 DAG.ExtractVectorElements(Op: Src, Args&: VecSrcs, Start: DWordOffset * NumElementsIn32,
15716 Count: NumAvailElements);
15717
15718 Ret = DAG.getBuildVector(
15719 VT: MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ScalarTySize), NumElements: NumAvailElements), DL: SL,
15720 Ops: VecSrcs);
15721 return Ret = DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15722 }
15723
15724 /// Scalar Type
15725 auto ShiftVal = 32 * DWordOffset;
15726 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Src.getValueType(), N1: Src,
15727 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15728 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15729}
15730
15731static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15732 SelectionDAG &DAG = DCI.DAG;
15733 [[maybe_unused]] EVT VT = N->getValueType(ResNo: 0);
15734 SmallVector<ByteProvider<SDValue>, 8> PermNodes;
15735
15736 // VT is known to be MVT::i32, so we need to provide 4 bytes.
15737 assert(VT == MVT::i32);
15738 for (int i = 0; i < 4; i++) {
15739 // Find the ByteProvider that provides the ith byte of the result of OR
15740 std::optional<ByteProvider<SDValue>> P =
15741 calculateByteProvider(Op: SDValue(N, 0), Index: i, Depth: 0, /*StartingIndex = */ i);
15742 // TODO support constantZero
15743 if (!P || P->isConstantZero())
15744 return SDValue();
15745
15746 PermNodes.push_back(Elt: *P);
15747 }
15748 if (PermNodes.size() != 4)
15749 return SDValue();
15750
15751 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
15752 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
15753 uint64_t PermMask = 0x00000000;
15754 for (size_t i = 0; i < PermNodes.size(); i++) {
15755 auto PermOp = PermNodes[i];
15756 // Since the mask is applied to Src1:Src2, Src1 bytes must be offset
15757 // by sizeof(Src2) = 4
15758 int SrcByteAdjust = 4;
15759
15760 // If the Src uses a byte from a different DWORD, then it corresponds
15761 // with a difference source
15762 if (!PermOp.hasSameSrc(Other: PermNodes[FirstSrc.first]) ||
15763 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
15764 if (SecondSrc)
15765 if (!PermOp.hasSameSrc(Other: PermNodes[SecondSrc->first]) ||
15766 ((PermOp.SrcOffset / 4) != SecondSrc->second))
15767 return SDValue();
15768
15769 // Set the index of the second distinct Src node
15770 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
15771 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
15772 SrcByteAdjust = 0;
15773 }
15774 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
15775 assert(!DAG.getDataLayout().isBigEndian());
15776 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
15777 }
15778 SDLoc DL(N);
15779 SDValue Op = *PermNodes[FirstSrc.first].Src;
15780 Op = getDWordFromOffset(DAG, SL: DL, Src: Op, DWordOffset: FirstSrc.second);
15781 assert(Op.getValueSizeInBits() == 32);
15782
15783 // Check that we are not just extracting the bytes in order from an op
15784 if (!SecondSrc) {
15785 int Low16 = PermMask & 0xffff;
15786 int Hi16 = (PermMask & 0xffff0000) >> 16;
15787
15788 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
15789 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
15790
15791 // The perm op would really just produce Op. So combine into Op
15792 if (WellFormedLow && WellFormedHi)
15793 return DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: 32), V: Op);
15794 }
15795
15796 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src : Op;
15797
15798 if (SecondSrc) {
15799 OtherOp = getDWordFromOffset(DAG, SL: DL, Src: OtherOp, DWordOffset: SecondSrc->second);
15800 assert(OtherOp.getValueSizeInBits() == 32);
15801 }
15802
15803 // Check that we haven't just recreated the same FSHR node.
15804 if (N->getOpcode() == ISD::FSHR &&
15805 (N->getOperand(Num: 0) == Op || N->getOperand(Num: 0) == OtherOp) &&
15806 (N->getOperand(Num: 1) == Op || N->getOperand(Num: 1) == OtherOp))
15807 return SDValue();
15808
15809 if (hasNon16BitAccesses(PermMask, Op, OtherOp)) {
15810
15811 assert(Op.getValueType().isByteSized() &&
15812 OtherOp.getValueType().isByteSized());
15813
15814 // If the ultimate src is less than 32 bits, then we will only be
15815 // using bytes 0: Op.getValueSizeInBytes() - 1 in the or.
15816 // CalculateByteProvider would not have returned Op as source if we
15817 // used a byte that is outside its ValueType. Thus, we are free to
15818 // ANY_EXTEND as the extended bits are dont-cares.
15819 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, VT: MVT::i32);
15820 OtherOp = DAG.getBitcastedAnyExtOrTrunc(Op: OtherOp, DL, VT: MVT::i32);
15821
15822 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op, N2: OtherOp,
15823 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
15824 }
15825 return SDValue();
15826}
15827
15828SDValue SITargetLowering::performOrCombine(SDNode *N,
15829 DAGCombinerInfo &DCI) const {
15830 SelectionDAG &DAG = DCI.DAG;
15831 SDValue LHS = N->getOperand(Num: 0);
15832 SDValue RHS = N->getOperand(Num: 1);
15833
15834 EVT VT = N->getValueType(ResNo: 0);
15835 if (VT == MVT::i1) {
15836 // or (fp_class x, c1), (fp_class x, c2) -> fp_class x, (c1 | c2)
15837 if (LHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15838 RHS.getOpcode() == AMDGPUISD::FP_CLASS) {
15839 SDValue Src = LHS.getOperand(i: 0);
15840 if (Src != RHS.getOperand(i: 0))
15841 return SDValue();
15842
15843 const ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
15844 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
15845 if (!CLHS || !CRHS)
15846 return SDValue();
15847
15848 // Only 10 bits are used.
15849 static const uint32_t MaxMask = 0x3ff;
15850
15851 uint32_t NewMask =
15852 (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask;
15853 SDLoc DL(N);
15854 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: Src,
15855 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
15856 }
15857
15858 return SDValue();
15859 }
15860
15861 // or (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15862 if (isa<ConstantSDNode>(Val: RHS) && LHS.hasOneUse() &&
15863 LHS.getOpcode() == AMDGPUISD::PERM &&
15864 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
15865 uint32_t Sel = getConstantPermuteMask(C: N->getConstantOperandVal(Num: 1));
15866 if (!Sel)
15867 return SDValue();
15868
15869 Sel |= LHS.getConstantOperandVal(i: 2);
15870 SDLoc DL(N);
15871 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15872 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15873 }
15874
15875 // or (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15876 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15877 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15878 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15879
15880 // If all the uses of an or need to extract the individual elements, do not
15881 // attempt to lower into v_perm
15882 auto usesCombinedOperand = [](SDNode *OrUse) {
15883 // If we have any non-vectorized use, then it is a candidate for v_perm
15884 if (OrUse->getOpcode() != ISD::BITCAST ||
15885 !OrUse->getValueType(ResNo: 0).isVector())
15886 return true;
15887
15888 // If we have any non-vectorized use, then it is a candidate for v_perm
15889 for (auto *VUser : OrUse->users()) {
15890 if (!VUser->getValueType(ResNo: 0).isVector())
15891 return true;
15892
15893 // If the use of a vector is a store, then combining via a v_perm
15894 // is beneficial.
15895 // TODO -- whitelist more uses
15896 for (auto VectorwiseOp : {ISD::STORE, ISD::CopyToReg, ISD::CopyFromReg})
15897 if (VUser->getOpcode() == VectorwiseOp)
15898 return true;
15899 }
15900 return false;
15901 };
15902
15903 if (!any_of(Range: N->users(), P: usesCombinedOperand))
15904 return SDValue();
15905
15906 uint32_t LHSMask = getPermuteMask(V: LHS);
15907 uint32_t RHSMask = getPermuteMask(V: RHS);
15908
15909 if (LHSMask != ~0u && RHSMask != ~0u) {
15910 // Canonicalize the expression in an attempt to have fewer unique masks
15911 // and therefore fewer registers used to hold the masks.
15912 if (LHSMask > RHSMask) {
15913 std::swap(a&: LHSMask, b&: RHSMask);
15914 std::swap(a&: LHS, b&: RHS);
15915 }
15916
15917 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15918 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15919 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15920 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15921
15922 // Check of we need to combine values from two sources within a byte.
15923 if (!(LHSUsedLanes & RHSUsedLanes) &&
15924 // If we select high and lower word keep it for SDWA.
15925 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15926 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15927 // Kill zero bytes selected by other mask. Zero value is 0xc.
15928 LHSMask &= ~RHSUsedLanes;
15929 RHSMask &= ~LHSUsedLanes;
15930 // Add 4 to each active LHS lane
15931 LHSMask |= LHSUsedLanes & 0x04040404;
15932 // Combine masks
15933 uint32_t Sel = LHSMask | RHSMask;
15934 SDLoc DL(N);
15935
15936 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15937 N2: RHS.getOperand(i: 0),
15938 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15939 }
15940 }
15941 if (LHSMask == ~0u || RHSMask == ~0u) {
15942 if (SDValue Perm = matchPERM(N, DCI))
15943 return Perm;
15944 }
15945 }
15946
15947 // Detect identity v2i32 OR and replace with identity source node.
15948 // Specifically an Or that has operands constructed from the same source node
15949 // via extract_vector_elt and build_vector. I.E.
15950 // v2i32 or(
15951 // v2i32 build_vector(
15952 // i32 extract_elt(%IdentitySrc, 0),
15953 // i32 0
15954 // ),
15955 // v2i32 build_vector(
15956 // i32 0,
15957 // i32 extract_elt(%IdentitySrc, 1)
15958 // ) )
15959 // =>
15960 // v2i32 %IdentitySrc
15961
15962 if (VT == MVT::v2i32 && LHS->getOpcode() == ISD::BUILD_VECTOR &&
15963 RHS->getOpcode() == ISD::BUILD_VECTOR) {
15964
15965 ConstantSDNode *LC = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1));
15966 ConstantSDNode *RC = dyn_cast<ConstantSDNode>(Val: RHS->getOperand(Num: 0));
15967
15968 // Test for and normalise build vectors.
15969 if (LC && RC && LC->getZExtValue() == 0 && RC->getZExtValue() == 0) {
15970
15971 // Get the extract_vector_element operands.
15972 SDValue LEVE = LHS->getOperand(Num: 0);
15973 SDValue REVE = RHS->getOperand(Num: 1);
15974
15975 if (LEVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15976 REVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
15977 // Check that different elements from the same vector are
15978 // extracted.
15979 if (LEVE->getOperand(Num: 0) == REVE->getOperand(Num: 0) &&
15980 LEVE->getOperand(Num: 1) != REVE->getOperand(Num: 1)) {
15981 SDValue IdentitySrc = LEVE.getOperand(i: 0);
15982 return IdentitySrc;
15983 }
15984 }
15985 }
15986 }
15987
15988 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
15989 return SDValue();
15990
15991 // TODO: This could be a generic combine with a predicate for extracting the
15992 // high half of an integer being free.
15993
15994 // (or i64:x, (zero_extend i32:y)) ->
15995 // i64 (bitcast (v2i32 build_vector (or i32:y, lo_32(x)), hi_32(x)))
15996 if (LHS.getOpcode() == ISD::ZERO_EXTEND &&
15997 RHS.getOpcode() != ISD::ZERO_EXTEND)
15998 std::swap(a&: LHS, b&: RHS);
15999
16000 if (RHS.getOpcode() == ISD::ZERO_EXTEND) {
16001 SDValue ExtSrc = RHS.getOperand(i: 0);
16002 EVT SrcVT = ExtSrc.getValueType();
16003 if (SrcVT == MVT::i32) {
16004 SDLoc SL(N);
16005 auto [LowLHS, HiBits] = split64BitValue(Op: LHS, DAG);
16006 SDValue LowOr = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: LowLHS, N2: ExtSrc);
16007
16008 DCI.AddToWorklist(N: LowOr.getNode());
16009 DCI.AddToWorklist(N: HiBits.getNode());
16010
16011 SDValue Vec =
16012 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: LowOr, N2: HiBits);
16013 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
16014 }
16015 }
16016
16017 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
16018 if (CRHS) {
16019 if (SDValue Split = splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::OR,
16020 LHS: N->getOperand(Num: 0), CRHS))
16021 return Split;
16022 }
16023
16024 return SDValue();
16025}
16026
16027SDValue SITargetLowering::performXorCombine(SDNode *N,
16028 DAGCombinerInfo &DCI) const {
16029 if (SDValue RV = reassociateScalarOps(N, DAG&: DCI.DAG))
16030 return RV;
16031
16032 SDValue LHS = N->getOperand(Num: 0);
16033 SDValue RHS = N->getOperand(Num: 1);
16034
16035 const ConstantSDNode *CRHS = isConstOrConstSplat(N: RHS);
16036 SelectionDAG &DAG = DCI.DAG;
16037
16038 EVT VT = N->getValueType(ResNo: 0);
16039 if (CRHS && VT == MVT::i64) {
16040 if (SDValue Split =
16041 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::XOR, LHS, CRHS))
16042 return Split;
16043 }
16044
16045 // v2i32 (xor (vselect cc, x, y), K) ->
16046 // (v2i32 svelect cc, (xor x, K), (xor y, K)) This enables the xor to be
16047 // replaced with source modifiers when the select is lowered to CNDMASK.
16048 unsigned Opc = LHS.getOpcode();
16049 if (((Opc == ISD::VSELECT && VT == MVT::v2i32) ||
16050 (Opc == ISD::SELECT && VT == MVT::i64)) &&
16051 CRHS && CRHS->getAPIntValue().isSignMask()) {
16052 SDValue CC = LHS->getOperand(Num: 0);
16053 SDValue TRUE = LHS->getOperand(Num: 1);
16054 SDValue FALSE = LHS->getOperand(Num: 2);
16055 SDValue XTrue = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: TRUE, N2: RHS);
16056 SDValue XFalse = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: FALSE, N2: RHS);
16057 SDValue XSelect =
16058 DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT, N1: CC, N2: XTrue, N3: XFalse);
16059 return XSelect;
16060 }
16061
16062 // Make sure to apply the 64-bit constant splitting fold before trying to fold
16063 // fneg-like xors into 64-bit select.
16064 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) {
16065 // This looks like an fneg, try to fold as a source modifier.
16066 if (CRHS && CRHS->getAPIntValue().isSignMask() &&
16067 shouldFoldFNegIntoSrc(FNeg: N, FNegSrc: LHS)) {
16068 // xor (select c, a, b), 0x80000000 ->
16069 // bitcast (select c, (fneg (bitcast a)), (fneg (bitcast b)))
16070 SDLoc DL(N);
16071 SDValue CastLHS =
16072 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 1));
16073 SDValue CastRHS =
16074 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 2));
16075 SDValue FNegLHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastLHS);
16076 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastRHS);
16077 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f32,
16078 N1: LHS->getOperand(Num: 0), N2: FNegLHS, N3: FNegRHS);
16079 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: NewSelect);
16080 }
16081 }
16082
16083 return SDValue();
16084}
16085
16086SDValue
16087SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
16088 DAGCombinerInfo &DCI) const {
16089 if (!Subtarget->has16BitInsts() ||
16090 DCI.getDAGCombineLevel() < AfterLegalizeTypes)
16091 return SDValue();
16092
16093 EVT VT = N->getValueType(ResNo: 0);
16094 if (VT != MVT::i32)
16095 return SDValue();
16096
16097 SDValue Src = N->getOperand(Num: 0);
16098 if (Src.getValueType() != MVT::i16)
16099 return SDValue();
16100
16101 if (!Src->hasOneUse())
16102 return SDValue();
16103
16104 if (SDValue Ext = performExtendTruncShiftCombine(N, DCI))
16105 return Ext;
16106
16107 // TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
16108 // possible we're missing out on some combine opportunities, but we'd need to
16109 // weigh the cost of extracting the byte from the upper dwords.
16110
16111 std::optional<ByteProvider<SDValue>> BP0 =
16112 calculateByteProvider(Op: SDValue(N, 0), Index: 0, Depth: 0, StartingIndex: 0);
16113 if (!BP0 || BP0->SrcOffset >= 4 || !BP0->Src)
16114 return SDValue();
16115 SDValue V0 = *BP0->Src;
16116
16117 std::optional<ByteProvider<SDValue>> BP1 =
16118 calculateByteProvider(Op: SDValue(N, 0), Index: 1, Depth: 0, StartingIndex: 1);
16119 if (!BP1 || BP1->SrcOffset >= 4 || !BP1->Src)
16120 return SDValue();
16121
16122 SDValue V1 = *BP1->Src;
16123
16124 if (V0 == V1)
16125 return SDValue();
16126
16127 SelectionDAG &DAG = DCI.DAG;
16128 SDLoc DL(N);
16129 uint32_t PermMask = 0x0c0c0c0c;
16130 if (V0) {
16131 V0 = DAG.getBitcastedAnyExtOrTrunc(Op: V0, DL, VT: MVT::i32);
16132 PermMask = (PermMask & ~0xFF) | (BP0->SrcOffset + 4);
16133 }
16134
16135 if (V1) {
16136 V1 = DAG.getBitcastedAnyExtOrTrunc(Op: V1, DL, VT: MVT::i32);
16137 PermMask = (PermMask & ~(0xFF << 8)) | (BP1->SrcOffset << 8);
16138 }
16139
16140 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: V0, N2: V1,
16141 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
16142}
16143
16144// zext (srl (trunc x), c) => and (srl x, c), lowbits
16145// aext (srl (trunc x), c) => and (srl x, c), lowbits
16146// sext (sra (trunc x), 8) => sext_inreg (srl x, 8), i8
16147SDValue
16148SITargetLowering::performExtendTruncShiftCombine(SDNode *N,
16149 DAGCombinerInfo &DCI) const {
16150 if (DCI.getDAGCombineLevel() < AfterLegalizeTypes)
16151 return SDValue();
16152
16153 const unsigned Opc = N->getOpcode();
16154 const bool IsSigned = Opc == ISD::SIGN_EXTEND;
16155 SDValue Src = N->getOperand(Num: 0);
16156 if (N->getValueType(ResNo: 0) != MVT::i32 || Src.getValueType() != MVT::i16 ||
16157 Src.getOpcode() != (IsSigned ? ISD::SRA : ISD::SRL) || !Src.hasOneUse() ||
16158 Src.getOperand(i: 0).getOpcode() != ISD::TRUNCATE)
16159 return SDValue();
16160
16161 // A uniform i16 srl is promoted to i32 later and yields the same nodes.
16162 // Skip it so that the node order, and with it the schedule, stays the same.
16163 if (!IsSigned && !Src->isDivergent())
16164 return SDValue();
16165
16166 SDValue X = Src.getOperand(i: 0).getOperand(i: 0);
16167 auto *Shift = dyn_cast<ConstantSDNode>(Val: Src.getOperand(i: 1));
16168 if (X.getValueType() != MVT::i32 || !Shift || Shift->getZExtValue() == 0 ||
16169 Shift->getZExtValue() >= 16 || (IsSigned && Shift->getZExtValue() != 8))
16170 return SDValue();
16171
16172 SelectionDAG &DAG = DCI.DAG;
16173 SDLoc SL(N);
16174 const unsigned Width = 16 - Shift->getZExtValue();
16175 SDValue Srl = DAG.getNode(
16176 Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: X,
16177 N2: DAG.getShiftAmountConstant(Val: Shift->getZExtValue(), VT: MVT::i32, DL: SL));
16178 if (IsSigned)
16179 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SL, VT: MVT::i32, N1: Srl,
16180 N2: DAG.getValueType(MVT::i8));
16181 return DAG.getNode(
16182 Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Srl,
16183 N2: DAG.getConstant(Val: maskTrailingOnes<uint32_t>(N: Width), DL: SL, VT: MVT::i32));
16184}
16185
16186SDValue
16187SITargetLowering::performSignExtendInRegCombine(SDNode *N,
16188 DAGCombinerInfo &DCI) const {
16189 SDValue Src = N->getOperand(Num: 0);
16190 auto *VTSign = cast<VTSDNode>(Val: N->getOperand(Num: 1));
16191
16192 // Combine s_buffer_load_u8 or s_buffer_load_u16 with sext and replace them
16193 // with s_buffer_load_i8 and s_buffer_load_i16 respectively.
16194 if (((Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE &&
16195 VTSign->getVT() == MVT::i8) ||
16196 (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_USHORT &&
16197 VTSign->getVT() == MVT::i16))) {
16198 assert(Subtarget->hasScalarSubwordLoads() &&
16199 "s_buffer_load_{u8, i8} are supported "
16200 "in GFX12 (or newer) architectures.");
16201 unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
16202 ? AMDGPUISD::SBUFFER_LOAD_BYTE
16203 : AMDGPUISD::SBUFFER_LOAD_SHORT;
16204 SDLoc DL(N);
16205 SDVTList ResList =
16206 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
16207 SDValue Ops[] = {
16208 Src.getOperand(i: 0), // Chain
16209 Src.getOperand(i: 1), // source register
16210 Src.getOperand(i: 2), // offset
16211 Src.getOperand(i: 3) // cachePolicy
16212 };
16213 auto *M = cast<MemSDNode>(Val&: Src);
16214 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
16215 Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
16216 return DCI.DAG.getMergeValues(Ops: {BufferLoad, BufferLoad.getValue(R: 1)}, dl: DL);
16217 }
16218 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
16219 VTSign->getVT() == MVT::i8) ||
16220 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT &&
16221 VTSign->getVT() == MVT::i16)) &&
16222 Src.hasOneUse()) {
16223 auto *M = cast<MemSDNode>(Val&: Src);
16224 SDValue Ops[] = {Src.getOperand(i: 0), // Chain
16225 Src.getOperand(i: 1), // rsrc
16226 Src.getOperand(i: 2), // vindex
16227 Src.getOperand(i: 3), // voffset
16228 Src.getOperand(i: 4), // soffset
16229 Src.getOperand(i: 5), // offset
16230 Src.getOperand(i: 6), Src.getOperand(i: 7)};
16231 // replace with BUFFER_LOAD_BYTE/SHORT
16232 SDVTList ResList =
16233 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
16234 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE)
16235 ? AMDGPUISD::BUFFER_LOAD_BYTE
16236 : AMDGPUISD::BUFFER_LOAD_SHORT;
16237 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
16238 Opcode: Opc, dl: SDLoc(N), VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
16239 return DCI.DAG.getMergeValues(
16240 Ops: {BufferLoadSignExt, BufferLoadSignExt.getValue(R: 1)}, dl: SDLoc(N));
16241 }
16242 return SDValue();
16243}
16244
16245SDValue SITargetLowering::performClassCombine(SDNode *N,
16246 DAGCombinerInfo &DCI) const {
16247 SelectionDAG &DAG = DCI.DAG;
16248 SDValue Mask = N->getOperand(Num: 1);
16249
16250 // fp_class x, 0 -> false
16251 if (isNullConstant(V: Mask))
16252 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i1);
16253
16254 if (N->getOperand(Num: 0).isUndef())
16255 return DAG.getUNDEF(VT: MVT::i1);
16256
16257 return SDValue();
16258}
16259
16260SDValue SITargetLowering::performRcpCombine(SDNode *N,
16261 DAGCombinerInfo &DCI) const {
16262 EVT VT = N->getValueType(ResNo: 0);
16263 SDValue N0 = N->getOperand(Num: 0);
16264
16265 if (N0.isUndef()) {
16266 return DCI.DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()),
16267 DL: SDLoc(N), VT);
16268 }
16269
16270 // TODO: Could handle f32 + amdgcn.sqrt but probably never reaches here.
16271 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) &&
16272 N->getFlags().hasAllowContract() && N0->getFlags().hasAllowContract()) {
16273 return DCI.DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(N), VT, Operand: N0.getOperand(i: 0),
16274 Flags: N->getFlags());
16275 }
16276
16277 return AMDGPUTargetLowering::performRcpCombine(N, DCI);
16278}
16279
16280bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
16281 SDNodeFlags UserFlags,
16282 unsigned MaxDepth) const {
16283 EVT VT = Op.getValueType();
16284 assert(VT.isFloatingPoint() &&
16285 "expected a floating-point value to query canonicality of");
16286 return isCanonicalized(DAG, Op, QueryVT: VT.getScalarType(), UserFlags, MaxDepth);
16287}
16288
16289bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
16290 EVT QueryVT, SDNodeFlags UserFlags,
16291 unsigned MaxDepth) const {
16292 assert(QueryVT.isFloatingPoint() && !QueryVT.isVector() &&
16293 "QueryVT must be a floating-point scalar type");
16294 EVT VT = Op.getValueType();
16295 if (VT.isFloatingPoint() && VT.getScalarType() != QueryVT)
16296 return false;
16297
16298 unsigned Opcode = Op.getOpcode();
16299 if (Opcode == ISD::FCANONICALIZE)
16300 return true;
16301
16302 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16303 const auto &F = CFP->getValueAPF();
16304 if (F.isNaN() && F.isSignaling())
16305 return false;
16306 if (!F.isDenormal())
16307 return true;
16308
16309 DenormalMode Mode =
16310 DAG.getMachineFunction().getDenormalMode(FPType: F.getSemantics());
16311 return Mode == DenormalMode::getIEEE();
16312 }
16313
16314 // If source is a result of another standard FP operation it is already in
16315 // canonical form.
16316 if (MaxDepth == 0)
16317 return false;
16318
16319 switch (Opcode) {
16320 // These will flush denorms if required.
16321 case ISD::FADD:
16322 case ISD::FSUB:
16323 case ISD::FMUL:
16324 case ISD::FCEIL:
16325 case ISD::FFLOOR:
16326 case ISD::FMA:
16327 case ISD::FMAD:
16328 case ISD::FSQRT:
16329 case ISD::FDIV:
16330 case ISD::FREM:
16331 case ISD::FP_ROUND:
16332 case ISD::FP_EXTEND:
16333 case ISD::FP16_TO_FP:
16334 case ISD::FP_TO_FP16:
16335 case ISD::BF16_TO_FP:
16336 case ISD::FP_TO_BF16:
16337 case ISD::FLDEXP:
16338 case AMDGPUISD::FMUL_LEGACY:
16339 case AMDGPUISD::FMAD_FTZ:
16340 case AMDGPUISD::RCP:
16341 case AMDGPUISD::RSQ:
16342 case AMDGPUISD::RSQ_CLAMP:
16343 case AMDGPUISD::RCP_LEGACY:
16344 case AMDGPUISD::RCP_IFLAG:
16345 case AMDGPUISD::LOG:
16346 case AMDGPUISD::EXP:
16347 case AMDGPUISD::DIV_SCALE:
16348 case AMDGPUISD::DIV_FMAS:
16349 case AMDGPUISD::DIV_FIXUP:
16350 case AMDGPUISD::FRACT:
16351 case AMDGPUISD::CVT_PKRTZ_F16_F32:
16352 case AMDGPUISD::CVT_F32_UBYTE0:
16353 case AMDGPUISD::CVT_F32_UBYTE1:
16354 case AMDGPUISD::CVT_F32_UBYTE2:
16355 case AMDGPUISD::CVT_F32_UBYTE3:
16356 case AMDGPUISD::FP_TO_FP16:
16357 case AMDGPUISD::SIN_HW:
16358 case AMDGPUISD::COS_HW:
16359 return true;
16360
16361 // It can/will be lowered or combined as a bit operation.
16362 // Need to check their input recursively to handle.
16363 case ISD::FNEG:
16364 case ISD::FABS:
16365 case ISD::FCOPYSIGN:
16366 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16367 MaxDepth: MaxDepth - 1);
16368
16369 case ISD::AND:
16370 if (Op.getValueType() == MVT::i32) {
16371 // Be careful as we only know it is a bitcast floating point type. It
16372 // could be f32, v2f16, we have no way of knowing. Luckily the constant
16373 // value that we optimize for, which comes up in fp32 to bf16 conversions,
16374 // is valid to optimize for all types.
16375 if (auto *RHS = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
16376 if (RHS->getZExtValue() == 0xffff0000) {
16377 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16378 MaxDepth: MaxDepth - 1);
16379 }
16380 }
16381 }
16382 break;
16383
16384 case ISD::FSIN:
16385 case ISD::FCOS:
16386 case ISD::FSINCOS:
16387 return Op.getValueType().getScalarType() != MVT::f16;
16388
16389 case ISD::FMINNUM:
16390 case ISD::FMAXNUM:
16391 case ISD::FMINNUM_IEEE:
16392 case ISD::FMAXNUM_IEEE:
16393 case ISD::FMINIMUM:
16394 case ISD::FMAXIMUM:
16395 case ISD::FMINIMUMNUM:
16396 case ISD::FMAXIMUMNUM:
16397 case AMDGPUISD::CLAMP:
16398 case AMDGPUISD::FMED3:
16399 case AMDGPUISD::FMAX3:
16400 case AMDGPUISD::FMIN3:
16401 case AMDGPUISD::FMAXIMUM3:
16402 case AMDGPUISD::FMINIMUM3: {
16403 // FIXME: Shouldn't treat the generic operations different based these.
16404 // However, we aren't really required to flush the result from
16405 // minnum/maxnum..
16406
16407 // snans will be quieted, so we only need to worry about denormals.
16408 if (Subtarget->supportsMinMaxDenormModes() ||
16409 // FIXME: denormalsEnabledForType is broken for dynamic
16410 denormalsEnabledForType(DAG, VT: Op.getValueType()))
16411 return true;
16412
16413 // Flushing may be required.
16414 // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. For such
16415 // targets need to check their input recursively.
16416
16417 // FIXME: Does this apply with clamp? It's implemented with max.
16418 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
16419 if (!isCanonicalized(DAG, Op: Op.getOperand(i: I), QueryVT, UserFlags,
16420 MaxDepth: MaxDepth - 1))
16421 return false;
16422 }
16423
16424 return true;
16425 }
16426 case ISD::SELECT: {
16427 return isCanonicalized(DAG, Op: Op.getOperand(i: 1), QueryVT, UserFlags,
16428 MaxDepth: MaxDepth - 1) &&
16429 isCanonicalized(DAG, Op: Op.getOperand(i: 2), QueryVT, UserFlags,
16430 MaxDepth: MaxDepth - 1);
16431 }
16432 case ISD::BUILD_VECTOR: {
16433 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
16434 SDValue SrcOp = Op.getOperand(i);
16435 if (!isCanonicalized(DAG, Op: SrcOp, QueryVT, UserFlags, MaxDepth: MaxDepth - 1))
16436 return false;
16437 }
16438
16439 return true;
16440 }
16441 case ISD::EXTRACT_VECTOR_ELT:
16442 case ISD::EXTRACT_SUBVECTOR: {
16443 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16444 MaxDepth: MaxDepth - 1);
16445 }
16446 case ISD::INSERT_VECTOR_ELT: {
16447 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16448 MaxDepth: MaxDepth - 1) &&
16449 isCanonicalized(DAG, Op: Op.getOperand(i: 1), QueryVT, UserFlags,
16450 MaxDepth: MaxDepth - 1);
16451 }
16452 case ISD::POISON:
16453 return true;
16454 case ISD::UNDEF:
16455 // Could be anything.
16456 return false;
16457
16458 case ISD::BITCAST: {
16459 // Carry QueryVT through the bitcast unchanged. The top-of-function guard
16460 // rejects a source whose FP format differs from the consumed type, so a
16461 // value canonical in one FP format is not assumed canonical in another.
16462 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
16463 return isCanonicalized(DAG, Op: Src, QueryVT, UserFlags, MaxDepth: MaxDepth - 1);
16464 }
16465 case ISD::TRUNCATE: {
16466 // Hack round the mess we make when legalizing extract_vector_elt
16467 if (Op.getValueType() == MVT::i16) {
16468 SDValue TruncSrc = Op.getOperand(i: 0);
16469 if (TruncSrc.getValueType() == MVT::i32 &&
16470 TruncSrc.getOpcode() == ISD::BITCAST &&
16471 TruncSrc.getOperand(i: 0).getValueType() == MVT::v2f16) {
16472 return isCanonicalized(DAG, Op: TruncSrc.getOperand(i: 0), QueryVT, UserFlags,
16473 MaxDepth: MaxDepth - 1);
16474 }
16475 }
16476 return false;
16477 }
16478 case ISD::INTRINSIC_WO_CHAIN: {
16479 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
16480 // TODO: Handle more intrinsics
16481 switch (IntrinsicID) {
16482 case Intrinsic::amdgcn_cvt_pkrtz:
16483 case Intrinsic::amdgcn_cubeid:
16484 case Intrinsic::amdgcn_frexp_mant:
16485 case Intrinsic::amdgcn_fdot2:
16486 case Intrinsic::amdgcn_rcp:
16487 case Intrinsic::amdgcn_rsq:
16488 case Intrinsic::amdgcn_rsq_clamp:
16489 case Intrinsic::amdgcn_rcp_legacy:
16490 case Intrinsic::amdgcn_rsq_legacy:
16491 case Intrinsic::amdgcn_trig_preop:
16492 case Intrinsic::amdgcn_tanh:
16493 case Intrinsic::amdgcn_log:
16494 case Intrinsic::amdgcn_exp2:
16495 case Intrinsic::amdgcn_sqrt:
16496 return true;
16497 default:
16498 break;
16499 }
16500
16501 break;
16502 }
16503 default:
16504 break;
16505 }
16506
16507 // FIXME: denormalsEnabledForType is broken for dynamic
16508 return denormalsEnabledForType(DAG, VT: Op.getValueType()) &&
16509 (UserFlags.hasNoNaNs() || DAG.isKnownNeverSNaN(Op));
16510}
16511
16512bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
16513 unsigned MaxDepth) const {
16514 const MachineRegisterInfo &MRI = MF.getRegInfo();
16515 MachineInstr *MI = MRI.getVRegDef(Reg);
16516 unsigned Opcode = MI->getOpcode();
16517
16518 if (Opcode == AMDGPU::G_FCANONICALIZE)
16519 return true;
16520
16521 std::optional<FPValueAndVReg> FCR;
16522 // Constant splat (can be padded with undef) or scalar constant.
16523 if (mi_match(R: Reg, MRI, P: MIPatternMatch::m_GFCstOrSplat(FPValReg&: FCR))) {
16524 if (FCR->Value.isSignaling())
16525 return false;
16526 if (!FCR->Value.isDenormal())
16527 return true;
16528
16529 DenormalMode Mode = MF.getDenormalMode(FPType: FCR->Value.getSemantics());
16530 return Mode == DenormalMode::getIEEE();
16531 }
16532
16533 if (MaxDepth == 0)
16534 return false;
16535
16536 switch (Opcode) {
16537 case AMDGPU::G_FADD:
16538 case AMDGPU::G_FSUB:
16539 case AMDGPU::G_FMUL:
16540 case AMDGPU::G_FCEIL:
16541 case AMDGPU::G_FFLOOR:
16542 case AMDGPU::G_FRINT:
16543 case AMDGPU::G_FNEARBYINT:
16544 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
16545 case AMDGPU::G_INTRINSIC_TRUNC:
16546 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
16547 case AMDGPU::G_FMA:
16548 case AMDGPU::G_FMAD:
16549 case AMDGPU::G_FSQRT:
16550 case AMDGPU::G_FDIV:
16551 case AMDGPU::G_FREM:
16552 case AMDGPU::G_FPOW:
16553 case AMDGPU::G_FPEXT:
16554 case AMDGPU::G_FLOG:
16555 case AMDGPU::G_FLOG2:
16556 case AMDGPU::G_FLOG10:
16557 case AMDGPU::G_FPTRUNC:
16558 case AMDGPU::G_AMDGPU_RCP_IFLAG:
16559 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
16560 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
16561 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
16562 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
16563 return true;
16564 case AMDGPU::G_FNEG:
16565 case AMDGPU::G_FABS:
16566 case AMDGPU::G_FCOPYSIGN:
16567 return isCanonicalized(Reg: MI->getOperand(i: 1).getReg(), MF, MaxDepth: MaxDepth - 1);
16568 case AMDGPU::G_FMINNUM:
16569 case AMDGPU::G_FMAXNUM:
16570 case AMDGPU::G_FMINNUM_IEEE:
16571 case AMDGPU::G_FMAXNUM_IEEE:
16572 case AMDGPU::G_FMINIMUM:
16573 case AMDGPU::G_FMAXIMUM:
16574 case AMDGPU::G_FMINIMUMNUM:
16575 case AMDGPU::G_FMAXIMUMNUM: {
16576 if (Subtarget->supportsMinMaxDenormModes() ||
16577 // FIXME: denormalsEnabledForType is broken for dynamic
16578 denormalsEnabledForType(Ty: MRI.getType(Reg), MF))
16579 return true;
16580
16581 [[fallthrough]];
16582 }
16583 case AMDGPU::G_BUILD_VECTOR:
16584 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
16585 if (!isCanonicalized(Reg: MO.getReg(), MF, MaxDepth: MaxDepth - 1))
16586 return false;
16587 return true;
16588 case AMDGPU::G_INTRINSIC:
16589 case AMDGPU::G_INTRINSIC_CONVERGENT:
16590 switch (cast<GIntrinsic>(Val: MI)->getIntrinsicID()) {
16591 case Intrinsic::amdgcn_fmul_legacy:
16592 case Intrinsic::amdgcn_fmad_ftz:
16593 case Intrinsic::amdgcn_sqrt:
16594 case Intrinsic::amdgcn_fmed3:
16595 case Intrinsic::amdgcn_sin:
16596 case Intrinsic::amdgcn_cos:
16597 case Intrinsic::amdgcn_log:
16598 case Intrinsic::amdgcn_exp2:
16599 case Intrinsic::amdgcn_log_clamp:
16600 case Intrinsic::amdgcn_rcp:
16601 case Intrinsic::amdgcn_rcp_legacy:
16602 case Intrinsic::amdgcn_rsq:
16603 case Intrinsic::amdgcn_rsq_clamp:
16604 case Intrinsic::amdgcn_rsq_legacy:
16605 case Intrinsic::amdgcn_div_scale:
16606 case Intrinsic::amdgcn_div_fmas:
16607 case Intrinsic::amdgcn_div_fixup:
16608 case Intrinsic::amdgcn_fract:
16609 case Intrinsic::amdgcn_cvt_pkrtz:
16610 case Intrinsic::amdgcn_cubeid:
16611 case Intrinsic::amdgcn_cubema:
16612 case Intrinsic::amdgcn_cubesc:
16613 case Intrinsic::amdgcn_cubetc:
16614 case Intrinsic::amdgcn_frexp_mant:
16615 case Intrinsic::amdgcn_fdot2:
16616 case Intrinsic::amdgcn_trig_preop:
16617 case Intrinsic::amdgcn_tanh:
16618 return true;
16619 default:
16620 break;
16621 }
16622
16623 [[fallthrough]];
16624 default:
16625 return false;
16626 }
16627
16628 llvm_unreachable("invalid operation");
16629}
16630
16631// Constant fold canonicalize.
16632SDValue SITargetLowering::getCanonicalConstantFP(SelectionDAG &DAG,
16633 const SDLoc &SL, EVT VT,
16634 const APFloat &C) const {
16635 // Flush denormals to 0 if not enabled.
16636 if (C.isDenormal()) {
16637 DenormalMode Mode =
16638 DAG.getMachineFunction().getDenormalMode(FPType: C.getSemantics());
16639 if (Mode == DenormalMode::getPreserveSign()) {
16640 return DAG.getConstantFP(
16641 Val: APFloat::getZero(Sem: C.getSemantics(), Negative: C.isNegative()), DL: SL, VT);
16642 }
16643
16644 if (Mode != DenormalMode::getIEEE())
16645 return SDValue();
16646 }
16647
16648 if (C.isNaN()) {
16649 if (C.isSignaling()) {
16650 // Quiet a signaling NaN.
16651 return DAG.getConstantFP(Val: C.makeQuiet(), DL: SL, VT);
16652 }
16653 }
16654
16655 // Already canonical.
16656 return DAG.getConstantFP(Val: C, DL: SL, VT);
16657}
16658
16659static bool vectorEltWillFoldAway(SDValue Op) {
16660 return Op.isUndef() || isa<ConstantFPSDNode>(Val: Op);
16661}
16662
16663SDValue
16664SITargetLowering::performFCanonicalizeCombine(SDNode *N,
16665 DAGCombinerInfo &DCI) const {
16666 SelectionDAG &DAG = DCI.DAG;
16667 SDValue N0 = N->getOperand(Num: 0);
16668 EVT VT = N->getValueType(ResNo: 0);
16669
16670 // fcanonicalize undef -> qnan
16671 if (N0.isUndef()) {
16672 APFloat QNaN = APFloat::getQNaN(Sem: VT.getFltSemantics());
16673 return DAG.getConstantFP(Val: QNaN, DL: SDLoc(N), VT);
16674 }
16675
16676 if (ConstantFPSDNode *CFP = isConstOrConstSplatFP(N: N0))
16677 return getCanonicalConstantFP(DAG, SL: SDLoc(N), VT, C: CFP->getValueAPF());
16678
16679 // fcanonicalize (build_vector x, k) -> build_vector (fcanonicalize x),
16680 // (fcanonicalize k)
16681 //
16682 // fcanonicalize (build_vector x, undef) -> build_vector (fcanonicalize x), 0
16683
16684 // TODO: This could be better with wider vectors that will be split to v2f16,
16685 // and to consider uses since there aren't that many packed operations.
16686 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNumOperands() == 2 &&
16687 isTypeLegal(VT)) {
16688 SDLoc SL(N);
16689 SDValue NewElts[2];
16690 SDValue Lo = N0.getOperand(i: 0);
16691 SDValue Hi = N0.getOperand(i: 1);
16692 EVT EltVT = Lo.getValueType();
16693
16694 // Only apply this optimization if scalar canonicalize is legal for the
16695 // element type. Otherwise, scalarizing may require widening the scalar back
16696 // to a vector, adding overhead (e.g., bf16 has no scalar instructions).
16697 if (getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT) != Legal)
16698 return SDValue();
16699
16700 if (vectorEltWillFoldAway(Op: Lo) || vectorEltWillFoldAway(Op: Hi)) {
16701 for (unsigned I = 0; I != 2; ++I) {
16702 SDValue Op = N0.getOperand(i: I);
16703 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16704 NewElts[I] =
16705 getCanonicalConstantFP(DAG, SL, VT: EltVT, C: CFP->getValueAPF());
16706 } else if (Op.isUndef()) {
16707 // Handled below based on what the other operand is.
16708 NewElts[I] = Op;
16709 } else {
16710 NewElts[I] = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: SL, VT: EltVT, Operand: Op);
16711 }
16712 }
16713
16714 // If one half is undef, and one is constant, prefer a splat vector.
16715 // Otherwise, convert the undef to 0.0 since that's cheaper to use and may
16716 // be free with a packed operation.
16717 if (NewElts[0].isUndef()) {
16718 NewElts[0] = isa<ConstantFPSDNode>(Val: NewElts[1])
16719 ? NewElts[1]
16720 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16721 }
16722
16723 if (NewElts[1].isUndef()) {
16724 NewElts[1] = isa<ConstantFPSDNode>(Val: NewElts[0])
16725 ? NewElts[0]
16726 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16727 }
16728
16729 return DAG.getBuildVector(VT, DL: SL, Ops: NewElts);
16730 }
16731 }
16732
16733 return SDValue();
16734}
16735
16736static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
16737 switch (Opc) {
16738 case ISD::FMAXNUM:
16739 case ISD::FMAXNUM_IEEE:
16740 case ISD::FMAXIMUMNUM:
16741 return AMDGPUISD::FMAX3;
16742 case ISD::FMAXIMUM:
16743 return AMDGPUISD::FMAXIMUM3;
16744 case ISD::SMAX:
16745 return AMDGPUISD::SMAX3;
16746 case ISD::UMAX:
16747 return AMDGPUISD::UMAX3;
16748 case ISD::FMINNUM:
16749 case ISD::FMINNUM_IEEE:
16750 case ISD::FMINIMUMNUM:
16751 return AMDGPUISD::FMIN3;
16752 case ISD::FMINIMUM:
16753 return AMDGPUISD::FMINIMUM3;
16754 case ISD::SMIN:
16755 return AMDGPUISD::SMIN3;
16756 case ISD::UMIN:
16757 return AMDGPUISD::UMIN3;
16758 default:
16759 llvm_unreachable("Not a min/max opcode");
16760 }
16761}
16762
16763SDValue SITargetLowering::performIntMed3ImmCombine(SelectionDAG &DAG,
16764 const SDLoc &SL, SDValue Src,
16765 SDValue MinVal,
16766 SDValue MaxVal,
16767 bool Signed) const {
16768
16769 // med3 comes from
16770 // min(max(x, K0), K1), K0 < K1
16771 // max(min(x, K0), K1), K1 < K0
16772 //
16773 // "MinVal" and "MaxVal" respectively refer to the rhs of the
16774 // min/max op.
16775 ConstantSDNode *MinK = dyn_cast<ConstantSDNode>(Val&: MinVal);
16776 ConstantSDNode *MaxK = dyn_cast<ConstantSDNode>(Val&: MaxVal);
16777
16778 if (!MinK || !MaxK)
16779 return SDValue();
16780
16781 if (Signed) {
16782 if (MaxK->getAPIntValue().sge(RHS: MinK->getAPIntValue()))
16783 return SDValue();
16784 } else {
16785 if (MaxK->getAPIntValue().uge(RHS: MinK->getAPIntValue()))
16786 return SDValue();
16787 }
16788
16789 EVT VT = MinK->getValueType(ResNo: 0);
16790 unsigned Med3Opc = Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3;
16791 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16()))
16792 return DAG.getNode(Opcode: Med3Opc, DL: SL, VT, N1: Src, N2: MaxVal, N3: MinVal);
16793
16794 // Note: we could also extend to i32 and use i32 med3 if i16 med3 is
16795 // not available, but this is unlikely to be profitable as constants
16796 // will often need to be materialized & extended, especially on
16797 // pre-GFX10 where VOP3 instructions couldn't take literal operands.
16798 return SDValue();
16799}
16800
16801static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
16802 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op))
16803 return C;
16804
16805 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
16806 if (ConstantFPSDNode *C = BV->getConstantFPSplatNode())
16807 return C;
16808 }
16809
16810 return nullptr;
16811}
16812
16813SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
16814 const SDLoc &SL, SDValue Op0,
16815 SDValue Op1,
16816 bool IsKnownNoNaNs) const {
16817 ConstantFPSDNode *K1 = getSplatConstantFP(Op: Op1);
16818 if (!K1)
16819 return SDValue();
16820
16821 ConstantFPSDNode *K0 = getSplatConstantFP(Op: Op0.getOperand(i: 1));
16822 if (!K0)
16823 return SDValue();
16824
16825 // Ordered >= (although NaN inputs should have folded away by now).
16826 if (K0->getValueAPF() > K1->getValueAPF())
16827 return SDValue();
16828
16829 // med3 with a nan input acts like
16830 // v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)
16831 //
16832 // So the result depends on whether the IEEE mode bit is enabled or not with a
16833 // signaling nan input.
16834 // ieee=1
16835 // s0 snan: yields s2
16836 // s1 snan: yields s2
16837 // s2 snan: qnan
16838
16839 // s0 qnan: min(s1, s2)
16840 // s1 qnan: min(s0, s2)
16841 // s2 qnan: min(s0, s1)
16842
16843 // ieee=0
16844 // s0 snan: min(s1, s2)
16845 // s1 snan: min(s0, s2)
16846 // s2 snan: qnan
16847
16848 // s0 qnan: min(s1, s2)
16849 // s1 qnan: min(s0, s2)
16850 // s2 qnan: min(s0, s1)
16851 const MachineFunction &MF = DAG.getMachineFunction();
16852 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16853
16854 // TODO: Check IEEE bit enabled. We can form fmed3 with IEEE=0 regardless of
16855 // whether the input is a signaling nan if op0 is fmaximum or fmaximumnum. We
16856 // can only form if op0 is fmaxnum_ieee if IEEE=1.
16857 EVT VT = Op0.getValueType();
16858 if (Info->getMode().DX10Clamp) {
16859 // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the
16860 // hardware fmed3 behavior converting to a min.
16861 // FIXME: Should this be allowing -0.0?
16862 if (K1->isOne() && K0->isPosZero())
16863 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Op0.getOperand(i: 0));
16864 }
16865
16866 // med3 for f16 is only available on gfx9+, and not available for v2f16.
16867 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) {
16868 // This isn't safe with signaling NaNs because in IEEE mode, min/max on a
16869 // signaling NaN gives a quiet NaN. The quiet NaN input to the min would
16870 // then give the other result, which is different from med3 with a NaN
16871 // input.
16872 SDValue Var = Op0.getOperand(i: 0);
16873 if (!IsKnownNoNaNs && !DAG.isKnownNeverSNaN(Op: Var))
16874 return SDValue();
16875
16876 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
16877
16878 if ((!K0->hasOneUse() || TII->isInlineConstant(Imm: K0->getValueAPF())) &&
16879 (!K1->hasOneUse() || TII->isInlineConstant(Imm: K1->getValueAPF()))) {
16880 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT: K0->getValueType(ResNo: 0), N1: Var,
16881 N2: SDValue(K0, 0), N3: SDValue(K1, 0));
16882 }
16883 }
16884
16885 return SDValue();
16886}
16887
16888/// \return true if the subtarget supports minimum3 and maximum3 with the given
16889/// base min/max opcode \p Opc for type \p VT.
16890static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc,
16891 EVT VT) {
16892 switch (Opc) {
16893 case ISD::FMINNUM:
16894 case ISD::FMAXNUM:
16895 case ISD::FMINNUM_IEEE:
16896 case ISD::FMAXNUM_IEEE:
16897 case ISD::FMINIMUMNUM:
16898 case ISD::FMAXIMUMNUM:
16899 case AMDGPUISD::FMIN_LEGACY:
16900 case AMDGPUISD::FMAX_LEGACY:
16901 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()) ||
16902 (VT == MVT::v2f16 && Subtarget.hasMin3Max3PKF16());
16903 case ISD::FMINIMUM:
16904 case ISD::FMAXIMUM:
16905 return (VT == MVT::f32 && Subtarget.hasMinimum3Maximum3F32()) ||
16906 (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()) ||
16907 (VT == MVT::v2f16 && Subtarget.hasMinimum3Maximum3PKF16());
16908 case ISD::SMAX:
16909 case ISD::SMIN:
16910 case ISD::UMAX:
16911 case ISD::UMIN:
16912 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.hasMin3Max3_16());
16913 default:
16914 return false;
16915 }
16916
16917 llvm_unreachable("not a min/max opcode");
16918}
16919
16920SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
16921 DAGCombinerInfo &DCI) const {
16922 SelectionDAG &DAG = DCI.DAG;
16923
16924 EVT VT = N->getValueType(ResNo: 0);
16925 unsigned Opc = N->getOpcode();
16926 SDValue Op0 = N->getOperand(Num: 0);
16927 SDValue Op1 = N->getOperand(Num: 1);
16928
16929 // Only do this if the inner op has one use since this will just increases
16930 // register pressure for no benefit.
16931
16932 if (supportsMin3Max3(Subtarget: *Subtarget, Opc, VT)) {
16933 auto IsTreeWithCombinableChildren = [Opc](SDValue Op) {
16934 return (Op.getOperand(i: 0).getOpcode() == Opc &&
16935 Op.getOperand(i: 0).hasOneUse()) ||
16936 (Op.getOperand(i: 1).getOpcode() == Opc &&
16937 Op.getOperand(i: 1).hasOneUse());
16938 };
16939
16940 bool CanTreeCombineApply = Op0.getOpcode() == Opc && Op0.hasOneUse() &&
16941 Op1.getOpcode() == Opc && Op1.hasOneUse();
16942 bool HasCombinableTreeChild =
16943 CanTreeCombineApply && (IsTreeWithCombinableChildren(Op0) ||
16944 IsTreeWithCombinableChildren(Op1));
16945
16946 // Tree reduction: when both operands are the same min/max op, restructure
16947 // to keep a 2-op node on top so higher tree levels can still combine.
16948 //
16949 // max(max(a, b), max(c, d)) -> max(max3(a, b, c), d)
16950 // min(min(a, b), min(c, d)) -> min(min3(a, b, c), d)
16951 //
16952 // Defer when either inner op is a tree node with combinable children.
16953 if (CanTreeCombineApply && !HasCombinableTreeChild) {
16954 SDLoc DL(N);
16955 SDValue Inner =
16956 DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT, N1: Op0.getOperand(i: 0),
16957 N2: Op0.getOperand(i: 1), N3: Op1.getOperand(i: 0));
16958 return DAG.getNode(Opcode: Opc, DL, VT, N1: Inner, N2: Op1.getOperand(i: 1));
16959 }
16960
16961 // max(max(a, b), c) -> max3(a, b, c)
16962 // min(min(a, b), c) -> min3(a, b, c)
16963 // Deferred when Op0 is a tree node with combinable children.
16964 if (Op0.getOpcode() == Opc && Op0.hasOneUse() && !HasCombinableTreeChild) {
16965 SDLoc DL(N);
16966 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16967 N1: Op0.getOperand(i: 0), N2: Op0.getOperand(i: 1), N3: Op1);
16968 }
16969
16970 // Try commuted.
16971 // max(a, max(b, c)) -> max3(a, b, c)
16972 // min(a, min(b, c)) -> min3(a, b, c)
16973 // Deferred when Op1 is a tree node with combinable children.
16974 if (Op1.getOpcode() == Opc && Op1.hasOneUse() && !HasCombinableTreeChild) {
16975 SDLoc DL(N);
16976 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16977 N1: Op0, N2: Op1.getOperand(i: 0), N3: Op1.getOperand(i: 1));
16978 }
16979 }
16980
16981 // umin(sffbh(x), bitwidth) -> sffbh(x) if x is known to be not 0 or -1.
16982 SDValue FfbhSrc;
16983 uint64_t Clamp = 0;
16984 if (Opc == ISD::UMIN &&
16985 sd_match(N: Op0,
16986 P: m_IntrinsicWOChain<Intrinsic::amdgcn_sffbh>(Opnds: m_Value(N&: FfbhSrc))) &&
16987 sd_match(N: Op1, P: m_ConstInt(V&: Clamp))) {
16988 unsigned BitWidth = FfbhSrc.getValueType().getScalarSizeInBits();
16989 if (Clamp >= BitWidth) {
16990 KnownBits Known = DAG.computeKnownBits(Op: FfbhSrc);
16991 if (Known.isNonZero() && Known.Zero.getBoolValue())
16992 return Op0;
16993 }
16994 }
16995
16996 // min(max(x, K0), K1), K0 < K1 -> med3(x, K0, K1)
16997 // max(min(x, K0), K1), K1 < K0 -> med3(x, K1, K0)
16998 if (Opc == ISD::SMIN && Op0.getOpcode() == ISD::SMAX && Op0.hasOneUse()) {
16999 if (SDValue Med3 = performIntMed3ImmCombine(
17000 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: true))
17001 return Med3;
17002 }
17003 if (Opc == ISD::SMAX && Op0.getOpcode() == ISD::SMIN && Op0.hasOneUse()) {
17004 if (SDValue Med3 = performIntMed3ImmCombine(
17005 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: true))
17006 return Med3;
17007 }
17008
17009 if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
17010 if (SDValue Med3 = performIntMed3ImmCombine(
17011 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: false))
17012 return Med3;
17013 }
17014 if (Opc == ISD::UMAX && Op0.getOpcode() == ISD::UMIN && Op0.hasOneUse()) {
17015 if (SDValue Med3 = performIntMed3ImmCombine(
17016 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: false))
17017 return Med3;
17018 }
17019
17020 // if !is_snan(x):
17021 // fminnum(fmaxnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
17022 // fminnum_ieee(fmaxnum_ieee(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
17023 // fminnumnum(fmaxnumnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
17024 // fmin_legacy(fmax_legacy(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
17025 if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) ||
17026 (Opc == ISD::FMINNUM_IEEE && Op0.getOpcode() == ISD::FMAXNUM_IEEE) ||
17027 (Opc == ISD::FMINIMUMNUM && Op0.getOpcode() == ISD::FMAXIMUMNUM) ||
17028 (Opc == AMDGPUISD::FMIN_LEGACY &&
17029 Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
17030 (VT == MVT::f32 || VT == MVT::f64 ||
17031 (VT == MVT::f16 && Subtarget->has16BitInsts()) ||
17032 (VT == MVT::bf16 && Subtarget->hasBF16PackedInsts()) ||
17033 (VT == MVT::v2bf16 && Subtarget->hasBF16PackedInsts()) ||
17034 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) &&
17035 Op0.hasOneUse()) {
17036 if (SDValue Res = performFPMed3ImmCombine(DAG, SL: SDLoc(N), Op0, Op1,
17037 IsKnownNoNaNs: N->getFlags().hasNoNaNs()))
17038 return Res;
17039 }
17040
17041 // Prefer fminnum_ieee over fminimum. For gfx950, minimum/maximum are legal
17042 // for some types, but at a higher cost since it's implemented with a 3
17043 // operand form.
17044 const SDNodeFlags Flags = N->getFlags();
17045 if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) && Flags.hasNoNaNs() &&
17046 !Subtarget->hasIEEEMinimumMaximumInsts() &&
17047 isOperationLegal(Op: ISD::FMINNUM_IEEE, VT: VT.getScalarType())) {
17048 unsigned NewOpc =
17049 Opc == ISD::FMINIMUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
17050 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(N), VT, N1: Op0, N2: Op1, Flags);
17051 }
17052
17053 return SDValue();
17054}
17055
17056static bool isClampZeroToOne(SDValue A, SDValue B) {
17057 if (ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A)) {
17058 if (ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B)) {
17059 // FIXME: Should this be allowing -0.0?
17060 return (CA->isPosZero() && CB->isOne()) ||
17061 (CA->isOne() && CB->isPosZero());
17062 }
17063 }
17064
17065 return false;
17066}
17067
17068// FIXME: Should only worry about snans for version with chain.
17069SDValue SITargetLowering::performFMed3Combine(SDNode *N,
17070 DAGCombinerInfo &DCI) const {
17071 EVT VT = N->getValueType(ResNo: 0);
17072 // v_med3_f32 and v_max_f32 behave identically wrt denorms, exceptions and
17073 // NaNs. With a NaN input, the order of the operands may change the result.
17074
17075 SelectionDAG &DAG = DCI.DAG;
17076 SDLoc SL(N);
17077
17078 SDValue Src0 = N->getOperand(Num: 0);
17079 SDValue Src1 = N->getOperand(Num: 1);
17080 SDValue Src2 = N->getOperand(Num: 2);
17081
17082 if (isClampZeroToOne(A: Src0, B: Src1)) {
17083 // const_a, const_b, x -> clamp is safe in all cases including signaling
17084 // nans.
17085 // FIXME: Should this be allowing -0.0?
17086 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src2);
17087 }
17088
17089 const MachineFunction &MF = DAG.getMachineFunction();
17090 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
17091
17092 // FIXME: dx10_clamp behavior assumed in instcombine. Should we really bother
17093 // handling no dx10-clamp?
17094 if (Info->getMode().DX10Clamp) {
17095 // If NaNs is clamped to 0, we are free to reorder the inputs.
17096
17097 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
17098 std::swap(a&: Src0, b&: Src1);
17099
17100 if (isa<ConstantFPSDNode>(Val: Src1) && !isa<ConstantFPSDNode>(Val: Src2))
17101 std::swap(a&: Src1, b&: Src2);
17102
17103 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
17104 std::swap(a&: Src0, b&: Src1);
17105
17106 if (isClampZeroToOne(A: Src1, B: Src2))
17107 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src0);
17108 }
17109
17110 return SDValue();
17111}
17112
17113SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
17114 DAGCombinerInfo &DCI) const {
17115 SDValue Src0 = N->getOperand(Num: 0);
17116 SDValue Src1 = N->getOperand(Num: 1);
17117 if (Src0.isUndef() && Src1.isUndef())
17118 return DCI.DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
17119 return SDValue();
17120}
17121
17122// Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be
17123// expanded into a set of cmp/select instructions.
17124bool SITargetLowering::shouldExpandVectorDynExt(unsigned EltSize,
17125 unsigned NumElem,
17126 bool IsDivergentIdx,
17127 const GCNSubtarget *Subtarget) {
17128 if (UseDivergentRegisterIndexing)
17129 return false;
17130
17131 unsigned VecSize = EltSize * NumElem;
17132
17133 // Sub-dword vectors of size 2 dword or less have better implementation.
17134 if (VecSize <= 64 && EltSize < 32)
17135 return false;
17136
17137 // Always expand the rest of sub-dword instructions, otherwise it will be
17138 // lowered via memory.
17139 if (EltSize < 32)
17140 return true;
17141
17142 // Always do this if var-idx is divergent, otherwise it will become a loop.
17143 if (IsDivergentIdx)
17144 return true;
17145
17146 // Large vectors would yield too many compares and v_cndmask_b32 instructions.
17147 unsigned NumInsts = NumElem /* Number of compares */ +
17148 ((EltSize + 31) / 32) * NumElem /* Number of cndmasks */;
17149
17150 // On some architectures (GFX9) movrel is not available and it's better
17151 // to expand.
17152 if (Subtarget->useVGPRIndexMode())
17153 return NumInsts <= 16;
17154
17155 // If movrel is available, use it instead of expanding for vector of 8
17156 // elements.
17157 if (Subtarget->hasMovrel())
17158 return NumInsts <= 15;
17159
17160 return true;
17161}
17162
17163bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
17164 SDValue Idx = N->getOperand(Num: N->getNumOperands() - 1);
17165 if (isa<ConstantSDNode>(Val: Idx))
17166 return false;
17167
17168 SDValue Vec = N->getOperand(Num: 0);
17169 EVT VecVT = Vec.getValueType();
17170 EVT EltVT = VecVT.getVectorElementType();
17171 unsigned EltSize = EltVT.getSizeInBits();
17172 unsigned NumElem = VecVT.getVectorNumElements();
17173
17174 return SITargetLowering::shouldExpandVectorDynExt(
17175 EltSize, NumElem, IsDivergentIdx: Idx->isDivergent(), Subtarget: getSubtarget());
17176}
17177
17178SDValue
17179SITargetLowering::performExtractVectorEltCombine(SDNode *N,
17180 DAGCombinerInfo &DCI) const {
17181 SDValue Vec = N->getOperand(Num: 0);
17182 SelectionDAG &DAG = DCI.DAG;
17183
17184 EVT VecVT = Vec.getValueType();
17185 EVT VecEltVT = VecVT.getVectorElementType();
17186 EVT ResVT = N->getValueType(ResNo: 0);
17187
17188 unsigned VecSize = VecVT.getSizeInBits();
17189 unsigned VecEltSize = VecEltVT.getSizeInBits();
17190
17191 if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
17192 allUsesHaveSourceMods(N)) {
17193 SDLoc SL(N);
17194 SDValue Idx = N->getOperand(Num: 1);
17195 SDValue Elt =
17196 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec.getOperand(i: 0), N2: Idx);
17197 return DAG.getNode(Opcode: Vec.getOpcode(), DL: SL, VT: ResVT, Operand: Elt);
17198 }
17199
17200 // (extract_vector_element (and {y0, y1}, (build_vector 0x1f, 0x1f)), index)
17201 // -> (and (extract_vector_element {y0, y1}, index), 0x1f)
17202 // There are optimisations to transform 64-bit shifts into 32-bit shifts
17203 // depending on the shift operand. See e.g. performSraCombine().
17204 // This combine ensures that the optimisation is compatible with v2i32
17205 // legalised AND.
17206 if (VecVT == MVT::v2i32 && Vec->getOpcode() == ISD::AND &&
17207 Vec->getOperand(Num: 1)->getOpcode() == ISD::BUILD_VECTOR) {
17208
17209 const ConstantSDNode *C = isConstOrConstSplat(N: Vec.getOperand(i: 1));
17210 if (!C || C->getZExtValue() != 0x1f)
17211 return SDValue();
17212
17213 SDLoc SL(N);
17214 SDValue AndMask = DAG.getConstant(Val: 0x1f, DL: SL, VT: MVT::i32);
17215 SDValue EVE = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32,
17216 N1: Vec->getOperand(Num: 0), N2: N->getOperand(Num: 1));
17217 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: EVE, N2: AndMask);
17218 DAG.ReplaceAllUsesWith(From: N, To: A.getNode());
17219 }
17220
17221 // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
17222 // =>
17223 // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
17224 // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
17225 // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
17226 if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
17227 SDLoc SL(N);
17228 SDValue Idx = N->getOperand(Num: 1);
17229 unsigned Opc = Vec.getOpcode();
17230
17231 switch (Opc) {
17232 default:
17233 break;
17234 // TODO: Support other binary operations.
17235 case ISD::FADD:
17236 case ISD::FSUB:
17237 case ISD::FMUL:
17238 case ISD::ADD:
17239 case ISD::UMIN:
17240 case ISD::UMAX:
17241 case ISD::SMIN:
17242 case ISD::SMAX:
17243 case ISD::FMAXNUM:
17244 case ISD::FMINNUM:
17245 case ISD::FMAXNUM_IEEE:
17246 case ISD::FMINNUM_IEEE:
17247 case ISD::FMAXIMUM:
17248 case ISD::FMINIMUM: {
17249 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
17250 N1: Vec.getOperand(i: 0), N2: Idx);
17251 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
17252 N1: Vec.getOperand(i: 1), N2: Idx);
17253
17254 DCI.AddToWorklist(N: Elt0.getNode());
17255 DCI.AddToWorklist(N: Elt1.getNode());
17256 return DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT, N1: Elt0, N2: Elt1, Flags: Vec->getFlags());
17257 }
17258 }
17259 }
17260
17261 // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
17262 if (shouldExpandVectorDynExt(N)) {
17263 SDLoc SL(N);
17264 SDValue Idx = N->getOperand(Num: 1);
17265 SDValue V;
17266 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17267 SDValue IC = DAG.getVectorIdxConstant(Val: I, DL: SL);
17268 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec, N2: IC);
17269 if (I == 0)
17270 V = Elt;
17271 else
17272 V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Elt, False: V, Cond: ISD::SETEQ);
17273 }
17274 return V;
17275 }
17276
17277 // EXTRACT_VECTOR_ELT (v2i32 bitcast (i64/f64:k), Idx)
17278 // =>
17279 // i32:Lo(k) if Idx == 0, or
17280 // i32:Hi(k) if Idx == 1
17281 auto *Idx = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17282 if (Vec.getOpcode() == ISD::BITCAST && VecVT == MVT::v2i32 && Idx) {
17283 SDLoc SL(N);
17284 SDValue PeekThrough = Vec.getOperand(i: 0);
17285 auto *KImm = dyn_cast<ConstantSDNode>(Val&: PeekThrough);
17286 if (KImm && KImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
17287 uint64_t KImmValue = KImm->getZExtValue();
17288 return DAG.getConstant(
17289 Val: (KImmValue >> (32 * Idx->getZExtValue())) & 0xffffffff, DL: SL, VT: MVT::i32);
17290 }
17291 auto *KFPImm = dyn_cast<ConstantFPSDNode>(Val&: PeekThrough);
17292 if (KFPImm && KFPImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
17293 uint64_t KFPImmValue =
17294 KFPImm->getValueAPF().bitcastToAPInt().getZExtValue();
17295 return DAG.getConstant(Val: (KFPImmValue >> (32 * Idx->getZExtValue())) &
17296 0xffffffff,
17297 DL: SL, VT: MVT::i32);
17298 }
17299 }
17300
17301 if (!DCI.isBeforeLegalize())
17302 return SDValue();
17303
17304 // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
17305 // elements. This exposes more load reduction opportunities by replacing
17306 // multiple small extract_vector_elements with a single 32-bit extract.
17307 if (isa<MemSDNode>(Val: Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
17308 VecSize > 32 && VecSize % 32 == 0 && Idx) {
17309 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: VecVT);
17310
17311 unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
17312 unsigned EltIdx = BitIndex / 32;
17313 unsigned LeftoverBitIdx = BitIndex % 32;
17314 SDLoc SL(N);
17315
17316 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: Vec);
17317 DCI.AddToWorklist(N: Cast.getNode());
17318
17319 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
17320 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
17321 DCI.AddToWorklist(N: Elt.getNode());
17322 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Elt,
17323 N2: DAG.getConstant(Val: LeftoverBitIdx, DL: SL, VT: MVT::i32));
17324 DCI.AddToWorklist(N: Srl.getNode());
17325
17326 EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
17327 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: VecEltAsIntVT, Operand: Srl);
17328 DCI.AddToWorklist(N: Trunc.getNode());
17329
17330 if (VecEltVT == ResVT) {
17331 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecEltVT, Operand: Trunc);
17332 }
17333
17334 assert(ResVT.isScalarInteger());
17335 return DAG.getAnyExtOrTrunc(Op: Trunc, DL: SL, VT: ResVT);
17336 }
17337
17338 return SDValue();
17339}
17340
17341SDValue
17342SITargetLowering::performInsertVectorEltCombine(SDNode *N,
17343 DAGCombinerInfo &DCI) const {
17344 SDValue Vec = N->getOperand(Num: 0);
17345 SDValue Idx = N->getOperand(Num: 2);
17346 EVT VecVT = Vec.getValueType();
17347 EVT EltVT = VecVT.getVectorElementType();
17348
17349 // INSERT_VECTOR_ELT (<n x e>, var-idx)
17350 // => BUILD_VECTOR n x select (e, const-idx)
17351 if (!shouldExpandVectorDynExt(N))
17352 return SDValue();
17353
17354 SelectionDAG &DAG = DCI.DAG;
17355 SDLoc SL(N);
17356 SDValue Ins = N->getOperand(Num: 1);
17357 EVT IdxVT = Idx.getValueType();
17358
17359 SmallVector<SDValue, 16> Ops;
17360 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17361 SDValue IC = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
17362 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec, N2: IC);
17363 SDValue V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Ins, False: Elt, Cond: ISD::SETEQ);
17364 Ops.push_back(Elt: V);
17365 }
17366
17367 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops);
17368}
17369
17370/// Return the source of an fp_extend from f16 to f32, or a converted FP
17371/// constant.
17372static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src) {
17373 if (Src.getOpcode() == ISD::FP_EXTEND &&
17374 Src.getOperand(i: 0).getValueType() == MVT::f16) {
17375 return Src.getOperand(i: 0);
17376 }
17377
17378 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
17379 APFloat Val = CFP->getValueAPF();
17380 bool LosesInfo = true;
17381 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
17382 if (!LosesInfo)
17383 return DAG.getConstantFP(Val, DL: SDLoc(Src), VT: MVT::f16);
17384 }
17385
17386 return SDValue();
17387}
17388
17389SDValue SITargetLowering::performFPRoundCombine(SDNode *N,
17390 DAGCombinerInfo &DCI) const {
17391 assert(Subtarget->has16BitInsts() && !Subtarget->hasMed3_16() &&
17392 "combine only useful on gfx8");
17393
17394 SDValue TruncSrc = N->getOperand(Num: 0);
17395 EVT VT = N->getValueType(ResNo: 0);
17396 if (VT != MVT::f16)
17397 return SDValue();
17398
17399 if (TruncSrc.getOpcode() != AMDGPUISD::FMED3 ||
17400 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse())
17401 return SDValue();
17402
17403 SelectionDAG &DAG = DCI.DAG;
17404 SDLoc SL(N);
17405
17406 // Optimize f16 fmed3 pattern performed on f32. On gfx8 there is no f16 fmed3,
17407 // and expanding it with min/max saves 1 instruction vs. casting to f32 and
17408 // casting back.
17409
17410 // fptrunc (f32 (fmed3 (fpext f16:a, fpext f16:b, fpext f16:c))) =>
17411 // fmin(fmax(a, b), fmax(fmin(a, b), c))
17412 SDValue A = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 0));
17413 if (!A)
17414 return SDValue();
17415
17416 SDValue B = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 1));
17417 if (!B)
17418 return SDValue();
17419
17420 SDValue C = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 2));
17421 if (!C)
17422 return SDValue();
17423
17424 // This changes signaling nan behavior. If an input is a signaling nan, it
17425 // would have been quieted by the fpext originally. We don't care because
17426 // these are unconstrained ops. If we needed to insert quieting canonicalizes
17427 // we would be worse off than just doing the promotion.
17428 SDValue A1 = DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: A, N2: B);
17429 SDValue B1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A, N2: B);
17430 SDValue C1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A1, N2: C);
17431 return DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: B1, N2: C1);
17432}
17433
17434unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG,
17435 const SDNode *N0,
17436 const SDNode *N1) const {
17437 EVT VT = N0->getValueType(ResNo: 0);
17438
17439 // Only do this if we are not trying to support denormals. v_mad_f32 does not
17440 // support denormals ever.
17441 if (((VT == MVT::f32 &&
17442 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction())) ||
17443 (VT == MVT::f16 && Subtarget->hasMadF16() &&
17444 denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction()))) &&
17445 isOperationLegal(Op: ISD::FMAD, VT))
17446 return ISD::FMAD;
17447
17448 if (N0->getFlags().hasAllowContract() && N1->getFlags().hasAllowContract() &&
17449 isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
17450 return ISD::FMA;
17451 }
17452
17453 return 0;
17454}
17455
17456// For a reassociatable opcode perform:
17457// op x, (op y, z) -> op (op x, z), y, if x and z are uniform
17458SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
17459 SelectionDAG &DAG) const {
17460 EVT VT = N->getValueType(ResNo: 0);
17461 if (VT != MVT::i32 && VT != MVT::i64)
17462 return SDValue();
17463
17464 if (DAG.isBaseWithConstantOffset(Op: SDValue(N, 0)))
17465 return SDValue();
17466
17467 unsigned Opc = N->getOpcode();
17468 SDValue Op0 = N->getOperand(Num: 0);
17469 SDValue Op1 = N->getOperand(Num: 1);
17470
17471 if (!(Op0->isDivergent() ^ Op1->isDivergent()))
17472 return SDValue();
17473
17474 if (Op0->isDivergent())
17475 std::swap(a&: Op0, b&: Op1);
17476
17477 if (Op1.getOpcode() != Opc || !Op1.hasOneUse())
17478 return SDValue();
17479
17480 SDValue Op2 = Op1.getOperand(i: 1);
17481 Op1 = Op1.getOperand(i: 0);
17482 if (!(Op1->isDivergent() ^ Op2->isDivergent()))
17483 return SDValue();
17484
17485 if (Op1->isDivergent())
17486 std::swap(a&: Op1, b&: Op2);
17487
17488 SDLoc SL(N);
17489 SDValue Add1 = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Op0, N2: Op1);
17490 return DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Add1, N2: Op2);
17491}
17492
17493static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
17494 SDValue N0, SDValue N1, SDValue N2, bool Signed) {
17495 unsigned MadOpc = Signed ? AMDGPUISD::MAD_I64_I32 : AMDGPUISD::MAD_U64_U32;
17496 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i1);
17497 SDValue Mad = DAG.getNode(Opcode: MadOpc, DL: SL, VTList: VTs, N1: N0, N2: N1, N3: N2);
17498 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Mad);
17499}
17500
17501// Fold
17502// y = lshr i64 x, 32
17503// res = add (mul i64 y, Const), x where "Const" is a 64-bit constant
17504// with Const.hi == -1
17505// To
17506// res = mad_u64_u32 y.lo ,Const.lo, x.lo
17507static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
17508 SDValue MulLHS, SDValue MulRHS,
17509 SDValue AddRHS) {
17510 if (MulRHS.getOpcode() == ISD::SRL)
17511 std::swap(a&: MulLHS, b&: MulRHS);
17512
17513 if (MulLHS.getValueType() != MVT::i64 || MulLHS.getOpcode() != ISD::SRL)
17514 return SDValue();
17515
17516 ConstantSDNode *ShiftVal = dyn_cast<ConstantSDNode>(Val: MulLHS.getOperand(i: 1));
17517 if (!ShiftVal || ShiftVal->getAsZExtVal() != 32 ||
17518 MulLHS.getOperand(i: 0) != AddRHS)
17519 return SDValue();
17520
17521 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: MulRHS.getNode());
17522 if (!Const || Hi_32(Value: Const->getZExtValue()) != uint32_t(-1))
17523 return SDValue();
17524
17525 SDValue ConstMul =
17526 DAG.getConstant(Val: Lo_32(Value: Const->getZExtValue()), DL: SL, VT: MVT::i32);
17527 return getMad64_32(DAG, SL, VT: MVT::i64,
17528 N0: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS), N1: ConstMul,
17529 N2: DAG.getZeroExtendInReg(Op: AddRHS, DL: SL, VT: MVT::i32), Signed: false);
17530}
17531
17532// Fold (add (mul x, y), z) --> (mad_[iu]64_[iu]32 x, y, z) plus high
17533// multiplies, if any.
17534//
17535// Full 64-bit multiplies that feed into an addition are lowered here instead
17536// of using the generic expansion. The generic expansion ends up with
17537// a tree of ADD nodes that prevents us from using the "add" part of the
17538// MAD instruction. The expansion produced here results in a chain of ADDs
17539// instead of a tree.
17540SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
17541 DAGCombinerInfo &DCI) const {
17542 assert(N->isAnyAdd());
17543
17544 SelectionDAG &DAG = DCI.DAG;
17545 EVT VT = N->getValueType(ResNo: 0);
17546 SDLoc SL(N);
17547 SDValue LHS = N->getOperand(Num: 0);
17548 SDValue RHS = N->getOperand(Num: 1);
17549
17550 if (VT.isVector())
17551 return SDValue();
17552
17553 // S_MUL_HI_[IU]32 was added in gfx9, which allows us to keep the overall
17554 // result in scalar registers for uniform values.
17555 if (!N->isDivergent() && Subtarget->hasSMulHi())
17556 return SDValue();
17557
17558 unsigned NumBits = VT.getScalarSizeInBits();
17559 if (NumBits <= 32 || NumBits > 64)
17560 return SDValue();
17561
17562 if (LHS.getOpcode() != ISD::MUL) {
17563 assert(RHS.getOpcode() == ISD::MUL);
17564 std::swap(a&: LHS, b&: RHS);
17565 }
17566
17567 // Avoid the fold if it would unduly increase the number of multiplies due to
17568 // multiple uses, except on hardware with full-rate multiply-add (which is
17569 // part of full-rate 64-bit ops).
17570 if (!Subtarget->hasFullRate64Ops()) {
17571 unsigned NumUsers = 0;
17572 for (SDNode *User : LHS->users()) {
17573 // There is a use that does not feed into addition, so the multiply can't
17574 // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
17575 if (!User->isAnyAdd())
17576 return SDValue();
17577
17578 // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
17579 // MUL + 3xADD + 3xADDC over 3xMAD.
17580 ++NumUsers;
17581 if (NumUsers >= 3)
17582 return SDValue();
17583 }
17584 }
17585
17586 SDValue MulLHS = LHS.getOperand(i: 0);
17587 SDValue MulRHS = LHS.getOperand(i: 1);
17588 SDValue AddRHS = RHS;
17589
17590 if (SDValue FoldedMAD = tryFoldMADwithSRL(DAG, SL, MulLHS, MulRHS, AddRHS))
17591 return FoldedMAD;
17592
17593 // Always check whether operands are small unsigned values, since that
17594 // knowledge is useful in more cases. Check for small signed values only if
17595 // doing so can unlock a shorter code sequence.
17596 bool MulLHSUnsigned32 = numBitsUnsigned(Op: MulLHS, DAG) <= 32;
17597 bool MulRHSUnsigned32 = numBitsUnsigned(Op: MulRHS, DAG) <= 32;
17598
17599 bool MulSignedLo = false;
17600 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
17601 MulSignedLo =
17602 numBitsSigned(Op: MulLHS, DAG) <= 32 && numBitsSigned(Op: MulRHS, DAG) <= 32;
17603 }
17604
17605 // The operands and final result all have the same number of bits. If
17606 // operands need to be extended, they can be extended with garbage. The
17607 // resulting garbage in the high bits of the mad_[iu]64_[iu]32 result is
17608 // truncated away in the end.
17609 if (VT != MVT::i64) {
17610 MulLHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulLHS);
17611 MulRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulRHS);
17612 AddRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: AddRHS);
17613 }
17614
17615 // The basic code generated is conceptually straightforward. Pseudo code:
17616 //
17617 // accum = mad_64_32 lhs.lo, rhs.lo, accum
17618 // accum.hi = add (mul lhs.hi, rhs.lo), accum.hi
17619 // accum.hi = add (mul lhs.lo, rhs.hi), accum.hi
17620 //
17621 // The second and third lines are optional, depending on whether the factors
17622 // are {sign,zero}-extended or not.
17623 //
17624 // The actual DAG is noisier than the pseudo code, but only due to
17625 // instructions that disassemble values into low and high parts, and
17626 // assemble the final result.
17627 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
17628
17629 auto MulLHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS);
17630 auto MulRHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulRHS);
17631 SDValue Accum =
17632 getMad64_32(DAG, SL, VT: MVT::i64, N0: MulLHSLo, N1: MulRHSLo, N2: AddRHS, Signed: MulSignedLo);
17633
17634 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
17635 auto [AccumLo, AccumHi] = DAG.SplitScalar(N: Accum, DL: SL, LoVT: MVT::i32, HiVT: MVT::i32);
17636
17637 if (!MulLHSUnsigned32) {
17638 auto MulLHSHi =
17639 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulLHS, N2: One);
17640 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSHi, N2: MulRHSLo);
17641 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17642 }
17643
17644 if (!MulRHSUnsigned32) {
17645 auto MulRHSHi =
17646 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulRHS, N2: One);
17647 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSLo, N2: MulRHSHi);
17648 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17649 }
17650
17651 Accum = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {AccumLo, AccumHi});
17652 Accum = DAG.getBitcast(VT: MVT::i64, V: Accum);
17653 }
17654
17655 if (VT != MVT::i64)
17656 Accum = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Accum);
17657 return Accum;
17658}
17659
17660SDValue
17661SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
17662 DAGCombinerInfo &DCI) const {
17663 SDValue RHS = N->getOperand(Num: 1);
17664 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
17665 if (!CRHS)
17666 return SDValue();
17667
17668 // TODO: Worth using computeKnownBits? Maybe expensive since it's so
17669 // common.
17670 uint64_t Val = CRHS->getZExtValue();
17671 if (countr_zero(Val) >= 32) {
17672 SelectionDAG &DAG = DCI.DAG;
17673 SDLoc SL(N);
17674 SDValue LHS = N->getOperand(Num: 0);
17675
17676 // Avoid carry machinery if we know the low half of the add does not
17677 // contribute to the final result.
17678 //
17679 // add i64:x, K if computeTrailingZeros(K) >= 32
17680 // => build_pair (add x.hi, K.hi), x.lo
17681
17682 // Breaking the 64-bit add here with this strange constant is unlikely
17683 // to interfere with addressing mode patterns.
17684
17685 SDValue Hi = getHiHalf64(Op: LHS, DAG);
17686 SDValue ConstHi32 = DAG.getConstant(Val: Hi_32(Value: Val), DL: SL, VT: MVT::i32);
17687 unsigned Opcode = N->getOpcode();
17688 if (Opcode == ISD::PTRADD)
17689 Opcode = ISD::ADD;
17690 SDValue AddHi =
17691 DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Hi, N2: ConstHi32, Flags: N->getFlags());
17692
17693 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: LHS);
17694 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: Lo, N2: AddHi);
17695 }
17696
17697 return SDValue();
17698}
17699
17700// Collect the ultimate src of each of the mul node's operands, and confirm
17701// each operand is 8 bytes.
17702static std::optional<ByteProvider<SDValue>>
17703handleMulOperand(const SDValue &MulOperand) {
17704 auto Byte0 = calculateByteProvider(Op: MulOperand, Index: 0, Depth: 0);
17705 if (!Byte0 || Byte0->isConstantZero()) {
17706 return std::nullopt;
17707 }
17708 auto Byte1 = calculateByteProvider(Op: MulOperand, Index: 1, Depth: 0);
17709 if (Byte1 && !Byte1->isConstantZero()) {
17710 return std::nullopt;
17711 }
17712 return Byte0;
17713}
17714
17715static unsigned addPermMasks(unsigned First, unsigned Second) {
17716 unsigned FirstCs = First & 0x0c0c0c0c;
17717 unsigned SecondCs = Second & 0x0c0c0c0c;
17718 unsigned FirstNoCs = First & ~0x0c0c0c0c;
17719 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
17720
17721 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
17722 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
17723 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
17724 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
17725
17726 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
17727}
17728
17729struct DotSrc {
17730 SDValue SrcOp;
17731 int64_t PermMask;
17732 int64_t DWordOffset;
17733};
17734
17735static void placeSources(ByteProvider<SDValue> &Src0,
17736 ByteProvider<SDValue> &Src1,
17737 SmallVectorImpl<DotSrc> &Src0s,
17738 SmallVectorImpl<DotSrc> &Src1s, int Step) {
17739
17740 assert(Src0.Src.has_value() && Src1.Src.has_value());
17741 // Src0s and Src1s are empty, just place arbitrarily.
17742 if (Step == 0) {
17743 Src0s.push_back(Elt: {.SrcOp: *Src0.Src, .PermMask: ((Src0.SrcOffset % 4) << 24) + 0x0c0c0c,
17744 .DWordOffset: Src0.SrcOffset / 4});
17745 Src1s.push_back(Elt: {.SrcOp: *Src1.Src, .PermMask: ((Src1.SrcOffset % 4) << 24) + 0x0c0c0c,
17746 .DWordOffset: Src1.SrcOffset / 4});
17747 return;
17748 }
17749
17750 for (int BPI = 0; BPI < 2; BPI++) {
17751 std::pair<ByteProvider<SDValue>, ByteProvider<SDValue>> BPP = {Src0, Src1};
17752 if (BPI == 1) {
17753 BPP = {Src1, Src0};
17754 }
17755 unsigned ZeroMask = 0x0c0c0c0c;
17756 unsigned FMask = 0xFF << (8 * (3 - Step));
17757
17758 unsigned FirstMask =
17759 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17760 unsigned SecondMask =
17761 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17762 // Attempt to find Src vector which contains our SDValue, if so, add our
17763 // perm mask to the existing one. If we are unable to find a match for the
17764 // first SDValue, attempt to find match for the second.
17765 int FirstGroup = -1;
17766 for (int I = 0; I < 2; I++) {
17767 SmallVectorImpl<DotSrc> &Srcs = I == 0 ? Src0s : Src1s;
17768 auto MatchesFirst = [&BPP](DotSrc &IterElt) {
17769 return IterElt.SrcOp == *BPP.first.Src &&
17770 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
17771 };
17772
17773 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesFirst);
17774 if (Match != Srcs.end()) {
17775 Match->PermMask = addPermMasks(First: FirstMask, Second: Match->PermMask);
17776 FirstGroup = I;
17777 break;
17778 }
17779 }
17780 if (FirstGroup != -1) {
17781 SmallVectorImpl<DotSrc> &Srcs = FirstGroup == 1 ? Src0s : Src1s;
17782 auto MatchesSecond = [&BPP](DotSrc &IterElt) {
17783 return IterElt.SrcOp == *BPP.second.Src &&
17784 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
17785 };
17786 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesSecond);
17787 if (Match != Srcs.end()) {
17788 Match->PermMask = addPermMasks(First: SecondMask, Second: Match->PermMask);
17789 } else
17790 Srcs.push_back(Elt: {.SrcOp: *BPP.second.Src, .PermMask: SecondMask, .DWordOffset: BPP.second.SrcOffset / 4});
17791 return;
17792 }
17793 }
17794
17795 // If we have made it here, then we could not find a match in Src0s or Src1s
17796 // for either Src0 or Src1, so just place them arbitrarily.
17797
17798 unsigned ZeroMask = 0x0c0c0c0c;
17799 unsigned FMask = 0xFF << (8 * (3 - Step));
17800
17801 Src0s.push_back(
17802 Elt: {.SrcOp: *Src0.Src,
17803 .PermMask: ((Src0.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17804 .DWordOffset: Src0.SrcOffset / 4});
17805 Src1s.push_back(
17806 Elt: {.SrcOp: *Src1.Src,
17807 .PermMask: ((Src1.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17808 .DWordOffset: Src1.SrcOffset / 4});
17809}
17810
17811static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
17812 SmallVectorImpl<DotSrc> &Srcs, bool IsSigned,
17813 bool IsAny) {
17814
17815 // If we just have one source, just permute it accordingly.
17816 if (Srcs.size() == 1) {
17817 auto *Elt = Srcs.begin();
17818 auto EltOp = getDWordFromOffset(DAG, SL, Src: Elt->SrcOp, DWordOffset: Elt->DWordOffset);
17819
17820 // v_perm will produce the original value
17821 if (Elt->PermMask == 0x3020100)
17822 return EltOp;
17823
17824 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17825 N3: DAG.getConstant(Val: Elt->PermMask, DL: SL, VT: MVT::i32));
17826 }
17827
17828 auto *FirstElt = Srcs.begin();
17829 auto *SecondElt = std::next(x: FirstElt);
17830
17831 SmallVector<SDValue, 2> Perms;
17832
17833 // If we have multiple sources in the chain, combine them via perms (using
17834 // calculated perm mask) and Ors.
17835 while (true) {
17836 auto FirstMask = FirstElt->PermMask;
17837 auto SecondMask = SecondElt->PermMask;
17838
17839 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
17840 unsigned FirstPlusFour = FirstMask | 0x04040404;
17841 // 0x0c + 0x04 = 0x10, so anding with 0x0F will produced 0x00 for any
17842 // original 0x0C.
17843 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
17844
17845 auto PermMask = addPermMasks(First: FirstMask, Second: SecondMask);
17846 auto FirstVal =
17847 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17848 auto SecondVal =
17849 getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp, DWordOffset: SecondElt->DWordOffset);
17850
17851 Perms.push_back(Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: FirstVal,
17852 N2: SecondVal,
17853 N3: DAG.getConstant(Val: PermMask, DL: SL, VT: MVT::i32)));
17854
17855 FirstElt = std::next(x: SecondElt);
17856 if (FirstElt == Srcs.end())
17857 break;
17858
17859 SecondElt = std::next(x: FirstElt);
17860 // If we only have a FirstElt, then just combine that into the cumulative
17861 // source node.
17862 if (SecondElt == Srcs.end()) {
17863 auto EltOp =
17864 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17865
17866 Perms.push_back(
17867 Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17868 N3: DAG.getConstant(Val: FirstElt->PermMask, DL: SL, VT: MVT::i32)));
17869 break;
17870 }
17871 }
17872
17873 assert(Perms.size() == 1 || Perms.size() == 2);
17874 return Perms.size() == 2
17875 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Perms[0], N2: Perms[1])
17876 : Perms[0];
17877}
17878
17879static void fixMasks(SmallVectorImpl<DotSrc> &Srcs, unsigned ChainLength) {
17880 for (auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
17881 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
17882 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
17883 EntryMask += ZeroMask;
17884 }
17885}
17886
17887static bool isMul(const SDValue Op) {
17888 auto Opcode = Op.getOpcode();
17889
17890 return (Opcode == ISD::MUL || Opcode == AMDGPUISD::MUL_U24 ||
17891 Opcode == AMDGPUISD::MUL_I24);
17892}
17893
17894static std::optional<bool>
17895checkDot4MulSignedness(const SDValue &N, ByteProvider<SDValue> &Src0,
17896 ByteProvider<SDValue> &Src1, const SDValue &S0Op,
17897 const SDValue &S1Op, const SelectionDAG &DAG) {
17898 // If we both ops are i8s (pre legalize-dag), then the signedness semantics
17899 // of the dot4 is irrelevant.
17900 if (S0Op.getValueSizeInBits() == 8 && S1Op.getValueSizeInBits() == 8)
17901 return false;
17902
17903 auto Known0 = DAG.computeKnownBits(Op: S0Op, Depth: 0);
17904 bool S0IsUnsigned = Known0.countMinLeadingZeros() > 0;
17905 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
17906 auto Known1 = DAG.computeKnownBits(Op: S1Op, Depth: 0);
17907 bool S1IsUnsigned = Known1.countMinLeadingZeros() > 0;
17908 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
17909
17910 assert(!(S0IsUnsigned && S0IsSigned));
17911 assert(!(S1IsUnsigned && S1IsSigned));
17912
17913 // There are 9 possible permutations of
17914 // {S0IsUnsigned, S0IsSigned, S1IsUnsigned, S1IsSigned}
17915
17916 // In two permutations, the sign bits are known to be the same for both Ops,
17917 // so simply return Signed / Unsigned corresponding to the MSB
17918
17919 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
17920 return S0IsSigned;
17921
17922 // In another two permutations, the sign bits are known to be opposite. In
17923 // this case return std::nullopt to indicate a bad match.
17924
17925 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
17926 return std::nullopt;
17927
17928 // In the remaining five permutations, we don't know the value of the sign
17929 // bit for at least one Op. Since we have a valid ByteProvider, we know that
17930 // the upper bits must be extension bits. Thus, the only ways for the sign
17931 // bit to be unknown is if it was sign extended from unknown value, or if it
17932 // was any extended. In either case, it is correct to use the signed
17933 // version of the signedness semantics of dot4
17934
17935 // In two of such permutations, we known the sign bit is set for
17936 // one op, and the other is unknown. It is okay to used signed version of
17937 // dot4.
17938 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
17939 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
17940 return true;
17941
17942 // In one such permutation, we don't know either of the sign bits. It is okay
17943 // to used the signed version of dot4.
17944 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
17945 return true;
17946
17947 // In two of such permutations, we known the sign bit is unset for
17948 // one op, and the other is unknown. Return std::nullopt to indicate a
17949 // bad match.
17950 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
17951 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
17952 return std::nullopt;
17953
17954 llvm_unreachable("Fully covered condition");
17955}
17956
17957SDValue SITargetLowering::performAddCombine(SDNode *N,
17958 DAGCombinerInfo &DCI) const {
17959 SelectionDAG &DAG = DCI.DAG;
17960 EVT VT = N->getValueType(ResNo: 0);
17961 SDLoc SL(N);
17962 SDValue LHS = N->getOperand(Num: 0);
17963 SDValue RHS = N->getOperand(Num: 1);
17964
17965 if (LHS.getOpcode() == ISD::MUL || RHS.getOpcode() == ISD::MUL) {
17966 if (Subtarget->hasMad64_32()) {
17967 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
17968 return Folded;
17969 }
17970 }
17971
17972 if (SDValue V = reassociateScalarOps(N, DAG)) {
17973 return V;
17974 }
17975
17976 if (VT == MVT::i64) {
17977 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17978 return Folded;
17979 }
17980
17981 // dot4 produces a 32-bit result, so a wider VT can't be folded.
17982 if (!VT.isVector() && VT.getSizeInBits() <= 32 &&
17983 (isMul(Op: LHS) || isMul(Op: RHS)) && Subtarget->hasDot7Insts() &&
17984 (Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
17985 SDValue TempNode(N, 0);
17986 std::optional<bool> IsSigned;
17987 SmallVector<DotSrc, 4> Src0s;
17988 SmallVector<DotSrc, 4> Src1s;
17989 SmallVector<SDValue, 4> Src2s;
17990
17991 // Match the v_dot4 tree, while collecting src nodes.
17992 int ChainLength = 0;
17993 for (int I = 0; I < 4; I++) {
17994 auto MulIdx = isMul(Op: LHS) ? 0 : isMul(Op: RHS) ? 1 : -1;
17995 if (MulIdx == -1)
17996 break;
17997 auto Src0 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0));
17998 if (!Src0)
17999 break;
18000 auto Src1 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1));
18001 if (!Src1)
18002 break;
18003
18004 auto IterIsSigned = checkDot4MulSignedness(
18005 N: TempNode->getOperand(Num: MulIdx), Src0&: *Src0, Src1&: *Src1,
18006 S0Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0),
18007 S1Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1), DAG);
18008 if (!IterIsSigned)
18009 break;
18010 if (!IsSigned)
18011 IsSigned = *IterIsSigned;
18012 if (*IterIsSigned != *IsSigned)
18013 break;
18014 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I);
18015 auto AddIdx = 1 - MulIdx;
18016 // Allow the special case where add (add (mul24, 0), mul24) became ->
18017 // add (mul24, mul24).
18018 if (I == 2 && isMul(Op: TempNode->getOperand(Num: AddIdx))) {
18019 Src2s.push_back(Elt: TempNode->getOperand(Num: AddIdx));
18020 auto Src0 =
18021 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0));
18022 if (!Src0)
18023 break;
18024 auto Src1 =
18025 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1));
18026 if (!Src1)
18027 break;
18028 auto IterIsSigned = checkDot4MulSignedness(
18029 N: TempNode->getOperand(Num: AddIdx), Src0&: *Src0, Src1&: *Src1,
18030 S0Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0),
18031 S1Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1), DAG);
18032 if (!IterIsSigned)
18033 break;
18034 assert(IsSigned);
18035 if (*IterIsSigned != *IsSigned)
18036 break;
18037 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I + 1);
18038 Src2s.push_back(Elt: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
18039 ChainLength = I + 2;
18040 break;
18041 }
18042
18043 TempNode = TempNode->getOperand(Num: AddIdx);
18044 Src2s.push_back(Elt: TempNode);
18045 ChainLength = I + 1;
18046 // The loop body treats TempNode's operands as addends.
18047 if (TempNode.getOpcode() != ISD::ADD)
18048 break;
18049 LHS = TempNode->getOperand(Num: 0);
18050 RHS = TempNode->getOperand(Num: 1);
18051 }
18052
18053 if (ChainLength < 2)
18054 return SDValue();
18055
18056 // Masks were constructed with assumption that we would find a chain of
18057 // length 4. If not, then we need to 0 out the MSB bits (via perm mask of
18058 // 0x0c) so they do not affect dot calculation.
18059 if (ChainLength < 4) {
18060 fixMasks(Srcs&: Src0s, ChainLength);
18061 fixMasks(Srcs&: Src1s, ChainLength);
18062 }
18063
18064 SDValue Src0, Src1;
18065
18066 // If we are just using a single source for both, and have permuted the
18067 // bytes consistently, we can just use the sources without permuting
18068 // (commutation).
18069 bool UseOriginalSrc = false;
18070 if (ChainLength == 4 && Src0s.size() == 1 && Src1s.size() == 1 &&
18071 Src0s.begin()->PermMask == Src1s.begin()->PermMask &&
18072 Src0s.begin()->SrcOp.getValueSizeInBits() >= 32 &&
18073 Src1s.begin()->SrcOp.getValueSizeInBits() >= 32) {
18074 SmallVector<unsigned, 4> SrcBytes;
18075 auto Src0Mask = Src0s.begin()->PermMask;
18076 SrcBytes.push_back(Elt: Src0Mask & 0xFF000000);
18077 bool UniqueEntries = true;
18078 for (auto I = 1; I < 4; I++) {
18079 auto NextByte = Src0Mask & (0xFF << ((3 - I) * 8));
18080
18081 if (is_contained(Range&: SrcBytes, Element: NextByte)) {
18082 UniqueEntries = false;
18083 break;
18084 }
18085 SrcBytes.push_back(Elt: NextByte);
18086 }
18087
18088 if (UniqueEntries) {
18089 UseOriginalSrc = true;
18090
18091 auto *FirstElt = Src0s.begin();
18092 auto FirstEltOp =
18093 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
18094
18095 auto *SecondElt = Src1s.begin();
18096 auto SecondEltOp = getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp,
18097 DWordOffset: SecondElt->DWordOffset);
18098
18099 Src0 = DAG.getBitcastedAnyExtOrTrunc(Op: FirstEltOp, DL: SL,
18100 VT: MVT::getIntegerVT(BitWidth: 32));
18101 Src1 = DAG.getBitcastedAnyExtOrTrunc(Op: SecondEltOp, DL: SL,
18102 VT: MVT::getIntegerVT(BitWidth: 32));
18103 }
18104 }
18105
18106 if (!UseOriginalSrc) {
18107 Src0 = resolveSources(DAG, SL, Srcs&: Src0s, IsSigned: false, IsAny: true);
18108 Src1 = resolveSources(DAG, SL, Srcs&: Src1s, IsSigned: false, IsAny: true);
18109 }
18110
18111 assert(IsSigned);
18112 SDValue Src2 =
18113 DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Src2s[ChainLength - 1], DL: SL, VT: MVT::i32);
18114
18115 SDValue IID = DAG.getTargetConstant(Val: *IsSigned ? Intrinsic::amdgcn_sdot4
18116 : Intrinsic::amdgcn_udot4,
18117 DL: SL, VT: MVT::i64);
18118
18119 assert(!VT.isVector());
18120 auto Dot = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32, N1: IID, N2: Src0,
18121 N3: Src1, N4: Src2, N5: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
18122
18123 return DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Dot, DL: SL, VT);
18124 }
18125
18126 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
18127 return SDValue();
18128
18129 // add x, zext (setcc) => uaddo_carry x, 0, setcc
18130 // add x, sext (setcc) => usubo_carry x, 0, setcc
18131 unsigned Opc = LHS.getOpcode();
18132 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND ||
18133 Opc == ISD::ANY_EXTEND || Opc == ISD::UADDO_CARRY)
18134 std::swap(a&: RHS, b&: LHS);
18135
18136 Opc = RHS.getOpcode();
18137 switch (Opc) {
18138 default:
18139 break;
18140 case ISD::ZERO_EXTEND:
18141 case ISD::SIGN_EXTEND:
18142 case ISD::ANY_EXTEND: {
18143 auto Cond = RHS.getOperand(i: 0);
18144 // If this won't be a real VOPC output, we would still need to insert an
18145 // extra instruction anyway.
18146 if (!isBoolSGPR(V: Cond))
18147 break;
18148 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
18149 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
18150 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::USUBO_CARRY : ISD::UADDO_CARRY;
18151 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
18152 }
18153 case ISD::UADDO_CARRY: {
18154 // add x, (uaddo_carry y, 0, cc) => uaddo_carry x, y, cc
18155 if (!isNullConstant(V: RHS.getOperand(i: 1)))
18156 break;
18157 SDValue Args[] = {LHS, RHS.getOperand(i: 0), RHS.getOperand(i: 2)};
18158 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: RHS->getVTList(), Ops: Args);
18159 }
18160 }
18161 return SDValue();
18162}
18163
18164SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
18165 DAGCombinerInfo &DCI) const {
18166 SelectionDAG &DAG = DCI.DAG;
18167 SDLoc DL(N);
18168 EVT VT = N->getValueType(ResNo: 0);
18169 SDValue N0 = N->getOperand(Num: 0);
18170 SDValue N1 = N->getOperand(Num: 1);
18171
18172 // The following folds transform PTRADDs into regular arithmetic in cases
18173 // where the PTRADD wouldn't be folded as an immediate offset into memory
18174 // instructions anyway. They are target-specific in that other targets might
18175 // prefer to not lose information about the pointer arithmetic.
18176
18177 // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
18178 // Adapted from DAGCombiner::visitADDLikeCommutative.
18179 SDValue V, K;
18180 if (sd_match(N: N1, P: m_Shl(L: m_Neg(V: m_Value(N&: V)), R: m_Value(N&: K)))) {
18181 SDNodeFlags ShlFlags = N1->getFlags();
18182 // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
18183 // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
18184 // preserved.
18185 SDNodeFlags NewShlFlags =
18186 ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
18187 ? SDNodeFlags::NoSignedWrap
18188 : SDNodeFlags();
18189 SDValue Inner = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: V, N2: K, Flags: NewShlFlags);
18190 DCI.AddToWorklist(N: Inner.getNode());
18191 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Inner);
18192 }
18193
18194 // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
18195 // performAddCombine.
18196 if (N1.getOpcode() == ISD::MUL) {
18197 if (Subtarget->hasMad64_32()) {
18198 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
18199 return Folded;
18200 }
18201 }
18202
18203 // If the 32 low bits of the constant are all zero, there is nothing to fold
18204 // into an immediate offset, so it's better to eliminate the unnecessary
18205 // addition for the lower 32 bits than to preserve the PTRADD.
18206 // Analogous to a fold in performAddCombine.
18207 if (VT == MVT::i64) {
18208 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18209 return Folded;
18210 }
18211
18212 if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
18213 return SDValue();
18214
18215 SDValue X = N0;
18216 SDValue Y = N1.getOperand(i: 0);
18217 SDValue Z = N1.getOperand(i: 1);
18218 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Y);
18219 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Z);
18220
18221 if (!YIsConstant && !ZIsConstant && !X->isDivergent() &&
18222 Y->isDivergent() != Z->isDivergent()) {
18223 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
18224 // y are uniform and z isn't.
18225 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
18226 // z are uniform and y isn't.
18227 // The goal is to push uniform operands up in the computation, so that they
18228 // can be handled with scalar operations. We can't use reassociateScalarOps
18229 // for this since it requires two identical commutative operations to
18230 // reassociate.
18231 if (Y->isDivergent())
18232 std::swap(a&: Y, b&: Z);
18233 // If both additions in the original were NUW, reassociation preserves that.
18234 SDNodeFlags ReassocFlags =
18235 (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
18236 SDValue UniformInner = DAG.getMemBasePlusOffset(Base: X, Offset: Y, DL, Flags: ReassocFlags);
18237 DCI.AddToWorklist(N: UniformInner.getNode());
18238 return DAG.getMemBasePlusOffset(Base: UniformInner, Offset: Z, DL, Flags: ReassocFlags);
18239 }
18240
18241 return SDValue();
18242}
18243
18244static bool isCtlzOpc(unsigned Opc) {
18245 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
18246}
18247
18248SDValue SITargetLowering::performSubCombine(SDNode *N,
18249 DAGCombinerInfo &DCI) const {
18250 SelectionDAG &DAG = DCI.DAG;
18251 EVT VT = N->getValueType(ResNo: 0);
18252
18253 if (VT == MVT::i64) {
18254 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18255 return Folded;
18256 }
18257
18258 if (VT != MVT::i32)
18259 return SDValue();
18260
18261 SDLoc SL(N);
18262 SDValue LHS = N->getOperand(Num: 0);
18263 SDValue RHS = N->getOperand(Num: 1);
18264
18265 // sub x, zext (setcc) => usubo_carry x, 0, setcc
18266 // sub x, sext (setcc) => uaddo_carry x, 0, setcc
18267 unsigned Opc = RHS.getOpcode();
18268 switch (Opc) {
18269 default:
18270 break;
18271 case ISD::ZERO_EXTEND:
18272 case ISD::SIGN_EXTEND:
18273 case ISD::ANY_EXTEND: {
18274 auto Cond = RHS.getOperand(i: 0);
18275 // If this won't be a real VOPC output, we would still need to insert an
18276 // extra instruction anyway.
18277 if (!isBoolSGPR(V: Cond))
18278 break;
18279 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
18280 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
18281 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
18282 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
18283 }
18284 }
18285
18286 if (LHS.getOpcode() == ISD::USUBO_CARRY) {
18287 // sub (usubo_carry x, 0, cc), y => usubo_carry x, y, cc
18288 if (!isNullConstant(V: LHS.getOperand(i: 1)))
18289 return SDValue();
18290 SDValue Args[] = {LHS.getOperand(i: 0), RHS, LHS.getOperand(i: 2)};
18291 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: SDLoc(N), VTList: LHS->getVTList(), Ops: Args);
18292 }
18293
18294 // sub (ctlz (xor x, (sra x, 31))), 1 -> ctls x.
18295 if (isOneConstant(V: RHS) && isCtlzOpc(Opc: LHS.getOpcode())) {
18296 SDValue CtlzSrc = LHS.getOperand(i: 0);
18297 // Check for xor x, (sra x, 31) pattern.
18298 if (CtlzSrc.getOpcode() == ISD::XOR) {
18299 SDValue X = CtlzSrc.getOperand(i: 0);
18300 SDValue SignExt = CtlzSrc.getOperand(i: 1);
18301 // Try both ordering of XOR operands.
18302 if (SignExt.getOpcode() != ISD::SRA)
18303 std::swap(a&: X, b&: SignExt);
18304 if (SignExt.getOpcode() == ISD::SRA && SignExt.getOperand(i: 0) == X) {
18305 ConstantSDNode *ShiftAmt =
18306 dyn_cast<ConstantSDNode>(Val: SignExt.getOperand(i: 1));
18307 unsigned BitWidth = X.getValueType().getScalarSizeInBits();
18308 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1)
18309 return DAG.getNode(Opcode: ISD::CTLS, DL: SL, VT, Operand: X);
18310 }
18311 }
18312 }
18313
18314 return SDValue();
18315}
18316
18317SDValue SITargetLowering::performFAddCombine(SDNode *N,
18318 DAGCombinerInfo &DCI) const {
18319 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18320 return SDValue();
18321
18322 SelectionDAG &DAG = DCI.DAG;
18323 EVT VT = N->getValueType(ResNo: 0);
18324
18325 SDLoc SL(N);
18326 SDValue LHS = N->getOperand(Num: 0);
18327 SDValue RHS = N->getOperand(Num: 1);
18328
18329 // These should really be instruction patterns, but writing patterns with
18330 // source modifiers is a pain.
18331
18332 // fadd (fadd (a, a), b) -> mad 2.0, a, b
18333 if (LHS.getOpcode() == ISD::FADD) {
18334 SDValue A = LHS.getOperand(i: 0);
18335 if (A == LHS.getOperand(i: 1)) {
18336 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
18337 if (FusedOp != 0) {
18338 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18339 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: RHS);
18340 }
18341 }
18342 }
18343
18344 // fadd (b, fadd (a, a)) -> mad 2.0, a, b
18345 if (RHS.getOpcode() == ISD::FADD) {
18346 SDValue A = RHS.getOperand(i: 0);
18347 if (A == RHS.getOperand(i: 1)) {
18348 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
18349 if (FusedOp != 0) {
18350 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18351 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: LHS);
18352 }
18353 }
18354 }
18355
18356 return SDValue();
18357}
18358
18359SDValue SITargetLowering::performFSubCombine(SDNode *N,
18360 DAGCombinerInfo &DCI) const {
18361 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18362 return SDValue();
18363
18364 SelectionDAG &DAG = DCI.DAG;
18365 SDLoc SL(N);
18366 EVT VT = N->getValueType(ResNo: 0);
18367 assert(!VT.isVector());
18368
18369 // Try to get the fneg to fold into the source modifier. This undoes generic
18370 // DAG combines and folds them into the mad.
18371 //
18372 // Only do this if we are not trying to support denormals. v_mad_f32 does
18373 // not support denormals ever.
18374 SDValue LHS = N->getOperand(Num: 0);
18375 SDValue RHS = N->getOperand(Num: 1);
18376 if (LHS.getOpcode() == ISD::FADD) {
18377 // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c)
18378 SDValue A = LHS.getOperand(i: 0);
18379 if (A == LHS.getOperand(i: 1)) {
18380 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
18381 if (FusedOp != 0) {
18382 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18383 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
18384
18385 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: NegRHS);
18386 }
18387 }
18388 }
18389
18390 if (RHS.getOpcode() == ISD::FADD) {
18391 // (fsub c, (fadd a, a)) -> mad -2.0, a, c
18392
18393 SDValue A = RHS.getOperand(i: 0);
18394 if (A == RHS.getOperand(i: 1)) {
18395 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
18396 if (FusedOp != 0) {
18397 const SDValue NegTwo = DAG.getConstantFP(Val: -2.0, DL: SL, VT);
18398 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: NegTwo, N3: LHS);
18399 }
18400 }
18401 }
18402
18403 return SDValue();
18404}
18405
18406SDValue SITargetLowering::performFDivCombine(SDNode *N,
18407 DAGCombinerInfo &DCI) const {
18408 SelectionDAG &DAG = DCI.DAG;
18409 SDLoc SL(N);
18410 EVT VT = N->getValueType(ResNo: 0);
18411
18412 if (VT != MVT::f16 && VT != MVT::bf16)
18413 return SDValue();
18414
18415 SDValue LHS = N->getOperand(Num: 0);
18416 SDValue RHS = N->getOperand(Num: 1);
18417
18418 SDNodeFlags Flags = N->getFlags();
18419 SDNodeFlags RHSFlags = RHS->getFlags();
18420 if (!Flags.hasAllowContract() || !RHSFlags.hasAllowContract() ||
18421 !RHS->hasOneUse())
18422 return SDValue();
18423
18424 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
18425 bool IsNegative = false;
18426 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
18427 // fdiv contract 1.0, (sqrt contract x) -> rsq
18428 // fdiv contract -1.0, (sqrt contract x) -> fneg(rsq)
18429 if (RHS.getOpcode() == ISD::FSQRT) {
18430 // TODO: Or in RHS flags, somehow missing from SDNodeFlags
18431 SDValue SqrtOp = RHS.getOperand(i: 0);
18432 SDValue Rsq;
18433 if (isOperationLegal(Op: ISD::FSQRT, VT)) {
18434 // fsqrt legality correlates to rsq availability of the same type.
18435 Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT, Operand: SqrtOp, Flags);
18436 } else if (VT == MVT::f16) {
18437 // Targets without 16-bit instructions (gfx6/gfx7) have no f16 rsq,
18438 // but v_rsq_f32 is more than accurate enough for f16. Unlike bf16,
18439 // every f16 value (including denormals) extends to a normal f32, and
18440 // an f16 rsq result is never denormal, so the f32 reciprocal square
18441 // root needs no denormal handling. Compute it in f32 and round back.
18442 SDValue Ext =
18443 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: SqrtOp, Flags);
18444 SDValue F32Rsq =
18445 DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
18446 Rsq = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: F32Rsq,
18447 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
18448 } else {
18449 // bf16 shares f32's exponent range, so bf16 denormals would extend to
18450 // f32 denormals that v_rsq_f32 does not handle. Leave it expanded.
18451 return SDValue();
18452 }
18453 return IsNegative ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Rsq, Flags) : Rsq;
18454 }
18455 }
18456 }
18457
18458 return SDValue();
18459}
18460
18461SDValue SITargetLowering::performFMulCombine(SDNode *N,
18462 DAGCombinerInfo &DCI) const {
18463 SelectionDAG &DAG = DCI.DAG;
18464 EVT VT = N->getValueType(ResNo: 0);
18465 EVT ScalarVT = VT.getScalarType();
18466 EVT IntVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
18467
18468 if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() &&
18469 (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
18470 // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32.
18471 return SDValue();
18472 }
18473
18474 SDValue LHS = N->getOperand(Num: 0);
18475 SDValue RHS = N->getOperand(Num: 1);
18476
18477 // It is cheaper to realize i32 inline constants as compared against
18478 // materializing f16 or f64 (or even non-inline f32) values,
18479 // possible via ldexp usage, as shown below :
18480 //
18481 // Given : A = 2^a & B = 2^b ; where a and b are integers.
18482 // fmul x, (select y, A, B) -> ldexp( x, (select i32 y, a, b) )
18483 // fmul x, (select y, -A, -B) -> ldexp( (fneg x), (select i32 y, a, b) )
18484 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
18485 (RHS.hasOneUse() && RHS.getOpcode() == ISD::SELECT)) {
18486 const ConstantFPSDNode *TrueNode = isConstOrConstSplatFP(N: RHS.getOperand(i: 1));
18487 if (!TrueNode)
18488 return SDValue();
18489 const ConstantFPSDNode *FalseNode =
18490 isConstOrConstSplatFP(N: RHS.getOperand(i: 2));
18491 if (!FalseNode)
18492 return SDValue();
18493
18494 if (TrueNode->isNegative() != FalseNode->isNegative())
18495 return SDValue();
18496
18497 // For f32, only non-inline constants should be transformed.
18498 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18499 if (ScalarVT == MVT::f32 &&
18500 TII->isInlineConstant(Imm: TrueNode->getValueAPF()) &&
18501 TII->isInlineConstant(Imm: FalseNode->getValueAPF()))
18502 return SDValue();
18503
18504 int TrueNodeExpVal = TrueNode->getValueAPF().getExactLog2Abs();
18505 if (TrueNodeExpVal == INT_MIN)
18506 return SDValue();
18507 int FalseNodeExpVal = FalseNode->getValueAPF().getExactLog2Abs();
18508 if (FalseNodeExpVal == INT_MIN)
18509 return SDValue();
18510
18511 SDLoc SL(N);
18512 SDValue SelectNode =
18513 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: IntVT, N1: RHS.getOperand(i: 0),
18514 N2: DAG.getSignedConstant(Val: TrueNodeExpVal, DL: SL, VT: IntVT),
18515 N3: DAG.getSignedConstant(Val: FalseNodeExpVal, DL: SL, VT: IntVT));
18516
18517 LHS = TrueNode->isNegative()
18518 ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS, Flags: LHS->getFlags())
18519 : LHS;
18520
18521 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: LHS, N2: SelectNode, Flags: N->getFlags());
18522 }
18523
18524 return SDValue();
18525}
18526
18527SDValue SITargetLowering::performFMACombine(SDNode *N,
18528 DAGCombinerInfo &DCI) const {
18529 SelectionDAG &DAG = DCI.DAG;
18530 EVT VT = N->getValueType(ResNo: 0);
18531 SDLoc SL(N);
18532
18533 if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
18534 return SDValue();
18535
18536 // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
18537 // FDOT2((V2F16)S0, (V2F16)S1, (F32)z))
18538 SDValue Op1 = N->getOperand(Num: 0);
18539 SDValue Op2 = N->getOperand(Num: 1);
18540 SDValue FMA = N->getOperand(Num: 2);
18541
18542 if (FMA.getOpcode() != ISD::FMA || Op1.getOpcode() != ISD::FP_EXTEND ||
18543 Op2.getOpcode() != ISD::FP_EXTEND)
18544 return SDValue();
18545
18546 // The fdot2 fold (fma_mix -> dot2) is only safe when both instructions agree
18547 // on how f16 subnormal inputs are handled. However, if both FMAs carry afn
18548 // the caller accepts approximate results, so any subnormal flushing
18549 // introduced by dot2 is acceptable regardless of mode.
18550 //
18551 // gfx90a (CDNA2) is the sole exception (dot2UnconditionalFlush): v_dot2c
18552 // unconditionally flushes f16 subnormal inputs to zero regardless of MODE,
18553 // while v_fma_mix_f32 preserves them when ieee=1 (the default compute kernel
18554 // mode). The fold is safe only when f32 denorm = PreserveSign, which implies
18555 // ieee=0 so both flush.
18556 //
18557 // All other GPUs: v_dot2 does NOT flush f16 subnormal inputs. v_fma_mix_f32
18558 // flushes them only when f32 denorm = PreserveSign. The fold is safe only
18559 // when f32 denorm is IEEE (both preserve the subnormal). Dynamic mode is
18560 // also rejected since the runtime value is unknown.
18561 bool AllowInaccuracy = N->getFlags().hasApproximateFuncs() &&
18562 FMA->getFlags().hasApproximateFuncs();
18563 if (!AllowInaccuracy) {
18564 const MachineFunction &MF = DAG.getMachineFunction();
18565 DenormalMode Mode = MF.getDenormalMode(FPType: APFloat::IEEEsingle());
18566 if (Subtarget->dot2UnconditionalFlush()) {
18567 // gfx90a: fold safe only when f32 denorm flushes.
18568 if (Mode != DenormalMode::getPreserveSign())
18569 return SDValue();
18570 } else {
18571 // All other GPUs: fold safe only when f32 denorm is IEEE.
18572 if (Mode != DenormalMode::getIEEE())
18573 return SDValue();
18574 }
18575 }
18576
18577 // fp-contract allows reassociating the fma tree into a dot product.
18578 if (N->getFlags().hasAllowContract() && FMA->getFlags().hasAllowContract()) {
18579 Op1 = Op1.getOperand(i: 0);
18580 Op2 = Op2.getOperand(i: 0);
18581 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18582 Op2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18583 return SDValue();
18584
18585 SDValue Vec1 = Op1.getOperand(i: 0);
18586 SDValue Idx1 = Op1.getOperand(i: 1);
18587 SDValue Vec2 = Op2.getOperand(i: 0);
18588
18589 SDValue FMAOp1 = FMA.getOperand(i: 0);
18590 SDValue FMAOp2 = FMA.getOperand(i: 1);
18591 SDValue FMAAcc = FMA.getOperand(i: 2);
18592
18593 if (FMAOp1.getOpcode() != ISD::FP_EXTEND ||
18594 FMAOp2.getOpcode() != ISD::FP_EXTEND)
18595 return SDValue();
18596
18597 FMAOp1 = FMAOp1.getOperand(i: 0);
18598 FMAOp2 = FMAOp2.getOperand(i: 0);
18599 if (FMAOp1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18600 FMAOp2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18601 return SDValue();
18602
18603 SDValue Vec3 = FMAOp1.getOperand(i: 0);
18604 SDValue Vec4 = FMAOp2.getOperand(i: 0);
18605 SDValue Idx2 = FMAOp1.getOperand(i: 1);
18606
18607 if (Idx1 != Op2.getOperand(i: 1) || Idx2 != FMAOp2.getOperand(i: 1))
18608 return SDValue();
18609
18610 if (!isa<ConstantSDNode>(Val: Idx1) || !isa<ConstantSDNode>(Val: Idx2) ||
18611 Idx1 == Idx2)
18612 return SDValue();
18613
18614 if (Vec1 == Vec2 || Vec3 == Vec4)
18615 return SDValue();
18616
18617 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16)
18618 return SDValue();
18619
18620 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
18621 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL: SL, VT: MVT::f32, N1: Vec1, N2: Vec2, N3: FMAAcc,
18622 N4: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
18623 }
18624 }
18625 return SDValue();
18626}
18627
18628// Given a double-precision ordered or unordered comparison, return the
18629// condition code for an equivalent integral comparison of the operands' upper
18630// 32 bits, or `SETCC_INVALID` if not possible.
18631// For simplicity, no simplification occurs if the operands are not both known
18632// to have sign bit zero.
18633//
18634// EQ/NE:
18635// If LHS.lo32 == RHS.lo32:
18636// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18637// If LHS.lo32 != RHS.lo32:
18638// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18639// The reduction is not possible if operands may be +0 and -0.
18640// For ordered eq / unordered ne, at most one operand may be NaN.
18641// For unordered eq / ordered ne, neither operand can be NaN.
18642//
18643// LT/GE:
18644// If LHS.lo32 >= RHS.lo32 (unsigned):
18645// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18646// If LHS.lo32 < RHS.lo32 (unsigned):
18647// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18648// The reduction is only supported if both operands are nonnegative.
18649// For ordered lt / unordered ge, the RHS cannot be NaN.
18650// For unordered lt / ordered ge, neither operand can be NaN.
18651//
18652// LE/GT:
18653// If LHS.lo32 > RHS.lo32 (unsigned):
18654// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18655// If LHS.lo32 <= RHS.lo32 (unsigned):
18656// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18657// The reduction is only supported if both operands are nonnegative.
18658// For unordered le / ordered gt, the LHS cannot be NaN.
18659// For ordered le / unordered gt, neither operand can be NaN.
18660static ISD::CondCode tryReduceF64CompareToHiHalf(const ISD::CondCode CC,
18661 const SDValue LHS,
18662 const SDValue RHS,
18663 const SelectionDAG &DAG) {
18664 EVT VT = LHS.getValueType();
18665 assert(VT == MVT::f64 && "Incorrect operand type!");
18666
18667 const KnownBits RHSBits = DAG.computeKnownBits(Op: RHS);
18668 // Bail if RHS sign bit is not known to be zero.
18669 if (!RHSBits.Zero.isSignBitSet())
18670 return ISD::SETCC_INVALID;
18671
18672 const KnownBits RHSKnownLo32 = RHSBits.trunc(BitWidth: 32);
18673 const KnownFPClass RHSFPClass =
18674 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: RHSBits);
18675 const bool RHSMaybeNaN = !RHSFPClass.isKnownNeverNaN();
18676
18677 const KnownBits LHSBits = DAG.computeKnownBits(Op: LHS);
18678 const KnownBits LHSKnownLo32 = LHSBits.trunc(BitWidth: 32);
18679 const KnownFPClass LHSFPClass =
18680 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: LHSBits);
18681 const bool LHSMaybeNaN = !LHSFPClass.isKnownNeverNaN();
18682
18683 // Bail if LHS sign bit is not known to be zero.
18684 if (!LHSBits.Zero.isSignBitSet())
18685 return ISD::SETCC_INVALID;
18686
18687 switch (CC) {
18688 default:
18689 break;
18690 case ISD::SETEQ:
18691 case ISD::SETOEQ:
18692 case ISD::SETUEQ:
18693 case ISD::SETONE:
18694 case ISD::SETUNE: {
18695 // OEQ should be false if either operand is NaN, so it suffices that at
18696 // least one operand is not NaN.
18697 if (CC == ISD::SETOEQ && LHSMaybeNaN && RHSMaybeNaN)
18698 break;
18699 // UEQ should be true if either operand is NaN, but this cannot be checked
18700 // on underlying bits.
18701 if (CC == ISD::SETUEQ && (LHSMaybeNaN || RHSMaybeNaN))
18702 break;
18703 // ONE should be false if either operand is NaN, but this cannot be
18704 // checked on underlying bits.
18705 if (CC == ISD::SETONE && (LHSMaybeNaN || RHSMaybeNaN))
18706 break;
18707 // UNE should be true if either operand is NaN, so it suffices that they
18708 // are not both NaN.
18709 if (CC == ISD::SETUNE && LHSMaybeNaN && RHSMaybeNaN)
18710 break;
18711
18712 const std::optional<bool> KnownEq =
18713 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18714
18715 if (!KnownEq)
18716 break;
18717
18718 if (*KnownEq)
18719 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18720 ? ISD::SETEQ
18721 : ISD::SETNE;
18722
18723 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18724 ? ISD::SETFALSE
18725 : ISD::SETTRUE;
18726 }
18727 case ISD::SETLT:
18728 case ISD::SETOLT:
18729 case ISD::SETULT:
18730 case ISD::SETGE:
18731 case ISD::SETOGE:
18732 case ISD::SETUGE: {
18733 // OLT should be false if either operand is NaN.
18734 // Since NaNs have maximum exponent and nonzero mantissa, false positives
18735 // are only possible if the RHS is NaN. (No issue with RHS == +inf since
18736 // the inequality is strict)
18737 if (CC == ISD::SETOLT && RHSMaybeNaN)
18738 break;
18739 // ULT should be true if either operand is NaN, but this cannot be ensured
18740 // with a truncated comparison.
18741 if (CC == ISD::SETULT && (LHSMaybeNaN || RHSMaybeNaN))
18742 break;
18743 // OGE should be false if either operand is NaN, but this cannot be
18744 // ensured with a truncated comparison.
18745 if (CC == ISD::SETOGE && (LHSMaybeNaN || RHSMaybeNaN))
18746 break;
18747 // UGE should be true if either operand is NaN.
18748 // False negatives are only possible if the RHS is NaN.
18749 // (No issue with RHS == +inf since the inequality is inclusive)
18750 if (CC == ISD::SETUGE && RHSMaybeNaN)
18751 break;
18752
18753 const std::optional<bool> KnownUge =
18754 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18755
18756 if (!KnownUge)
18757 break;
18758
18759 if (*KnownUge) {
18760 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18761 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18762 ? ISD::SETLT
18763 : ISD::SETGE;
18764 }
18765 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18766 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18767 ? ISD::SETLE
18768 : ISD::SETGT;
18769 }
18770 case ISD::SETLE:
18771 case ISD::SETOLE:
18772 case ISD::SETULE:
18773 case ISD::SETGT:
18774 case ISD::SETOGT:
18775 case ISD::SETUGT: {
18776 // OLE should be false if either operand is NaN, but this cannot be
18777 // ensured with a truncated comparison.
18778 if (CC == ISD::SETOLE && (LHSMaybeNaN || RHSMaybeNaN))
18779 break;
18780 // ULE should be true if either operand is NaN.
18781 // False negatives are only possible if the LHS is NaN.
18782 // (No issue with LHS == +inf since the inequality is inclusive)
18783 if (CC == ISD::SETULE && LHSMaybeNaN)
18784 break;
18785 // OGT should be false if either operand is NaN.
18786 // False positives are only possible if the LHS is NaN.
18787 // (No issue with LHS == +inf since the inequality is strict)
18788 if (CC == ISD::SETOGT && LHSMaybeNaN)
18789 break;
18790 // UGT should be true if either operand is NaN, but this cannot be ensured
18791 // with a truncated comparison.
18792 if (CC == ISD::SETUGT && (LHSMaybeNaN || RHSMaybeNaN))
18793 break;
18794
18795 const std::optional<bool> KnownUle =
18796 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18797
18798 if (!KnownUle)
18799 break;
18800
18801 if (*KnownUle) {
18802 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18803 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18804 ? ISD::SETLE
18805 : ISD::SETGT;
18806 }
18807 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18808 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18809 ? ISD::SETLT
18810 : ISD::SETGE;
18811 }
18812 }
18813
18814 return ISD::SETCC_INVALID;
18815}
18816
18817SDValue SITargetLowering::performSetCCCombine(SDNode *N,
18818 DAGCombinerInfo &DCI) const {
18819 SelectionDAG &DAG = DCI.DAG;
18820 SDLoc SL(N);
18821
18822 SDValue LHS = N->getOperand(Num: 0);
18823 SDValue RHS = N->getOperand(Num: 1);
18824 EVT VT = LHS.getValueType();
18825 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
18826
18827 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
18828 if (!CRHS) {
18829 CRHS = dyn_cast<ConstantSDNode>(Val&: LHS);
18830 if (CRHS) {
18831 std::swap(a&: LHS, b&: RHS);
18832 CC = getSetCCSwappedOperands(Operation: CC);
18833 }
18834 }
18835
18836 if (CRHS) {
18837 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND &&
18838 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18839 // setcc (sext from i1 cc), -1, ne|sgt|ult) => not cc => xor cc, -1
18840 // setcc (sext from i1 cc), -1, eq|sle|uge) => cc
18841 // setcc (sext from i1 cc), 0, eq|sge|ule) => not cc => xor cc, -1
18842 // setcc (sext from i1 cc), 0, ne|ugt|slt) => cc
18843 if ((CRHS->isAllOnes() &&
18844 (CC == ISD::SETNE || CC == ISD::SETGT || CC == ISD::SETULT)) ||
18845 (CRHS->isZero() &&
18846 (CC == ISD::SETEQ || CC == ISD::SETGE || CC == ISD::SETULE)))
18847 return DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
18848 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i1));
18849 if ((CRHS->isAllOnes() &&
18850 (CC == ISD::SETEQ || CC == ISD::SETLE || CC == ISD::SETUGE)) ||
18851 (CRHS->isZero() &&
18852 (CC == ISD::SETNE || CC == ISD::SETUGT || CC == ISD::SETLT)))
18853 return LHS.getOperand(i: 0);
18854 }
18855
18856 const APInt &CRHSVal = CRHS->getAPIntValue();
18857 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
18858 LHS.getOpcode() == ISD::SELECT &&
18859 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
18860 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2)) &&
18861 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18862 // Given CT != FT:
18863 // setcc (select cc, CT, CF), CF, eq => xor cc, -1
18864 // setcc (select cc, CT, CF), CF, ne => cc
18865 // setcc (select cc, CT, CF), CT, ne => xor cc, -1
18866 // setcc (select cc, CT, CF), CT, eq => cc
18867 const APInt &CT = LHS.getConstantOperandAPInt(i: 1);
18868 const APInt &CF = LHS.getConstantOperandAPInt(i: 2);
18869
18870 if (CT != CF) {
18871 if ((CF == CRHSVal && CC == ISD::SETEQ) ||
18872 (CT == CRHSVal && CC == ISD::SETNE))
18873 return DAG.getNOT(DL: SL, Val: LHS.getOperand(i: 0), VT: MVT::i1);
18874 if ((CF == CRHSVal && CC == ISD::SETNE) ||
18875 (CT == CRHSVal && CC == ISD::SETEQ))
18876 return LHS.getOperand(i: 0);
18877 }
18878 }
18879 }
18880
18881 // Truncate 64-bit setcc to only test upper or lower 32-bits.
18882 if (VT == MVT::i64) {
18883 const KnownBits LHSKnown = DAG.computeKnownBits(Op: LHS);
18884 const KnownBits RHSKnown = DAG.computeKnownBits(Op: RHS);
18885 const KnownBits LHSKnownLo32 = LHSKnown.trunc(BitWidth: 32);
18886 const KnownBits RHSKnownLo32 = RHSKnown.trunc(BitWidth: 32);
18887 // Truncate 64-bit setcc to test only upper 32-bits of its operands in the
18888 // following cases where information about the lower 32-bits of its operands
18889 // is known:
18890 //
18891 // If LHS.lo32 == RHS.lo32:
18892 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18893 // If LHS.lo32 != RHS.lo32:
18894 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18895 // If LHS.lo32 >= RHS.lo32 (unsigned):
18896 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18897 // If LHS.lo32 > RHS.lo32 (unsigned):
18898 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18899 // If LHS.lo32 <= RHS.lo32 (unsigned):
18900 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18901 // If LHS.lo32 < RHS.lo32 (unsigned):
18902 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18903
18904 // NewCC is valid iff we can truncate the setcc to only test the upper 32
18905 // bits
18906 ISD::CondCode NewCC = ISD::SETCC_INVALID;
18907
18908 switch (CC) {
18909 default:
18910 llvm_unreachable("Unexpected condition code for integer setcc");
18911 break;
18912 case ISD::SETEQ: {
18913 const std::optional<bool> KnownEq =
18914 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18915 if (KnownEq)
18916 NewCC = *KnownEq ? ISD::SETEQ : ISD::SETFALSE;
18917
18918 break;
18919 }
18920 case ISD::SETNE: {
18921 const std::optional<bool> KnownEq =
18922 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18923 if (KnownEq)
18924 NewCC = *KnownEq ? ISD::SETNE : ISD::SETTRUE;
18925
18926 break;
18927 }
18928 case ISD::SETULT:
18929 case ISD::SETUGE:
18930 case ISD::SETLT:
18931 case ISD::SETGE: {
18932 const std::optional<bool> KnownUge =
18933 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18934 if (KnownUge) {
18935 if (*KnownUge) {
18936 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18937 NewCC = CC;
18938 } else {
18939 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18940 NewCC = CC == ISD::SETULT ? ISD::SETULE
18941 : CC == ISD::SETUGE ? ISD::SETUGT
18942 : CC == ISD::SETLT ? ISD::SETLE
18943 : ISD::SETGT;
18944 }
18945 }
18946 break;
18947 }
18948 case ISD::SETULE:
18949 case ISD::SETUGT:
18950 case ISD::SETLE:
18951 case ISD::SETGT: {
18952 const std::optional<bool> KnownUle =
18953 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18954 if (KnownUle) {
18955 if (*KnownUle) {
18956 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18957 NewCC = CC;
18958 } else {
18959 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18960 NewCC = CC == ISD::SETULE ? ISD::SETULT
18961 : CC == ISD::SETUGT ? ISD::SETUGE
18962 : CC == ISD::SETLE ? ISD::SETLT
18963 : ISD::SETGE;
18964 }
18965 }
18966 break;
18967 }
18968 }
18969
18970 if (NewCC != ISD::SETCC_INVALID)
18971 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
18972 RHS: getHiHalf64(Op: RHS, DAG), Cond: NewCC);
18973
18974 // Truncate 64-bit setcc to test only lower 32-bits when upper 32-bits are
18975 // known equal.
18976 const KnownBits LHSKnownHi32 = LHSKnown.extractBits(NumBits: 32, BitPosition: 32);
18977 const KnownBits RHSKnownHi32 = RHSKnown.extractBits(NumBits: 32, BitPosition: 32);
18978 const std::optional<bool> KnownEq =
18979 KnownBits::eq(LHS: LHSKnownHi32, RHS: RHSKnownHi32);
18980
18981 if (KnownEq == true) {
18982 // For signed comparisons, convert to unsigned since upper halves match
18983 ISD::CondCode NewCC = ISD::getUnsignedSetCC(Code: CC);
18984
18985 SDValue LHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: LHS);
18986 SDValue RHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: RHS);
18987 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: LHSLo, RHS: RHSLo, Cond: NewCC);
18988 }
18989 }
18990
18991 // Eliminate setcc by using carryout from add/sub instruction
18992
18993 // LHS = ADD i64 RHS, Z LHSlo = UADDO i32 RHSlo, Zlo
18994 // setcc LHS ult RHS -> LHSHi = UADDO_CARRY i32 RHShi, Zhi
18995 // similarly for subtraction
18996
18997 // LHS = ADD i64 Y, 1 LHSlo = UADDO i32 Ylo, 1
18998 // setcc LHS eq 0 -> LHSHi = UADDO_CARRY i32 Yhi, 0
18999
19000 if (VT == MVT::i64 && ((CC == ISD::SETULT &&
19001 sd_match(N: LHS, P: m_Add(L: m_Specific(N: RHS), R: m_Value()))) ||
19002 (CC == ISD::SETUGT &&
19003 sd_match(N: LHS, P: m_Sub(L: m_Specific(N: RHS), R: m_Value()))) ||
19004 (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
19005 sd_match(N: LHS, P: m_Add(L: m_Value(), R: m_One()))))) {
19006 bool IsAdd = LHS.getOpcode() == ISD::ADD;
19007
19008 SDValue Op0 = LHS.getOperand(i: 0);
19009 SDValue Op1 = LHS.getOperand(i: 1);
19010
19011 SDValue Op0Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op0);
19012 SDValue Op1Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op1);
19013
19014 SDValue Op0Hi = getHiHalf64(Op: Op0, DAG);
19015 SDValue Op1Hi = getHiHalf64(Op: Op1, DAG);
19016
19017 SDValue NodeLo =
19018 DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: SL,
19019 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1), Ops: {Op0Lo, Op1Lo});
19020
19021 SDValue CarryInHi = NodeLo.getValue(R: 1);
19022 SDValue NodeHi = DAG.getNode(Opcode: IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
19023 DL: SL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1),
19024 Ops: {Op0Hi, Op1Hi, CarryInHi});
19025
19026 SDValue ResultLo = NodeLo.getValue(R: 0);
19027 SDValue ResultHi = NodeHi.getValue(R: 0);
19028
19029 SDValue JoinedResult =
19030 DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {ResultLo, ResultHi});
19031
19032 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: JoinedResult);
19033 SDValue Overflow = NodeHi.getValue(R: 1);
19034 DCI.CombineTo(N: LHS.getNode(), Res: Result);
19035 return Overflow;
19036 }
19037
19038 if (VT != MVT::f32 && VT != MVT::f64 &&
19039 (!Subtarget->has16BitInsts() || VT != MVT::f16))
19040 return SDValue();
19041
19042 // Match isinf/isfinite pattern
19043 // (fcmp oeq (fabs x), inf) -> (fp_class x, (p_infinity | n_infinity))
19044 // (fcmp one (fabs x), inf) -> (fp_class x,
19045 // (p_normal | n_normal | p_subnormal | n_subnormal | p_zero | n_zero)
19046 if ((CC == ISD::SETOEQ || CC == ISD::SETONE) &&
19047 LHS.getOpcode() == ISD::FABS) {
19048 const ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
19049 if (!CRHS)
19050 return SDValue();
19051
19052 const APFloat &APF = CRHS->getValueAPF();
19053 if (APF.isInfinity() && !APF.isNegative()) {
19054 const unsigned IsInfMask =
19055 SIInstrFlags::P_INFINITY | SIInstrFlags::N_INFINITY;
19056 const unsigned IsFiniteMask =
19057 SIInstrFlags::N_ZERO | SIInstrFlags::P_ZERO | SIInstrFlags::N_NORMAL |
19058 SIInstrFlags::P_NORMAL | SIInstrFlags::N_SUBNORMAL |
19059 SIInstrFlags::P_SUBNORMAL;
19060 unsigned Mask = CC == ISD::SETOEQ ? IsInfMask : IsFiniteMask;
19061 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
19062 N2: DAG.getConstant(Val: Mask, DL: SL, VT: MVT::i32));
19063 }
19064 }
19065
19066 if (VT == MVT::f64) {
19067 ISD::CondCode HiHalfCC = tryReduceF64CompareToHiHalf(CC, LHS, RHS, DAG);
19068 if (HiHalfCC != ISD::SETCC_INVALID)
19069 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
19070 RHS: getHiHalf64(Op: RHS, DAG), Cond: HiHalfCC);
19071 }
19072
19073 return SDValue();
19074}
19075
19076SDValue
19077SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
19078 DAGCombinerInfo &DCI) const {
19079 SelectionDAG &DAG = DCI.DAG;
19080 SDLoc SL(N);
19081 unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
19082
19083 SDValue Src = N->getOperand(Num: 0);
19084 SDValue Shift = N->getOperand(Num: 0);
19085
19086 // TODO: Extend type shouldn't matter (assuming legal types).
19087 if (Shift.getOpcode() == ISD::ZERO_EXTEND)
19088 Shift = Shift.getOperand(i: 0);
19089
19090 if (Shift.getOpcode() == ISD::SRL || Shift.getOpcode() == ISD::SHL) {
19091 // cvt_f32_ubyte1 (shl x, 8) -> cvt_f32_ubyte0 x
19092 // cvt_f32_ubyte3 (shl x, 16) -> cvt_f32_ubyte1 x
19093 // cvt_f32_ubyte0 (srl x, 16) -> cvt_f32_ubyte2 x
19094 // cvt_f32_ubyte1 (srl x, 16) -> cvt_f32_ubyte3 x
19095 // cvt_f32_ubyte0 (srl x, 8) -> cvt_f32_ubyte1 x
19096 if (auto *C = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1))) {
19097 SDValue Shifted = DAG.getZExtOrTrunc(
19098 Op: Shift.getOperand(i: 0), DL: SDLoc(Shift.getOperand(i: 0)), VT: MVT::i32);
19099
19100 unsigned ShiftOffset = 8 * Offset;
19101 if (Shift.getOpcode() == ISD::SHL)
19102 ShiftOffset -= C->getZExtValue();
19103 else
19104 ShiftOffset += C->getZExtValue();
19105
19106 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
19107 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, DL: SL,
19108 VT: MVT::f32, Operand: Shifted);
19109 }
19110 }
19111 }
19112
19113 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19114 APInt DemandedBits = APInt::getBitsSet(numBits: 32, loBit: 8 * Offset, hiBit: 8 * Offset + 8);
19115 if (TLI.SimplifyDemandedBits(Op: Src, DemandedBits, DCI)) {
19116 // We simplified Src. If this node is not dead, visit it again so it is
19117 // folded properly.
19118 if (N->getOpcode() != ISD::DELETED_NODE)
19119 DCI.AddToWorklist(N);
19120 return SDValue(N, 0);
19121 }
19122
19123 // Handle (or x, (srl y, 8)) pattern when known bits are zero.
19124 if (SDValue DemandedSrc =
19125 TLI.SimplifyMultipleUseDemandedBits(Op: Src, DemandedBits, DAG))
19126 return DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: MVT::f32, Operand: DemandedSrc);
19127
19128 return SDValue();
19129}
19130
19131SDValue SITargetLowering::performClampCombine(SDNode *N,
19132 DAGCombinerInfo &DCI) const {
19133 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
19134 if (!CSrc)
19135 return SDValue();
19136
19137 const MachineFunction &MF = DCI.DAG.getMachineFunction();
19138 const APFloat &F = CSrc->getValueAPF();
19139 APFloat Zero = APFloat::getZero(Sem: F.getSemantics());
19140 if (F < Zero ||
19141 (F.isNaN() && MF.getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) {
19142 return DCI.DAG.getConstantFP(Val: Zero, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
19143 }
19144
19145 APFloat One = APFloat::getOne(Sem: F.getSemantics());
19146 if (F > One)
19147 return DCI.DAG.getConstantFP(Val: One, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
19148
19149 return getCanonicalConstantFP(DAG&: DCI.DAG, SL: SDLoc(N), VT: N->getValueType(ResNo: 0), C: F);
19150}
19151
19152// Check if V is the exponent result of a frexp operation. Returns the frexp
19153// input via FrexpInput if matched. We only match the exponent (not mantissa)
19154// because V_FREXP_MANT returns its input for Inf/NaN, not zero.
19155static bool isFrexpExp(SDValue V, SDValue &FrexpInput) {
19156 // ISD::FFREXP returns {mant, exp} - only match if using the exp result
19157 // (result number 1).
19158 if (V.getOpcode() == ISD::FFREXP && V.getResNo() == 1) {
19159 FrexpInput = V.getOperand(i: 0);
19160 return true;
19161 }
19162 if (sd_match(N: V, P: m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
19163 Opnds: m_Value(N&: FrexpInput))))
19164 return true;
19165 return false;
19166}
19167
19168SDValue
19169SITargetLowering::performFrexpSelectCombine(SDNode *N,
19170 DAGCombinerInfo &DCI) const {
19171 // This optimization only applies when the hardware handles inf/nan correctly.
19172 if (Subtarget->hasFractBug())
19173 return SDValue();
19174
19175 SDValue Cond = N->getOperand(Num: 0);
19176 SDValue TrueVal = N->getOperand(Num: 1);
19177 SDValue FalseVal = N->getOperand(Num: 2);
19178
19179 // Identify which operand is the frexp result and which is the zero constant.
19180 // Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
19181 // Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
19182 SDValue FrexpVal;
19183 SDValue ZeroVal;
19184 bool CondSelectsZero; // If true, condition=true selects zero
19185
19186 // Check if FrexpVal comes from ISD::FFREXP (exponent result only) or
19187 // amdgcn_frexp_exp intrinsic.
19188 SDValue FrexpInput;
19189 if (isFrexpExp(V: FalseVal, FrexpInput)) {
19190 FrexpVal = FalseVal;
19191 ZeroVal = TrueVal;
19192 CondSelectsZero = true;
19193 } else if (isFrexpExp(V: TrueVal, FrexpInput)) {
19194 FrexpVal = TrueVal;
19195 ZeroVal = FalseVal;
19196 CondSelectsZero = false;
19197 } else {
19198 return SDValue();
19199 }
19200
19201 // frexp_exp returns integer, so check for integer zero.
19202 if (!isNullConstant(V: ZeroVal))
19203 return SDValue();
19204
19205 // The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
19206 SDValue FrexpInputStripped = peekFPSignOps(Val: FrexpInput);
19207
19208 bool IsNonFiniteTest = false;
19209
19210 // Handle SETCC conditions for inf/nan tests.
19211 // The canonical form of these checks is fcmp + fabs.
19212 if (Cond.getOpcode() == ISD::SETCC) {
19213 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
19214 SDValue CondLHS = Cond.getOperand(i: 0);
19215 SDValue CondRHS = Cond.getOperand(i: 1);
19216
19217 // Check if LHS is fabs(FrexpInput) - required for infinity comparisons.
19218 SDValue FAbsInput;
19219 bool LHSIsFabs = sd_match(N: CondLHS, P: m_FAbs(Op: m_Value(N&: FAbsInput)));
19220 bool LHSMatchesFrexp =
19221 (CondLHS == FrexpInput) ||
19222 (LHSIsFabs && peekFPSignOps(Val: FAbsInput) == FrexpInputStripped) ||
19223 (peekFPSignOps(Val: CondLHS) == FrexpInputStripped);
19224 bool RHSMatchesFrexp = (CondRHS == FrexpInput) ||
19225 (peekFPSignOps(Val: CondRHS) == FrexpInputStripped);
19226
19227 if (CC == ISD::SETUO) {
19228 // fcmp uno x, y - true if either x or y is NaN
19229 // We can only fold if the non-frexp operand is known to never be NaN,
19230 // otherwise the comparison could be true due to the other operand.
19231 // Special case: fcmp uno x, x (same operand) is a valid NaN test.
19232 SelectionDAG &DAG = DCI.DAG;
19233 if (LHSMatchesFrexp &&
19234 (CondLHS == CondRHS || DAG.isKnownNeverNaN(Op: CondRHS)))
19235 IsNonFiniteTest = CondSelectsZero;
19236 else if (RHSMatchesFrexp && DAG.isKnownNeverNaN(Op: CondLHS))
19237 IsNonFiniteTest = CondSelectsZero;
19238 } else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) && LHSMatchesFrexp &&
19239 LHSIsFabs &&
19240 sd_match(N: CondRHS,
19241 P: m_SpecificFP(V: APFloat::getInf(
19242 Sem: CondRHS.getValueType().getFltSemantics())))) {
19243 // fcmp oeq/ueq fabs(x), +inf - true if x is inf (or inf/nan for ueq)
19244 IsNonFiniteTest = CondSelectsZero;
19245 } else if ((CC == ISD::SETONE || CC == ISD::SETUNE) && LHSMatchesFrexp &&
19246 LHSIsFabs &&
19247 sd_match(N: CondRHS,
19248 P: m_SpecificFP(V: APFloat::getInf(
19249 Sem: CondRHS.getValueType().getFltSemantics())))) {
19250 // fcmp one/une fabs(x), +inf - true if x is NOT inf
19251 IsNonFiniteTest = !CondSelectsZero;
19252 } else if (CC == ISD::SETO) {
19253 // fcmp ord x, y - true if both are NOT NaN
19254 // We can only fold if the non-frexp operand is known to never be NaN,
19255 // otherwise the comparison could be false due to the other operand.
19256 // Special case: fcmp ord x, x (same operand) is a valid not-NaN test.
19257 SelectionDAG &DAG = DCI.DAG;
19258 if (LHSMatchesFrexp &&
19259 (CondLHS == CondRHS || DAG.isKnownNeverNaN(Op: CondRHS)))
19260 IsNonFiniteTest = !CondSelectsZero;
19261 else if (RHSMatchesFrexp && DAG.isKnownNeverNaN(Op: CondLHS))
19262 IsNonFiniteTest = !CondSelectsZero;
19263 }
19264 }
19265
19266 if (!IsNonFiniteTest)
19267 return SDValue();
19268
19269 // The select can be eliminated - just return the frexp result directly.
19270 return FrexpVal;
19271}
19272
19273SDValue SITargetLowering::performSelectCombine(SDNode *N,
19274 DAGCombinerInfo &DCI) const {
19275
19276 // Try to fold CMP + SELECT patterns with shared constants (both FP and
19277 // integer).
19278 // Detect when CMP and SELECT use the same constant and fold them to avoid
19279 // loading the constant twice. Specifically handles patterns like:
19280 // %cmp = icmp eq i32 %val, 4242
19281 // %sel = select i1 %cmp, i32 4242, i32 %other
19282 // It can be optimized to reuse %val instead of 4242 in select.
19283 SDValue Cond = N->getOperand(Num: 0);
19284 SDValue TrueVal = N->getOperand(Num: 1);
19285 SDValue FalseVal = N->getOperand(Num: 2);
19286
19287 // Check if condition is a comparison.
19288 if (Cond.getOpcode() != ISD::SETCC)
19289 return SDValue();
19290
19291 SDValue LHS = Cond.getOperand(i: 0);
19292 SDValue RHS = Cond.getOperand(i: 1);
19293 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
19294
19295 bool isFloatingPoint = LHS.getValueType().isFloatingPoint();
19296 bool isInteger = LHS.getValueType().isInteger();
19297
19298 // Handle simple floating-point and integer types only.
19299 if (!isFloatingPoint && !isInteger)
19300 return SDValue();
19301
19302 // Bare SETEQ/SETNE is the builder's NaN-impossible downgrade.
19303 bool isEquality = CC == ISD::SETEQ || (isFloatingPoint && CC == ISD::SETOEQ);
19304 bool isNonEquality =
19305 CC == ISD::SETNE || (isFloatingPoint && CC == ISD::SETONE);
19306 if (!isEquality && !isNonEquality)
19307 return SDValue();
19308
19309 SDValue ArgVal, ConstVal;
19310 if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: RHS)) ||
19311 (isInteger && isa<ConstantSDNode>(Val: RHS))) {
19312 ConstVal = RHS;
19313 ArgVal = LHS;
19314 } else if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: LHS)) ||
19315 (isInteger && isa<ConstantSDNode>(Val: LHS))) {
19316 ConstVal = LHS;
19317 ArgVal = RHS;
19318 } else {
19319 return SDValue();
19320 }
19321
19322 // Skip optimization for inlinable immediates.
19323 if (isFloatingPoint) {
19324 const APFloat &Val = cast<ConstantFPSDNode>(Val&: ConstVal)->getValueAPF();
19325 if (!Val.isNormal() || Subtarget->getInstrInfo()->isInlineConstant(Imm: Val))
19326 return SDValue();
19327 } else {
19328 const std::optional<int64_t> Val =
19329 cast<ConstantSDNode>(Val&: ConstVal)->getAPIntValue().trySExtValue();
19330 if (Val && AMDGPU::isInlinableIntLiteral(Literal: *Val))
19331 return SDValue();
19332 }
19333
19334 // For equality and non-equality comparisons, patterns:
19335 // select (setcc x, const), const, y -> select (setcc x, const), x, y
19336 // select (setccinv x, const), y, const -> select (setccinv x, const), y, x
19337 if (!(isEquality && TrueVal == ConstVal) &&
19338 !(isNonEquality && FalseVal == ConstVal))
19339 return SDValue();
19340
19341 // SETONE's false arm is also taken for NaN ArgVal, so require NaN excluded.
19342 if (isFloatingPoint && isNonEquality && FalseVal == ConstVal &&
19343 !Cond->getFlags().hasNoNaNs() && !DCI.DAG.isKnownNeverNaN(Op: ArgVal))
19344 return SDValue();
19345
19346 SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
19347 SDValue SelectRHS =
19348 (isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
19349 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Cond,
19350 N2: SelectLHS, N3: SelectRHS);
19351}
19352
19353SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
19354 DAGCombinerInfo &DCI) const {
19355 switch (N->getOpcode()) {
19356 case ISD::ABS:
19357 if (SDValue Res = promoteUniformUnaryOpToI32(Op: SDValue(N, 0), DCI))
19358 return Res;
19359 break;
19360 case ISD::ADD:
19361 case ISD::SUB:
19362 case ISD::SHL:
19363 case ISD::SRL:
19364 case ISD::SRA:
19365 case ISD::AND:
19366 case ISD::OR:
19367 case ISD::XOR:
19368 case ISD::MUL:
19369 case ISD::SETCC:
19370 case ISD::SELECT:
19371 case ISD::SMIN:
19372 case ISD::SMAX:
19373 case ISD::UMIN:
19374 case ISD::UMAX:
19375 case ISD::USUBSAT:
19376 case ISD::UADDSAT:
19377 if (auto Res = promoteUniformOpToI32(Op: SDValue(N, 0), DCI))
19378 return Res;
19379 break;
19380 default:
19381 break;
19382 }
19383
19384 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
19385 return SDValue();
19386
19387 switch (N->getOpcode()) {
19388 case ISD::ADD:
19389 return performAddCombine(N, DCI);
19390 case ISD::PTRADD:
19391 return performPtrAddCombine(N, DCI);
19392 case ISD::SUB:
19393 return performSubCombine(N, DCI);
19394 case ISD::FADD:
19395 return performFAddCombine(N, DCI);
19396 case ISD::FSUB:
19397 return performFSubCombine(N, DCI);
19398 case ISD::FDIV:
19399 return performFDivCombine(N, DCI);
19400 case ISD::FMUL:
19401 return performFMulCombine(N, DCI);
19402 case ISD::SETCC:
19403 return performSetCCCombine(N, DCI);
19404 case ISD::SELECT:
19405 if (auto Res = performFrexpSelectCombine(N, DCI))
19406 return Res;
19407 if (auto Res = performSelectCombine(N, DCI))
19408 return Res;
19409 break;
19410 case ISD::FMAXNUM:
19411 case ISD::FMINNUM:
19412 case ISD::FMAXNUM_IEEE:
19413 case ISD::FMINNUM_IEEE:
19414 case ISD::FMAXIMUM:
19415 case ISD::FMINIMUM:
19416 case ISD::FMAXIMUMNUM:
19417 case ISD::FMINIMUMNUM:
19418 case ISD::SMAX:
19419 case ISD::SMIN:
19420 case ISD::UMAX:
19421 case ISD::UMIN:
19422 case AMDGPUISD::FMIN_LEGACY:
19423 case AMDGPUISD::FMAX_LEGACY:
19424 return performMinMaxCombine(N, DCI);
19425 case ISD::FMA:
19426 return performFMACombine(N, DCI);
19427 case ISD::AND:
19428 return performAndCombine(N, DCI);
19429 case ISD::OR:
19430 return performOrCombine(N, DCI);
19431 case ISD::FSHR: {
19432 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19433 if (N->getValueType(ResNo: 0) == MVT::i32 && N->isDivergent() &&
19434 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
19435 return matchPERM(N, DCI);
19436 }
19437 break;
19438 }
19439 case ISD::XOR:
19440 return performXorCombine(N, DCI);
19441 case ISD::ANY_EXTEND:
19442 case ISD::ZERO_EXTEND:
19443 return performZeroOrAnyExtendCombine(N, DCI);
19444 case ISD::SIGN_EXTEND:
19445 return performExtendTruncShiftCombine(N, DCI);
19446 case ISD::SIGN_EXTEND_INREG:
19447 return performSignExtendInRegCombine(N, DCI);
19448 case AMDGPUISD::FP_CLASS:
19449 return performClassCombine(N, DCI);
19450 case ISD::FCANONICALIZE:
19451 return performFCanonicalizeCombine(N, DCI);
19452 case AMDGPUISD::RCP:
19453 return performRcpCombine(N, DCI);
19454 case ISD::FLDEXP:
19455 case AMDGPUISD::FRACT:
19456 case AMDGPUISD::RSQ:
19457 case AMDGPUISD::RCP_LEGACY:
19458 case AMDGPUISD::RCP_IFLAG:
19459 case AMDGPUISD::RSQ_CLAMP: {
19460 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
19461 SDValue Src = N->getOperand(Num: 0);
19462 if (Src.isUndef())
19463 return Src;
19464 break;
19465 }
19466 case ISD::SINT_TO_FP:
19467 case ISD::UINT_TO_FP:
19468 return performUCharToFloatCombine(N, DCI);
19469 case ISD::FCOPYSIGN:
19470 return performFCopySignCombine(N, DCI);
19471 case AMDGPUISD::CVT_F32_UBYTE0:
19472 case AMDGPUISD::CVT_F32_UBYTE1:
19473 case AMDGPUISD::CVT_F32_UBYTE2:
19474 case AMDGPUISD::CVT_F32_UBYTE3:
19475 return performCvtF32UByteNCombine(N, DCI);
19476 case AMDGPUISD::FMED3:
19477 return performFMed3Combine(N, DCI);
19478 case AMDGPUISD::CVT_PKRTZ_F16_F32:
19479 return performCvtPkRTZCombine(N, DCI);
19480 case AMDGPUISD::CLAMP:
19481 return performClampCombine(N, DCI);
19482 case ISD::SCALAR_TO_VECTOR: {
19483 SelectionDAG &DAG = DCI.DAG;
19484 EVT VT = N->getValueType(ResNo: 0);
19485
19486 // When bf16 inline constants live in the upper half of the expanded fp32
19487 // constant, only a splat is encodable as an inline constant. The high lane
19488 // is dead here, so splat it.
19489 if (VT == MVT::v2bf16 && Subtarget->hasBF16InlineConstFromUpperFP32()) {
19490 auto *C = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
19491 if (C && AMDGPU::isInlinableLiteralBF16(
19492 Literal: C->getValueAPF().bitcastToAPInt().getSExtValue(),
19493 HasInv2Pi: Subtarget->hasInv2PiInlineImm()))
19494 return DAG.getBuildVector(VT, DL: SDLoc(N),
19495 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 0)});
19496 }
19497
19498 // v2i16 (scalar_to_vector i16:x) -> v2i16 (bitcast (any_extend i16:x))
19499 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
19500 SDLoc SL(N);
19501 SDValue Src = N->getOperand(Num: 0);
19502 EVT EltVT = Src.getValueType();
19503 if (EltVT != MVT::i16)
19504 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
19505
19506 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Src);
19507 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Ext);
19508 }
19509
19510 break;
19511 }
19512 case ISD::EXTRACT_VECTOR_ELT:
19513 return performExtractVectorEltCombine(N, DCI);
19514 case ISD::INSERT_VECTOR_ELT:
19515 return performInsertVectorEltCombine(N, DCI);
19516 case ISD::FP_ROUND:
19517 return performFPRoundCombine(N, DCI);
19518 case ISD::LOAD: {
19519 if (SDValue Widened = widenLoad(Ld: cast<LoadSDNode>(Val: N), DCI))
19520 return Widened;
19521 [[fallthrough]];
19522 }
19523 default: {
19524 if (!DCI.isBeforeLegalize()) {
19525 if (MemSDNode *MemNode = dyn_cast<MemSDNode>(Val: N))
19526 return performMemSDNodeCombine(N: MemNode, DCI);
19527 }
19528
19529 break;
19530 }
19531 }
19532
19533 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
19534}
19535
19536/// Helper function for adjustWritemask
19537static unsigned SubIdx2Lane(unsigned Idx) {
19538 switch (Idx) {
19539 default:
19540 return ~0u;
19541 case AMDGPU::sub0:
19542 return 0;
19543 case AMDGPU::sub1:
19544 return 1;
19545 case AMDGPU::sub2:
19546 return 2;
19547 case AMDGPU::sub3:
19548 return 3;
19549 case AMDGPU::sub4:
19550 return 4; // Possible with TFE/LWE
19551 }
19552}
19553
19554/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
19555SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
19556 SelectionDAG &DAG) const {
19557 unsigned Opcode = Node->getMachineOpcode();
19558
19559 // Subtract 1 because the vdata output is not a MachineSDNode operand.
19560 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::d16) - 1;
19561 if (D16Idx >= 0 && Node->getConstantOperandVal(Num: D16Idx))
19562 return Node; // not implemented for D16
19563
19564 SDNode *Users[5] = {nullptr};
19565 unsigned Lane = 0;
19566 unsigned DmaskIdx =
19567 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::dmask) - 1;
19568 unsigned OldDmask = Node->getConstantOperandVal(Num: DmaskIdx);
19569 unsigned NewDmask = 0;
19570 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::tfe) - 1;
19571 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::lwe) - 1;
19572 bool UsesTFC = (int(TFEIdx) >= 0 && Node->getConstantOperandVal(Num: TFEIdx)) ||
19573 (int(LWEIdx) >= 0 && Node->getConstantOperandVal(Num: LWEIdx));
19574 unsigned TFCLane = 0;
19575 bool HasChain = Node->getNumValues() > 1;
19576
19577 if (OldDmask == 0) {
19578 // These are folded out, but on the chance it happens don't assert.
19579 return Node;
19580 }
19581
19582 unsigned OldBitsSet = llvm::popcount(Value: OldDmask);
19583 // Work out which is the TFE/LWE lane if that is enabled.
19584 if (UsesTFC) {
19585 TFCLane = OldBitsSet;
19586 }
19587
19588 // Try to figure out the used register components
19589 for (SDUse &Use : Node->uses()) {
19590
19591 // Don't look at users of the chain.
19592 if (Use.getResNo() != 0)
19593 continue;
19594
19595 SDNode *User = Use.getUser();
19596
19597 // Abort if we can't understand the usage
19598 if (!User->isMachineOpcode() ||
19599 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
19600 return Node;
19601
19602 // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
19603 // Note that subregs are packed, i.e. Lane==0 is the first bit set
19604 // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
19605 // set, etc.
19606 Lane = SubIdx2Lane(Idx: User->getConstantOperandVal(Num: 1));
19607 if (Lane == ~0u)
19608 return Node;
19609
19610 // Check if the use is for the TFE/LWE generated result at VGPRn+1.
19611 if (UsesTFC && Lane == TFCLane) {
19612 Users[Lane] = User;
19613 } else {
19614 // Set which texture component corresponds to the lane.
19615 unsigned Comp;
19616 for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
19617 Comp = llvm::countr_zero(Val: Dmask);
19618 Dmask &= ~(1 << Comp);
19619 }
19620
19621 // Abort if we have more than one user per component.
19622 if (Users[Lane])
19623 return Node;
19624
19625 Users[Lane] = User;
19626 NewDmask |= 1 << Comp;
19627 }
19628 }
19629
19630 // Don't allow 0 dmask, as hardware assumes one channel enabled.
19631 bool NoChannels = !NewDmask;
19632 if (NoChannels) {
19633 if (!UsesTFC) {
19634 // No uses of the result and not using TFC. Then do nothing.
19635 return Node;
19636 }
19637 // If the original dmask has one channel - then nothing to do
19638 if (OldBitsSet == 1)
19639 return Node;
19640 // Use an arbitrary dmask - required for the instruction to work
19641 NewDmask = 1;
19642 }
19643 // Abort if there's no change
19644 if (NewDmask == OldDmask)
19645 return Node;
19646
19647 unsigned BitsSet = llvm::popcount(Value: NewDmask);
19648
19649 // Check for TFE or LWE - increase the number of channels by one to account
19650 // for the extra return value
19651 // This will need adjustment for D16 if this is also included in
19652 // adjustWriteMask (this function) but at present D16 are excluded.
19653 unsigned NewChannels = BitsSet + UsesTFC;
19654
19655 int NewOpcode =
19656 AMDGPU::getMaskedMIMGOp(Opc: Node->getMachineOpcode(), NewChannels);
19657 assert(NewOpcode != -1 &&
19658 NewOpcode != static_cast<int>(Node->getMachineOpcode()) &&
19659 "failed to find equivalent MIMG op");
19660
19661 // Adjust the writemask in the node
19662 SmallVector<SDValue, 12> Ops;
19663 llvm::append_range(C&: Ops, R: Node->ops().take_front(N: DmaskIdx));
19664 Ops.push_back(Elt: DAG.getTargetConstant(Val: NewDmask, DL: SDLoc(Node), VT: MVT::i32));
19665 llvm::append_range(C&: Ops, R: Node->ops().drop_front(N: DmaskIdx + 1));
19666
19667 MVT SVT = Node->getValueType(ResNo: 0).getVectorElementType().getSimpleVT();
19668
19669 MVT ResultVT = NewChannels == 1
19670 ? SVT
19671 : MVT::getVectorVT(VT: SVT, NumElements: NewChannels == 3 ? 4
19672 : NewChannels == 5 ? 8
19673 : NewChannels);
19674 SDVTList NewVTList =
19675 HasChain ? DAG.getVTList(VT1: ResultVT, VT2: MVT::Other) : DAG.getVTList(VT: ResultVT);
19676
19677 MachineSDNode *NewNode =
19678 DAG.getMachineNode(Opcode: NewOpcode, dl: SDLoc(Node), VTs: NewVTList, Ops);
19679
19680 if (HasChain) {
19681 // Update chain.
19682 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: Node->memoperands());
19683 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: SDValue(NewNode, 1));
19684 }
19685
19686 if (NewChannels == 1) {
19687 assert(Node->hasNUsesOfValue(1, 0));
19688 SDNode *Copy =
19689 DAG.getMachineNode(Opcode: TargetOpcode::COPY, dl: SDLoc(Node),
19690 VT: Users[Lane]->getValueType(ResNo: 0), Op1: SDValue(NewNode, 0));
19691 DAG.ReplaceAllUsesWith(From: Users[Lane], To: Copy);
19692 return nullptr;
19693 }
19694
19695 // Update the users of the node with the new indices
19696 for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
19697 SDNode *User = Users[i];
19698 if (!User) {
19699 // Handle the special case of NoChannels. We set NewDmask to 1 above, but
19700 // Users[0] is still nullptr because channel 0 doesn't really have a use.
19701 if (i || !NoChannels)
19702 continue;
19703 } else {
19704 SDValue Op = DAG.getTargetConstant(Val: Idx, DL: SDLoc(User), VT: MVT::i32);
19705 SDNode *NewUser = DAG.UpdateNodeOperands(N: User, Op1: SDValue(NewNode, 0), Op2: Op);
19706 if (NewUser != User) {
19707 DAG.ReplaceAllUsesWith(From: SDValue(User, 0), To: SDValue(NewUser, 0));
19708 DAG.RemoveDeadNode(N: User);
19709 }
19710 }
19711
19712 switch (Idx) {
19713 default:
19714 break;
19715 case AMDGPU::sub0:
19716 Idx = AMDGPU::sub1;
19717 break;
19718 case AMDGPU::sub1:
19719 Idx = AMDGPU::sub2;
19720 break;
19721 case AMDGPU::sub2:
19722 Idx = AMDGPU::sub3;
19723 break;
19724 case AMDGPU::sub3:
19725 Idx = AMDGPU::sub4;
19726 break;
19727 }
19728 }
19729
19730 DAG.RemoveDeadNode(N: Node);
19731 return nullptr;
19732}
19733
19734static bool isFrameIndexOp(SDValue Op) {
19735 if (Op.getOpcode() == ISD::AssertZext)
19736 Op = Op.getOperand(i: 0);
19737
19738 return isa<FrameIndexSDNode>(Val: Op);
19739}
19740
19741/// Legalize target independent instructions (e.g. INSERT_SUBREG)
19742/// with frame index operands.
19743/// LLVM assumes that inputs are to these instructions are registers.
19744SDNode *
19745SITargetLowering::legalizeTargetIndependentNode(SDNode *Node,
19746 SelectionDAG &DAG) const {
19747 if (Node->getOpcode() == ISD::CopyToReg) {
19748 RegisterSDNode *DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1));
19749 SDValue SrcVal = Node->getOperand(Num: 2);
19750
19751 // Insert a copy to a VReg_1 virtual register so LowerI1Copies doesn't have
19752 // to try understanding copies to physical registers.
19753 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) {
19754 SDLoc SL(Node);
19755 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19756 SDValue VReg = DAG.getRegister(
19757 Reg: MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_1RegClass), VT: MVT::i1);
19758
19759 SDNode *Glued = Node->getGluedNode();
19760 SDValue ToVReg = DAG.getCopyToReg(
19761 Chain: Node->getOperand(Num: 0), dl: SL, Reg: VReg, N: SrcVal,
19762 Glue: SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0));
19763 SDValue ToResultReg = DAG.getCopyToReg(Chain: ToVReg, dl: SL, Reg: SDValue(DestReg, 0),
19764 N: VReg, Glue: ToVReg.getValue(R: 1));
19765 DAG.ReplaceAllUsesWith(From: Node, To: ToResultReg.getNode());
19766 DAG.RemoveDeadNode(N: Node);
19767 return ToResultReg.getNode();
19768 }
19769 }
19770
19771 SmallVector<SDValue, 8> Ops;
19772 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
19773 if (!isFrameIndexOp(Op: Node->getOperand(Num: i))) {
19774 Ops.push_back(Elt: Node->getOperand(Num: i));
19775 continue;
19776 }
19777
19778 SDLoc DL(Node);
19779 Ops.push_back(Elt: SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL,
19780 VT: Node->getOperand(Num: i).getValueType(),
19781 Op1: Node->getOperand(Num: i)),
19782 0));
19783 }
19784
19785 return DAG.UpdateNodeOperands(N: Node, Ops);
19786}
19787
19788/// Fold the instructions after selecting them.
19789/// Returns null if users were already updated.
19790SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
19791 SelectionDAG &DAG) const {
19792 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19793 unsigned Opcode = Node->getMachineOpcode();
19794
19795 if (TII->isImage(Opcode) && !TII->get(Opcode).mayStore() &&
19796 !TII->isGather4(Opcode) &&
19797 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::dmask)) {
19798 return adjustWritemask(Node, DAG);
19799 }
19800
19801 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
19802 legalizeTargetIndependentNode(Node, DAG);
19803 return Node;
19804 }
19805
19806 switch (Opcode) {
19807 case AMDGPU::V_DIV_SCALE_F32_e64:
19808 case AMDGPU::V_DIV_SCALE_F64_e64: {
19809 // Satisfy the operand register constraint when one of the inputs is
19810 // undefined. Ordinarily each undef value will have its own implicit_def of
19811 // a vreg, so force these to use a single register.
19812 SDValue Src0 = Node->getOperand(Num: 1);
19813 SDValue Src1 = Node->getOperand(Num: 3);
19814 SDValue Src2 = Node->getOperand(Num: 5);
19815
19816 if ((Src0.isMachineOpcode() &&
19817 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) &&
19818 (Src0 == Src1 || Src0 == Src2))
19819 break;
19820
19821 MVT VT = Src0.getValueType().getSimpleVT();
19822 const TargetRegisterClass *RC =
19823 getRegClassFor(VT, isDivergent: Src0.getNode()->isDivergent());
19824
19825 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19826 SDValue UndefReg = DAG.getRegister(Reg: MRI.createVirtualRegister(RegClass: RC), VT);
19827
19828 SDValue ImpDef = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: SDLoc(Node), Reg: UndefReg,
19829 N: Src0, Glue: SDValue());
19830
19831 // src0 must be the same register as src1 or src2, even if the value is
19832 // undefined, so make sure we don't violate this constraint.
19833 if (Src0.isMachineOpcode() &&
19834 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) {
19835 if (Src1.isMachineOpcode() &&
19836 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19837 Src0 = Src1;
19838 else if (Src2.isMachineOpcode() &&
19839 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19840 Src0 = Src2;
19841 else {
19842 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF);
19843 Src0 = UndefReg;
19844 Src1 = UndefReg;
19845 }
19846 } else
19847 break;
19848
19849 SmallVector<SDValue, 9> Ops(Node->ops());
19850 Ops[1] = Src0;
19851 Ops[3] = Src1;
19852 Ops[5] = Src2;
19853 Ops.push_back(Elt: ImpDef.getValue(R: 1));
19854 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
19855 }
19856 default:
19857 break;
19858 }
19859
19860 return Node;
19861}
19862
19863// Any MIMG instructions that use tfe or lwe require an initialization of the
19864// result register that will be written in the case of a memory access failure.
19865// The required code is also added to tie this init code to the result of the
19866// img instruction.
19867void SITargetLowering::AddMemOpInit(MachineInstr &MI) const {
19868 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19869 const SIRegisterInfo &TRI = TII->getRegisterInfo();
19870 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
19871 MachineBasicBlock &MBB = *MI.getParent();
19872
19873 int DstIdx =
19874 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
19875 unsigned InitIdx = 0;
19876
19877 if (TII->isImage(MI)) {
19878 MachineOperand *TFE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
19879 MachineOperand *LWE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
19880 MachineOperand *D16 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
19881
19882 if (!TFE && !LWE) // intersect_ray
19883 return;
19884
19885 unsigned TFEVal = TFE ? TFE->getImm() : 0;
19886 unsigned LWEVal = LWE ? LWE->getImm() : 0;
19887 unsigned D16Val = D16 ? D16->getImm() : 0;
19888
19889 if (!TFEVal && !LWEVal)
19890 return;
19891
19892 // At least one of TFE or LWE are non-zero
19893 // We have to insert a suitable initialization of the result value and
19894 // tie this to the dest of the image instruction.
19895
19896 // Calculate which dword we have to initialize to 0.
19897 MachineOperand *MO_Dmask = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
19898
19899 // check that dmask operand is found.
19900 assert(MO_Dmask && "Expected dmask operand in instruction");
19901
19902 unsigned dmask = MO_Dmask->getImm();
19903 // Determine the number of active lanes taking into account the
19904 // Gather4 special case
19905 unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(Value: dmask);
19906
19907 bool Packed = !Subtarget->hasUnpackedD16VMem();
19908
19909 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
19910
19911 // Abandon attempt if the dst size isn't large enough
19912 // - this is in fact an error but this is picked up elsewhere and
19913 // reported correctly.
19914 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19915
19916 uint32_t DstSize = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19917 if (DstSize < InitIdx)
19918 return;
19919 } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(Opc: MI.getOpcode())) {
19920 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19921 InitIdx = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19922 } else {
19923 return;
19924 }
19925
19926 const DebugLoc &DL = MI.getDebugLoc();
19927
19928 // Create a register for the initialization value.
19929 Register PrevDst = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: DstIdx).getReg());
19930 unsigned NewDst = 0; // Final initialized value will be in here
19931
19932 // If PRTStrictNull feature is enabled (the default) then initialize
19933 // all the result registers to 0, otherwise just the error indication
19934 // register (VGPRn+1)
19935 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1;
19936 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1);
19937
19938 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: PrevDst);
19939 for (; SizeLeft; SizeLeft--, CurrIdx++) {
19940 NewDst = MRI.createVirtualRegister(RegClass: TII->getOpRegClass(MI, OpNo: DstIdx));
19941 // Initialize dword
19942 Register SubReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
19943 // clang-format off
19944 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: SubReg)
19945 .addImm(Val: 0);
19946 // clang-format on
19947 // Insert into the super-reg
19948 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewDst)
19949 .addReg(RegNo: PrevDst)
19950 .addReg(RegNo: SubReg)
19951 .addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: CurrIdx));
19952
19953 PrevDst = NewDst;
19954 }
19955
19956 // Add as an implicit operand
19957 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewDst, isDef: false, isImp: true));
19958
19959 // Tie the just added implicit operand to the dst
19960 MI.tieOperands(DefIdx: DstIdx, UseIdx: MI.getNumOperands() - 1);
19961}
19962
19963/// Assign the register class depending on the number of
19964/// bits set in the writemask
19965void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
19966 SDNode *Node) const {
19967 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19968
19969 MachineFunction *MF = MI.getMF();
19970 MachineRegisterInfo &MRI = MF->getRegInfo();
19971
19972 if (TII->isVOP3(Opcode: MI.getOpcode())) {
19973 // Make sure constant bus requirements are respected.
19974 TII->legalizeOperandsVOP3(MRI, MI);
19975
19976 // Fix GFX950 v_pk_fma_f32 invalid op_sel by decomposing.
19977 if (MI.getOpcode() == AMDGPU::V_PK_FMA_F32 &&
19978 !TII->isLegalPkFMAF32OpSel(MI)) {
19979 TII->decomposePkFMAF32(MI, MRI);
19980 return;
19981 }
19982
19983 if (TII->isMAI(MI)) {
19984 // The ordinary src0, src1, src2 were legalized above.
19985 //
19986 // We have to also legalize the appended v_mfma_ld_scale_b32 operands,
19987 // as a separate instruction.
19988 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19989 Name: AMDGPU::OpName::scale_src0);
19990 if (Src0Idx != -1) {
19991 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19992 Name: AMDGPU::OpName::scale_src1);
19993 if (TII->usesConstantBus(MRI, MI, OpIdx: Src0Idx) &&
19994 TII->usesConstantBus(MRI, MI, OpIdx: Src1Idx))
19995 TII->legalizeOpWithMove(MI, OpIdx: Src1Idx);
19996 }
19997 }
19998
19999 return;
20000 }
20001
20002 if (TII->isImage(MI))
20003 TII->enforceOperandRCAlignment(MI, OpName: AMDGPU::OpName::vaddr);
20004}
20005
20006static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
20007 uint64_t Val) {
20008 SDValue K = DAG.getTargetConstant(Val, DL, VT: MVT::i32);
20009 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: K), 0);
20010}
20011
20012MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG,
20013 const SDLoc &DL,
20014 SDValue Ptr) const {
20015 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20016
20017 // Build the half of the subregister with the constants before building the
20018 // full 128-bit register. If we are building multiple resource descriptors,
20019 // this will allow CSEing of the 2-component register.
20020 const SDValue Ops0[] = {
20021 DAG.getTargetConstant(Val: AMDGPU::SGPR_64RegClassID, DL, VT: MVT::i32),
20022 buildSMovImm32(DAG, DL, Val: 0),
20023 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
20024 buildSMovImm32(DAG, DL, Val: TII->getDefaultRsrcDataFormat() >> 32),
20025 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32)};
20026
20027 SDValue SubRegHi = SDValue(
20028 DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v2i32, Ops: Ops0), 0);
20029
20030 // Combine the constants and the pointer.
20031 const SDValue Ops1[] = {
20032 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32), Ptr,
20033 DAG.getTargetConstant(Val: AMDGPU::sub0_sub1, DL, VT: MVT::i32), SubRegHi,
20034 DAG.getTargetConstant(Val: AMDGPU::sub2_sub3, DL, VT: MVT::i32)};
20035
20036 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops: Ops1);
20037}
20038
20039/// Return a resource descriptor with the 'Add TID' bit enabled
20040/// The TID (Thread ID) is multiplied by the stride value (bits [61:48]
20041/// of the resource descriptor) to create an offset, which is added to
20042/// the resource pointer.
20043MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL,
20044 SDValue Ptr, uint32_t RsrcDword1,
20045 uint64_t RsrcDword2And3) const {
20046 SDValue PtrLo = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub0, DL, VT: MVT::i32, Operand: Ptr);
20047 SDValue PtrHi = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub1, DL, VT: MVT::i32, Operand: Ptr);
20048 if (RsrcDword1) {
20049 PtrHi = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: PtrHi,
20050 N2: DAG.getConstant(Val: RsrcDword1, DL, VT: MVT::i32));
20051 }
20052
20053 SDValue DataLo =
20054 buildSMovImm32(DAG, DL, Val: RsrcDword2And3 & UINT64_C(0xFFFFFFFF));
20055 SDValue DataHi = buildSMovImm32(DAG, DL, Val: RsrcDword2And3 >> 32);
20056
20057 const SDValue Ops[] = {
20058 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32),
20059 PtrLo,
20060 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
20061 PtrHi,
20062 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32),
20063 DataLo,
20064 DAG.getTargetConstant(Val: AMDGPU::sub2, DL, VT: MVT::i32),
20065 DataHi,
20066 DAG.getTargetConstant(Val: AMDGPU::sub3, DL, VT: MVT::i32)};
20067
20068 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops);
20069}
20070
20071//===----------------------------------------------------------------------===//
20072// SI Inline Assembly Support
20073//===----------------------------------------------------------------------===//
20074
20075std::pair<unsigned, const TargetRegisterClass *>
20076SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
20077 StringRef Constraint,
20078 MVT VT) const {
20079 const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(TRI_);
20080
20081 const TargetRegisterClass *RC = nullptr;
20082 if (Constraint.size() == 1) {
20083 // Check if we cannot determine the bit size of the given value type. This
20084 // can happen, for example, in this situation where we have an empty struct
20085 // (size 0): `call void asm "", "v"({} poison)`-
20086 if (VT == MVT::Other)
20087 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20088 const unsigned BitWidth = VT.getSizeInBits();
20089 switch (Constraint[0]) {
20090 default:
20091 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20092 case 's':
20093 case 'r':
20094 switch (BitWidth) {
20095 case 16:
20096 RC = &AMDGPU::SReg_32RegClass;
20097 break;
20098 case 64:
20099 RC = &AMDGPU::SGPR_64RegClass;
20100 break;
20101 default:
20102 RC = SIRegisterInfo::getSGPRClassForBitWidth(BitWidth);
20103 if (!RC)
20104 return std::pair(0U, nullptr);
20105 break;
20106 }
20107 break;
20108 case 'v':
20109 switch (BitWidth) {
20110 case 1:
20111 return std::pair(0U, nullptr);
20112 case 16:
20113 RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass
20114 : &AMDGPU::VGPR_32_Lo256RegClass;
20115 break;
20116 default:
20117 RC = Subtarget->has1024AddressableVGPRs()
20118 ? TRI->getAlignedLo256VGPRClassForBitWidth(BitWidth)
20119 : TRI->getVGPRClassForBitWidth(BitWidth);
20120 if (!RC)
20121 return std::pair(0U, nullptr);
20122 break;
20123 }
20124 break;
20125 case 'a':
20126 if (!Subtarget->hasMAIInsts())
20127 break;
20128 switch (BitWidth) {
20129 case 1:
20130 return std::pair(0U, nullptr);
20131 case 16:
20132 RC = &AMDGPU::AGPR_32RegClass;
20133 break;
20134 default:
20135 RC = TRI->getAGPRClassForBitWidth(BitWidth);
20136 if (!RC)
20137 return std::pair(0U, nullptr);
20138 break;
20139 }
20140 break;
20141 }
20142 } else if (Constraint == "VA" && Subtarget->hasGFX90AInsts()) {
20143 const unsigned BitWidth = VT.getSizeInBits();
20144 switch (BitWidth) {
20145 case 16:
20146 RC = &AMDGPU::AV_32RegClass;
20147 break;
20148 default:
20149 RC = TRI->getVectorSuperClassForBitWidth(BitWidth);
20150 if (!RC)
20151 return std::pair(0U, nullptr);
20152 break;
20153 }
20154 }
20155
20156 // We actually support i128, i16 and f16 as inline parameters
20157 // even if they are not reported as legal
20158 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
20159 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
20160 return std::pair(0U, RC);
20161
20162 auto [Kind, Idx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint);
20163 if (Kind != '\0') {
20164 if (Kind == 'v') {
20165 RC = &AMDGPU::VGPR_32_Lo256RegClass;
20166 } else if (Kind == 's') {
20167 RC = &AMDGPU::SGPR_32RegClass;
20168 } else if (Kind == 'a') {
20169 RC = &AMDGPU::AGPR_32RegClass;
20170 }
20171
20172 if (RC) {
20173 if (NumRegs > 1) {
20174 if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 >= RC->getNumRegs())
20175 return std::pair(0U, nullptr);
20176
20177 uint32_t Width = NumRegs * 32;
20178 // Prohibit constraints for register ranges with a width that does not
20179 // match the required type.
20180 if (VT.SimpleTy != MVT::Other && Width != VT.getSizeInBits())
20181 return std::pair(0U, nullptr);
20182
20183 MCRegister Reg = RC->getRegister(i: Idx);
20184 if (SIRegisterInfo::isVGPRClass(RC))
20185 RC = TRI->getVGPRClassForBitWidth(BitWidth: Width);
20186 else if (SIRegisterInfo::isSGPRClass(RC))
20187 RC = TRI->getSGPRClassForBitWidth(BitWidth: Width);
20188 else if (SIRegisterInfo::isAGPRClass(RC))
20189 RC = TRI->getAGPRClassForBitWidth(BitWidth: Width);
20190 if (RC) {
20191 Reg = TRI->getMatchingSuperReg(Reg, SubIdx: AMDGPU::sub0, RC);
20192 if (!Reg) {
20193 // The register class does not contain the requested register,
20194 // e.g., because it is an SGPR pair that would violate alignment
20195 // requirements.
20196 return std::pair(0U, nullptr);
20197 }
20198 return std::pair(Reg, RC);
20199 }
20200 }
20201
20202 // Reject types that do not fit a single 32-bit register: any scalar wider
20203 // than 32 bits, or a vector that is not exactly 32 bits.
20204 if (VT.SimpleTy != MVT::Other &&
20205 (VT.getSizeInBits() > 32 ||
20206 (VT.isVector() && VT.getSizeInBits() != 32)))
20207 return std::pair(0U, nullptr);
20208 if (RC && Idx < RC->getNumRegs())
20209 return std::pair(RC->getRegister(i: Idx), RC);
20210 return std::pair(0U, nullptr);
20211 }
20212 }
20213
20214 auto Ret = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20215 if (Ret.first)
20216 Ret.second = TRI->getPhysRegBaseClass(Reg: Ret.first);
20217
20218 return Ret;
20219}
20220
20221static bool isImmConstraint(StringRef Constraint) {
20222 if (Constraint.size() == 1) {
20223 switch (Constraint[0]) {
20224 default:
20225 break;
20226 case 'I':
20227 case 'J':
20228 case 'A':
20229 case 'B':
20230 case 'C':
20231 return true;
20232 }
20233 } else if (Constraint == "DA" || Constraint == "DB") {
20234 return true;
20235 }
20236 return false;
20237}
20238
20239SITargetLowering::ConstraintType
20240SITargetLowering::getConstraintType(StringRef Constraint) const {
20241 if (Constraint.size() == 1) {
20242 switch (Constraint[0]) {
20243 default:
20244 break;
20245 case 's':
20246 case 'v':
20247 case 'a':
20248 return C_RegisterClass;
20249 }
20250 } else if (Constraint.size() == 2) {
20251 if (Constraint == "VA")
20252 return C_RegisterClass;
20253 }
20254 if (isImmConstraint(Constraint)) {
20255 return C_Other;
20256 }
20257 return TargetLowering::getConstraintType(Constraint);
20258}
20259
20260static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) {
20261 if (!AMDGPU::isInlinableIntLiteral(Literal: Val)) {
20262 Val = Val & maskTrailingOnes<uint64_t>(N: Size);
20263 }
20264 return Val;
20265}
20266
20267void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op,
20268 StringRef Constraint,
20269 std::vector<SDValue> &Ops,
20270 SelectionDAG &DAG) const {
20271 if (isImmConstraint(Constraint)) {
20272 uint64_t Val;
20273 if (getAsmOperandConstVal(Op, Val) &&
20274 checkAsmConstraintVal(Op, Constraint, Val)) {
20275 Val = clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits());
20276 Ops.push_back(x: DAG.getTargetConstant(Val, DL: SDLoc(Op), VT: MVT::i64));
20277 }
20278 } else {
20279 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
20280 }
20281}
20282
20283bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const {
20284 unsigned Size = Op.getScalarValueSizeInBits();
20285 if (Size > 64)
20286 return false;
20287
20288 if (Size == 16 && !Subtarget->has16BitInsts())
20289 return false;
20290
20291 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
20292 Val = C->getSExtValue();
20293 return true;
20294 }
20295 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
20296 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
20297 return true;
20298 }
20299 if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
20300 if (Size != 16 || Op.getNumOperands() != 2)
20301 return false;
20302 if (Op.getOperand(i: 0).isUndef() || Op.getOperand(i: 1).isUndef())
20303 return false;
20304 if (ConstantSDNode *C = V->getConstantSplatNode()) {
20305 Val = C->getSExtValue();
20306 return true;
20307 }
20308 if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
20309 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
20310 return true;
20311 }
20312 }
20313
20314 return false;
20315}
20316
20317bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint,
20318 uint64_t Val) const {
20319 if (Constraint.size() == 1) {
20320 switch (Constraint[0]) {
20321 case 'I':
20322 return AMDGPU::isInlinableIntLiteral(Literal: Val);
20323 case 'J':
20324 return isInt<16>(x: Val);
20325 case 'A':
20326 return checkAsmConstraintValA(Op, Val);
20327 case 'B':
20328 return isInt<32>(x: Val);
20329 case 'C':
20330 return isUInt<32>(x: clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits())) ||
20331 AMDGPU::isInlinableIntLiteral(Literal: Val);
20332 default:
20333 break;
20334 }
20335 } else if (Constraint.size() == 2) {
20336 if (Constraint == "DA") {
20337 int64_t HiBits = static_cast<int32_t>(Val >> 32);
20338 int64_t LoBits = static_cast<int32_t>(Val);
20339 return checkAsmConstraintValA(Op, Val: HiBits, MaxSize: 32) &&
20340 checkAsmConstraintValA(Op, Val: LoBits, MaxSize: 32);
20341 }
20342 if (Constraint == "DB") {
20343 return true;
20344 }
20345 }
20346 llvm_unreachable("Invalid asm constraint");
20347}
20348
20349bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val,
20350 unsigned MaxSize) const {
20351 unsigned Size = std::min<unsigned>(a: Op.getScalarValueSizeInBits(), b: MaxSize);
20352 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
20353 if (Size == 16) {
20354 MVT VT = Op.getSimpleValueType();
20355 switch (VT.SimpleTy) {
20356 default:
20357 return false;
20358 case MVT::i16:
20359 return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi);
20360 case MVT::f16:
20361 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi);
20362 case MVT::bf16:
20363 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi);
20364 case MVT::v2i16:
20365 return AMDGPU::getInlineEncodingV2I16(Literal: Val).has_value();
20366 case MVT::v2f16:
20367 return AMDGPU::getInlineEncodingV2F16(Literal: Val).has_value();
20368 case MVT::v2bf16:
20369 return AMDGPU::getInlineEncodingV2BF16(Literal: Val).has_value();
20370 }
20371 }
20372 if ((Size == 32 && AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi)) ||
20373 (Size == 64 && AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi)))
20374 return true;
20375 return false;
20376}
20377
20378static int getAlignedAGPRClassID(unsigned UnalignedClassID) {
20379 switch (UnalignedClassID) {
20380 case AMDGPU::VReg_64RegClassID:
20381 return AMDGPU::VReg_64_Align2RegClassID;
20382 case AMDGPU::VReg_96RegClassID:
20383 return AMDGPU::VReg_96_Align2RegClassID;
20384 case AMDGPU::VReg_128RegClassID:
20385 return AMDGPU::VReg_128_Align2RegClassID;
20386 case AMDGPU::VReg_160RegClassID:
20387 return AMDGPU::VReg_160_Align2RegClassID;
20388 case AMDGPU::VReg_192RegClassID:
20389 return AMDGPU::VReg_192_Align2RegClassID;
20390 case AMDGPU::VReg_224RegClassID:
20391 return AMDGPU::VReg_224_Align2RegClassID;
20392 case AMDGPU::VReg_256RegClassID:
20393 return AMDGPU::VReg_256_Align2RegClassID;
20394 case AMDGPU::VReg_288RegClassID:
20395 return AMDGPU::VReg_288_Align2RegClassID;
20396 case AMDGPU::VReg_320RegClassID:
20397 return AMDGPU::VReg_320_Align2RegClassID;
20398 case AMDGPU::VReg_352RegClassID:
20399 return AMDGPU::VReg_352_Align2RegClassID;
20400 case AMDGPU::VReg_384RegClassID:
20401 return AMDGPU::VReg_384_Align2RegClassID;
20402 case AMDGPU::VReg_512RegClassID:
20403 return AMDGPU::VReg_512_Align2RegClassID;
20404 case AMDGPU::VReg_1024RegClassID:
20405 return AMDGPU::VReg_1024_Align2RegClassID;
20406 case AMDGPU::AReg_64RegClassID:
20407 return AMDGPU::AReg_64_Align2RegClassID;
20408 case AMDGPU::AReg_96RegClassID:
20409 return AMDGPU::AReg_96_Align2RegClassID;
20410 case AMDGPU::AReg_128RegClassID:
20411 return AMDGPU::AReg_128_Align2RegClassID;
20412 case AMDGPU::AReg_160RegClassID:
20413 return AMDGPU::AReg_160_Align2RegClassID;
20414 case AMDGPU::AReg_192RegClassID:
20415 return AMDGPU::AReg_192_Align2RegClassID;
20416 case AMDGPU::AReg_256RegClassID:
20417 return AMDGPU::AReg_256_Align2RegClassID;
20418 case AMDGPU::AReg_512RegClassID:
20419 return AMDGPU::AReg_512_Align2RegClassID;
20420 case AMDGPU::AReg_1024RegClassID:
20421 return AMDGPU::AReg_1024_Align2RegClassID;
20422 default:
20423 return -1;
20424 }
20425}
20426
20427// Figure out which registers should be reserved for stack access. Only after
20428// the function is legalized do we know all of the non-spill stack objects or if
20429// calls are present.
20430void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
20431 MachineRegisterInfo &MRI = MF.getRegInfo();
20432 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20433 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
20434 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20435 const SIInstrInfo *TII = ST.getInstrInfo();
20436
20437 if (Info->isEntryFunction()) {
20438 // Callable functions have fixed registers used for stack access.
20439 reservePrivateMemoryRegs(TM: getTargetMachine(), MF, TRI: *TRI, Info&: *Info);
20440 }
20441
20442 // TODO: Move this logic to getReservedRegs()
20443 // Reserve the SGPR(s) to save/restore EXEC for WWM spill/copy handling.
20444 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
20445 Register SReg = ST.isWave32()
20446 ? AMDGPU::SGPR_32RegClass.getRegister(i: MaxNumSGPRs - 1)
20447 : TRI->getAlignedHighSGPRForRC(MF, /*Align=*/2,
20448 RC: &AMDGPU::SGPR_64RegClass);
20449 Info->setSGPRForEXECCopy(SReg);
20450
20451 assert(!TRI->isSubRegister(Info->getScratchRSrcReg(),
20452 Info->getStackPtrOffsetReg()));
20453 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
20454 MRI.replaceRegWith(FromReg: AMDGPU::SP_REG, ToReg: Info->getStackPtrOffsetReg());
20455
20456 // We need to worry about replacing the default register with itself in case
20457 // of MIR testcases missing the MFI.
20458 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
20459 MRI.replaceRegWith(FromReg: AMDGPU::PRIVATE_RSRC_REG, ToReg: Info->getScratchRSrcReg());
20460
20461 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG)
20462 MRI.replaceRegWith(FromReg: AMDGPU::FP_REG, ToReg: Info->getFrameOffsetReg());
20463
20464 Info->limitOccupancy(MF);
20465
20466 if (ST.isWave32() && !MF.empty()) {
20467 for (auto &MBB : MF) {
20468 for (auto &MI : MBB) {
20469 TII->fixImplicitOperands(MI);
20470 }
20471 }
20472 }
20473
20474 // FIXME: This is a hack to fixup AGPR classes to use the properly aligned
20475 // classes if required. Ideally the register class constraints would differ
20476 // per-subtarget, but there's no easy way to achieve that right now. This is
20477 // not a problem for VGPRs because the correctly aligned VGPR class is implied
20478 // from using them as the register class for legal types.
20479 if (ST.needsAlignedVGPRs()) {
20480 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
20481 const Register Reg = Register::index2VirtReg(Index: I);
20482 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
20483 if (!RC)
20484 continue;
20485 int NewClassID = getAlignedAGPRClassID(UnalignedClassID: RC->getID());
20486 if (NewClassID != -1)
20487 MRI.setRegClass(Reg, RC: TRI->getRegClass(i: NewClassID));
20488 }
20489 }
20490
20491 TargetLoweringBase::finalizeLowering(MF);
20492}
20493
20494void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
20495 KnownBits &Known,
20496 const APInt &DemandedElts,
20497 const SelectionDAG &DAG,
20498 unsigned Depth) const {
20499 Known.resetAll();
20500 unsigned Opc = Op.getOpcode();
20501 switch (Opc) {
20502 case ISD::INTRINSIC_WO_CHAIN: {
20503 unsigned IID = Op.getConstantOperandVal(i: 0);
20504 switch (IID) {
20505 case Intrinsic::amdgcn_mbcnt_lo:
20506 case Intrinsic::amdgcn_mbcnt_hi: {
20507 const GCNSubtarget &ST =
20508 DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
20509 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20510 // most 31 + src1.
20511 Known.Zero.setBitsFrom(
20512 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
20513 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
20514 Known = KnownBits::add(LHS: Known, RHS: Known2);
20515 return;
20516 }
20517 }
20518 break;
20519 }
20520 }
20521 return AMDGPUTargetLowering::computeKnownBitsForTargetNode(
20522 Op, Known, DemandedElts, DAG, Depth);
20523}
20524
20525void SITargetLowering::computeKnownBitsForStackObjectPointer(
20526 KnownBits &Known, const MachineFunction &MF, Align Alignment) const {
20527 TargetLowering::computeKnownBitsForStackObjectPointer(Known, MF, Alignment);
20528
20529 // Set the high bits to zero based on the maximum allowed scratch size per
20530 // wave. We can't use vaddr in MUBUF instructions if we don't know the address
20531 // calculation won't overflow, so assume the sign bit is never set.
20532 Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex());
20533}
20534
20535static void knownBitsForWorkitemID(const GCNSubtarget &ST,
20536 GISelValueTracking &VT, KnownBits &Known,
20537 unsigned Dim) {
20538 unsigned MaxValue =
20539 ST.getMaxWorkitemID(Kernel: VT.getMachineFunction().getFunction(), Dimension: Dim);
20540 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
20541}
20542
20543static void knownBitsForSBFE(const MachineInstr &MI, GISelValueTracking &VT,
20544 KnownBits &Known, const APInt &DemandedElts,
20545 unsigned BFEWidth, bool SExt, unsigned Depth) {
20546 const MachineRegisterInfo &MRI = VT.getMachineFunction().getRegInfo();
20547 const MachineOperand &Src1 = MI.getOperand(i: 2);
20548
20549 unsigned Src1Cst = 0;
20550 if (Src1.isImm()) {
20551 Src1Cst = Src1.getImm();
20552 } else if (Src1.isReg()) {
20553 auto Cst = getIConstantVRegValWithLookThrough(VReg: Src1.getReg(), MRI);
20554 if (!Cst)
20555 return;
20556 Src1Cst = Cst->Value.getZExtValue();
20557 } else {
20558 return;
20559 }
20560
20561 // Offset is at bits [4:0] for 32 bit, [5:0] for 64 bit.
20562 // Width is always [22:16].
20563 const unsigned Offset =
20564 Src1Cst & maskTrailingOnes<unsigned>(N: (BFEWidth == 32) ? 5 : 6);
20565 const unsigned Width = (Src1Cst >> 16) & maskTrailingOnes<unsigned>(N: 6);
20566
20567 if (Width >= BFEWidth) // Ill-formed.
20568 return;
20569
20570 VT.computeKnownBitsImpl(R: MI.getOperand(i: 1).getReg(), Known, DemandedElts,
20571 Depth: Depth + 1);
20572
20573 Known = Known.extractBits(NumBits: Width, BitPosition: Offset);
20574
20575 if (SExt)
20576 Known = Known.sext(BitWidth: BFEWidth);
20577 else
20578 Known = Known.zext(BitWidth: BFEWidth);
20579}
20580
20581void SITargetLowering::computeKnownBitsForTargetInstr(
20582 GISelValueTracking &VT, Register R, KnownBits &Known,
20583 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
20584 unsigned Depth) const {
20585 Known.resetAll();
20586 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
20587 switch (MI->getOpcode()) {
20588 case AMDGPU::S_BFE_I32:
20589 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
20590 /*SExt=*/true, Depth);
20591 case AMDGPU::S_BFE_U32:
20592 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
20593 /*SExt=*/false, Depth);
20594 case AMDGPU::S_BFE_I64:
20595 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
20596 /*SExt=*/true, Depth);
20597 case AMDGPU::S_BFE_U64:
20598 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
20599 /*SExt=*/false, Depth);
20600 case AMDGPU::G_INTRINSIC:
20601 case AMDGPU::G_INTRINSIC_CONVERGENT: {
20602 Intrinsic::ID IID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
20603 switch (IID) {
20604 case Intrinsic::amdgcn_workitem_id_x:
20605 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 0);
20606 break;
20607 case Intrinsic::amdgcn_workitem_id_y:
20608 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 1);
20609 break;
20610 case Intrinsic::amdgcn_workitem_id_z:
20611 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 2);
20612 break;
20613 case Intrinsic::amdgcn_mbcnt_lo:
20614 case Intrinsic::amdgcn_mbcnt_hi: {
20615 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20616 // most 31 + src1.
20617 Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
20618 ? getSubtarget()->getWavefrontSizeLog2()
20619 : 5);
20620 KnownBits Known2;
20621 VT.computeKnownBitsImpl(R: MI->getOperand(i: 3).getReg(), Known&: Known2, DemandedElts,
20622 Depth: Depth + 1);
20623 Known = KnownBits::add(LHS: Known, RHS: Known2);
20624 break;
20625 }
20626 case Intrinsic::amdgcn_groupstaticsize: {
20627 // We can report everything over the maximum size as 0. We can't report
20628 // based on the actual size because we don't know if it's accurate or not
20629 // at any given point.
20630 Known.Zero.setHighBits(
20631 llvm::countl_zero(Val: getSubtarget()->getAddressableLocalMemorySize()));
20632 break;
20633 }
20634 case Intrinsic::amdgcn_readfirstlane:
20635 case Intrinsic::amdgcn_readlane: {
20636 // Result is the data operand's value from some lane.
20637 VT.computeKnownBitsImpl(R: MI->getOperand(i: 2).getReg(), Known, DemandedElts,
20638 Depth: Depth + 1);
20639 break;
20640 }
20641 }
20642 break;
20643 }
20644 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
20645 Known.Zero.setHighBits(24);
20646 break;
20647 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
20648 Known.Zero.setHighBits(16);
20649 break;
20650 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
20651 // G_AMDGPU_COPY_SCC_VCC converts a uniform boolean in VCC to SGPR s32,
20652 // producing exactly 0 or 1.
20653 Known.Zero.setHighBits(Known.getBitWidth() - 1);
20654 break;
20655 case AMDGPU::G_AMDGPU_SMED3:
20656 case AMDGPU::G_AMDGPU_UMED3: {
20657 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
20658
20659 KnownBits Known2;
20660 VT.computeKnownBitsImpl(R: Src2, Known&: Known2, DemandedElts, Depth: Depth + 1);
20661 if (Known2.isUnknown())
20662 break;
20663
20664 KnownBits Known1;
20665 VT.computeKnownBitsImpl(R: Src1, Known&: Known1, DemandedElts, Depth: Depth + 1);
20666 if (Known1.isUnknown())
20667 break;
20668
20669 KnownBits Known0;
20670 VT.computeKnownBitsImpl(R: Src0, Known&: Known0, DemandedElts, Depth: Depth + 1);
20671 if (Known0.isUnknown())
20672 break;
20673
20674 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
20675 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
20676 Known.One = Known0.One & Known1.One & Known2.One;
20677 break;
20678 }
20679 }
20680}
20681
20682Align SITargetLowering::computeKnownAlignForTargetInstr(
20683 GISelValueTracking &VT, Register R, const MachineRegisterInfo &MRI,
20684 unsigned Depth) const {
20685 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
20686 if (auto *GI = dyn_cast<GIntrinsic>(Val: MI)) {
20687 // FIXME: Can this move to generic code? What about the case where the call
20688 // site specifies a lower alignment?
20689 Intrinsic::ID IID = GI->getIntrinsicID();
20690 LLVMContext &Ctx = VT.getMachineFunction().getFunction().getContext();
20691 AttributeList Attrs =
20692 Intrinsic::getAttributes(C&: Ctx, id: IID, FT: Intrinsic::getType(Context&: Ctx, id: IID));
20693 if (MaybeAlign RetAlign = Attrs.getRetAlignment())
20694 return *RetAlign;
20695 }
20696 return Align(1);
20697}
20698
20699Align SITargetLowering::getPrefLoopAlignment(
20700 MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
20701 const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
20702 const Align CacheLineAlign = Align(64);
20703
20704 // GFX950: Prevent an 8-byte instruction at the block being aligned from being
20705 // split by the 32-byte instruction fetch window boundary. This avoids a
20706 // significant fetch delay after a backward branch. We use 32-byte alignment
20707 // with max padding of 4 bytes (one s_nop), see
20708 // getMaxPermittedBytesForAlignment().
20709 if (ML && !DisableLoopAlignment &&
20710 getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
20711 // Loop rotation can make the backedge destination a block other than the
20712 // LoopInfo header, so prefer the block the caller is actually aligning.
20713 if (!BlockToAlign)
20714 BlockToAlign = ML->getHeader();
20715 // Respect user-specified or previously set alignment.
20716 if (BlockToAlign->getAlignment() != PrefAlign)
20717 return BlockToAlign->getAlignment();
20718 if (needsFetchWindowAlignment(MBB: *BlockToAlign))
20719 return Align(32);
20720 }
20721
20722 // Pre-GFX10 target did not benefit from loop alignment
20723 if (!ML || DisableLoopAlignment || !getSubtarget()->hasInstPrefetch() ||
20724 getSubtarget()->hasInstFwdPrefetchBug())
20725 return PrefAlign;
20726
20727 // On GFX10 I$ is 4 x 64 bytes cache lines.
20728 // By default prefetcher keeps one cache line behind and reads two ahead.
20729 // We can modify it with S_INST_PREFETCH for larger loops to have two lines
20730 // behind and one ahead.
20731 // Therefor we can benefit from aligning loop headers if loop fits 192 bytes.
20732 // If loop fits 64 bytes it always spans no more than two cache lines and
20733 // does not need an alignment.
20734 // Else if loop is less or equal 128 bytes we do not need to modify prefetch,
20735 // Else if loop is less or equal 192 bytes we need two lines behind.
20736
20737 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20738 const MachineBasicBlock *Header = ML->getHeader();
20739 if (Header->getAlignment() != PrefAlign)
20740 return Header->getAlignment(); // Already processed.
20741
20742 unsigned LoopSize = 0;
20743 for (const MachineBasicBlock *MBB : ML->blocks()) {
20744 // If inner loop block is aligned assume in average half of the alignment
20745 // size to be added as nops.
20746 if (MBB != Header)
20747 LoopSize += MBB->getAlignment().value() / 2;
20748
20749 for (const MachineInstr &MI : *MBB) {
20750 LoopSize += TII->getInstSizeInBytes(MI);
20751 if (LoopSize > 192)
20752 return PrefAlign;
20753 }
20754 }
20755
20756 if (LoopSize <= 64)
20757 return PrefAlign;
20758
20759 if (LoopSize <= 128)
20760 return CacheLineAlign;
20761
20762 // If any of parent loops is surrounded by prefetch instructions do not
20763 // insert new for inner loop, which would reset parent's settings.
20764 for (MachineLoop *P = ML->getParentLoop(); P; P = P->getParentLoop()) {
20765 if (MachineBasicBlock *Exit = P->getExitBlock()) {
20766 auto I = Exit->getFirstNonDebugInstr();
20767 if (I != Exit->end() && I->getOpcode() == AMDGPU::S_INST_PREFETCH)
20768 return CacheLineAlign;
20769 }
20770 }
20771
20772 MachineBasicBlock *Pre = ML->getLoopPreheader();
20773 MachineBasicBlock *Exit = ML->getExitBlock();
20774
20775 if (Pre && Exit) {
20776 auto PreTerm = Pre->getFirstTerminator();
20777 if (PreTerm == Pre->begin() ||
20778 std::prev(x: PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
20779 BuildMI(BB&: *Pre, I: PreTerm, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20780 .addImm(Val: 1); // prefetch 2 lines behind PC
20781
20782 auto ExitHead = Exit->getFirstNonDebugInstr();
20783 if (ExitHead == Exit->end() ||
20784 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
20785 BuildMI(BB&: *Exit, I: ExitHead, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20786 .addImm(Val: 2); // prefetch 1 line behind PC
20787 }
20788
20789 return CacheLineAlign;
20790}
20791
20792unsigned SITargetLowering::getMaxPermittedBytesForAlignment(
20793 MachineBasicBlock *MBB) const {
20794 // GFX950: Limit padding to 4 bytes (one s_nop) for blocks where an 8-byte
20795 // instruction could be split by the 32-byte fetch window boundary.
20796 // See getPrefLoopAlignment() for context.
20797 if (needsFetchWindowAlignment(MBB: *MBB))
20798 return 4;
20799 return TargetLowering::getMaxPermittedBytesForAlignment(MBB);
20800}
20801
20802bool SITargetLowering::needsFetchWindowAlignment(
20803 const MachineBasicBlock &MBB) const {
20804 if (!getSubtarget()->hasLoopHeadInstSplitSensitivity())
20805 return false;
20806 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20807 for (const MachineInstr &MI : MBB) {
20808 if (MI.isMetaInstruction())
20809 continue;
20810 // Instructions larger than 4 bytes can be split by a 32-byte boundary.
20811 return TII->getInstSizeInBytes(MI) > 4;
20812 }
20813 return false;
20814}
20815
20816[[maybe_unused]]
20817static bool isCopyFromRegOfInlineAsm(const SDNode *N) {
20818 assert(N->getOpcode() == ISD::CopyFromReg);
20819 do {
20820 // Follow the chain until we find an INLINEASM node.
20821 N = N->getOperand(Num: 0).getNode();
20822 if (N->getOpcode() == ISD::INLINEASM || N->getOpcode() == ISD::INLINEASM_BR)
20823 return true;
20824 } while (N->getOpcode() == ISD::CopyFromReg);
20825 return false;
20826}
20827
20828bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
20829 FunctionLoweringInfo *FLI,
20830 UniformityInfo *UA) const {
20831 switch (N->getOpcode()) {
20832 case ISD::CopyFromReg: {
20833 const RegisterSDNode *R = cast<RegisterSDNode>(Val: N->getOperand(Num: 1));
20834 const MachineRegisterInfo &MRI = FLI->MF->getRegInfo();
20835 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20836 Register Reg = R->getReg();
20837
20838 // FIXME: Why does this need to consider isLiveIn?
20839 if (Reg.isPhysical() || MRI.isLiveIn(Reg))
20840 return !TRI->isSGPRReg(MRI, Reg);
20841
20842 if (const Value *V = FLI->getValueFromVirtualReg(Vreg: R->getReg()))
20843 return UA->isDivergentAtDef(V);
20844
20845 assert(Reg == FLI->DemoteRegister || isCopyFromRegOfInlineAsm(N));
20846 return !TRI->isSGPRReg(MRI, Reg);
20847 }
20848 case ISD::LOAD: {
20849 const LoadSDNode *L = cast<LoadSDNode>(Val: N);
20850 unsigned AS = L->getAddressSpace();
20851 // A flat load may access private memory.
20852 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
20853 }
20854 case ISD::CALLSEQ_END:
20855 return true;
20856 case ISD::INTRINSIC_WO_CHAIN:
20857 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 0));
20858 case ISD::INTRINSIC_W_CHAIN:
20859 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 1));
20860 case AMDGPUISD::ATOMIC_CMP_SWAP:
20861 case AMDGPUISD::BUFFER_ATOMIC_SWAP:
20862 case AMDGPUISD::BUFFER_ATOMIC_ADD:
20863 case AMDGPUISD::BUFFER_ATOMIC_SUB:
20864 case AMDGPUISD::BUFFER_ATOMIC_SMIN:
20865 case AMDGPUISD::BUFFER_ATOMIC_UMIN:
20866 case AMDGPUISD::BUFFER_ATOMIC_SMAX:
20867 case AMDGPUISD::BUFFER_ATOMIC_UMAX:
20868 case AMDGPUISD::BUFFER_ATOMIC_AND:
20869 case AMDGPUISD::BUFFER_ATOMIC_OR:
20870 case AMDGPUISD::BUFFER_ATOMIC_XOR:
20871 case AMDGPUISD::BUFFER_ATOMIC_INC:
20872 case AMDGPUISD::BUFFER_ATOMIC_DEC:
20873 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP:
20874 case AMDGPUISD::BUFFER_ATOMIC_FADD:
20875 case AMDGPUISD::BUFFER_ATOMIC_FMIN:
20876 case AMDGPUISD::BUFFER_ATOMIC_FMAX:
20877 // Target-specific read-modify-write atomics are sources of divergence.
20878 return true;
20879 default:
20880 if (auto *A = dyn_cast<AtomicSDNode>(Val: N)) {
20881 // Generic read-modify-write atomics are sources of divergence.
20882 return A->readMem() && A->writeMem();
20883 }
20884 return false;
20885 }
20886}
20887
20888bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG,
20889 EVT VT) const {
20890 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
20891 case MVT::f32:
20892 return !denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
20893 case MVT::f64:
20894 case MVT::f16:
20895 return !denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction());
20896 default:
20897 return false;
20898 }
20899}
20900
20901bool SITargetLowering::denormalsEnabledForType(
20902 LLT Ty, const MachineFunction &MF) const {
20903 switch (Ty.getScalarSizeInBits()) {
20904 case 32:
20905 return !denormalModeIsFlushAllF32(MF);
20906 case 64:
20907 case 16:
20908 return !denormalModeIsFlushAllF64F16(MF);
20909 default:
20910 return false;
20911 }
20912}
20913
20914bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
20915 const APInt &DemandedElts,
20916 const SelectionDAG &DAG,
20917 bool SNaN,
20918 unsigned Depth) const {
20919 if (Op.getOpcode() == AMDGPUISD::CLAMP) {
20920 const MachineFunction &MF = DAG.getMachineFunction();
20921 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20922
20923 if (Info->getMode().DX10Clamp)
20924 return true; // Clamped to 0.
20925 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
20926 }
20927
20928 return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
20929 DAG, SNaN, Depth);
20930}
20931
20932namespace {
20933
20934/// Why a floating-point atomic instruction which may flush denormals is
20935/// acceptable.
20936enum class AtomicFlushDenormalReason {
20937 Native,
20938 IEEE,
20939 IgnoreDenormalMode,
20940 FunctionFlushesDenormals
20941};
20942
20943/// Why a native floating-point atomic instruction is acceptable for a global
20944/// memory address.
20945enum class GlobalFPAtomicLegality {
20946 Illegal,
20947 AgentScopeFineGrainedRemoteMemory,
20948 EmulatedSystemScope,
20949 NoRemoteMemory,
20950 NoFineGrainedMemory
20951};
20952
20953} // end anonymous namespace
20954
20955// On older subtargets, global FP atomic instructions have a hardcoded FP mode
20956// and do not support FP32 denormals, and only support v2f16/f64 denormals.
20957static AtomicFlushDenormalReason
20958getAtomicFlushDenormalReason(const AtomicRMWInst *RMW) {
20959 if (RMW->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode))
20960 return AtomicFlushDenormalReason::IgnoreDenormalMode;
20961
20962 const fltSemantics &Flt = RMW->getType()->getScalarType()->getFltSemantics();
20963 auto DenormMode = RMW->getFunction()->getDenormalMode(FPType: Flt);
20964 return DenormMode == DenormalMode::getPreserveSign()
20965 ? AtomicFlushDenormalReason::FunctionFlushesDenormals
20966 : AtomicFlushDenormalReason::IEEE;
20967}
20968
20969static OptimizationRemark
20970emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW,
20971 GlobalFPAtomicLegality MemLegality,
20972 AtomicFlushDenormalReason DenormReason) {
20973 LLVMContext &Ctx = RMW->getContext();
20974 StringRef MemScope = Ctx.getSyncScopeName(Id: RMW->getSyncScopeID()).value_or(u: "");
20975 if (MemScope.empty())
20976 MemScope = "system";
20977
20978 OptimizationRemark R(DEBUG_TYPE, "Passed", RMW);
20979 R << "hardware instruction generated for atomic "
20980 << ore::NV("Operation", RMW->getOperationName(Op: RMW->getOperation()))
20981 << " at " << ore::NV("SyncScope", MemScope) << " scope since ";
20982
20983 switch (MemLegality) {
20984 case GlobalFPAtomicLegality::AgentScopeFineGrainedRemoteMemory:
20985 R << "fine-grained remote memory atomics work below system scope";
20986 break;
20987 case GlobalFPAtomicLegality::EmulatedSystemScope:
20988 R << "system scope atomics are emulated in hardware";
20989 break;
20990 case GlobalFPAtomicLegality::NoRemoteMemory:
20991 R << "memory is not remote (!amdgpu.no.remote.memory)";
20992 break;
20993 case GlobalFPAtomicLegality::NoFineGrainedMemory:
20994 R << "memory is not fine-grained (!amdgpu.no.fine.grained.memory)";
20995 break;
20996 case GlobalFPAtomicLegality::Illegal:
20997 llvm_unreachable("remark for illegal atomic");
20998 }
20999
21000 switch (DenormReason) {
21001 case AtomicFlushDenormalReason::Native:
21002 break;
21003 case AtomicFlushDenormalReason::IgnoreDenormalMode:
21004 R << ", and denormals may be flushed (!atomic.ignore.denormal.mode)";
21005 break;
21006 case AtomicFlushDenormalReason::FunctionFlushesDenormals:
21007 R << ", and the floating-point environment flushes denormals";
21008 break;
21009 case AtomicFlushDenormalReason::IEEE:
21010 llvm_unreachable("remark for illegal atomic");
21011 }
21012
21013 return R;
21014}
21015
21016static bool isV2F16OrV2BF16(Type *Ty) {
21017 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
21018 Type *EltTy = VT->getElementType();
21019 return VT->getNumElements() == 2 &&
21020 (EltTy->isHalfTy() || EltTy->isBFloatTy());
21021 }
21022
21023 return false;
21024}
21025
21026static bool isV2F16(Type *Ty) {
21027 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
21028 return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
21029}
21030
21031static bool isV2BF16(Type *Ty) {
21032 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
21033 return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
21034}
21035
21036/// \return true if atomicrmw integer ops work for the type.
21037static bool isAtomicRMWLegalIntTy(Type *Ty) {
21038 if (auto *IT = dyn_cast<IntegerType>(Val: Ty)) {
21039 unsigned BW = IT->getBitWidth();
21040 return BW == 32 || BW == 64;
21041 }
21042
21043 return false;
21044}
21045
21046/// \return true if this atomicrmw xchg type can be selected.
21047static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
21048 Type *Ty = RMW->getType();
21049 if (isAtomicRMWLegalIntTy(Ty))
21050 return true;
21051
21052 if (PointerType *PT = dyn_cast<PointerType>(Val: Ty)) {
21053 const DataLayout &DL = RMW->getFunction()->getDataLayout();
21054 unsigned BW = DL.getPointerSizeInBits(AS: PT->getAddressSpace());
21055 return BW == 32 || BW == 64;
21056 }
21057
21058 if (Ty->isFloatTy() || Ty->isDoubleTy())
21059 return true;
21060
21061 if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
21062 return VT->getNumElements() == 2 &&
21063 VT->getElementType()->getPrimitiveSizeInBits() == 16;
21064 }
21065
21066 return false;
21067}
21068
21069/// \returns whether it's valid to emit a native instruction for \p RMW, and
21070/// why, based on the properties of the target memory.
21071static GlobalFPAtomicLegality
21072getGlobalMemoryFPAtomicLegality(const GCNSubtarget &Subtarget,
21073 const AtomicRMWInst *RMW, bool HasSystemScope) {
21074 // The remote/fine-grained access logic is different from the integer
21075 // atomics. Without AgentScopeFineGrainedRemoteMemoryAtomics support,
21076 // fine-grained access does not work, even for a device local allocation.
21077 //
21078 // With AgentScopeFineGrainedRemoteMemoryAtomics, system scoped device local
21079 // allocations work.
21080 if (HasSystemScope) {
21081 if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics() &&
21082 RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
21083 return GlobalFPAtomicLegality::NoRemoteMemory;
21084 if (Subtarget.hasEmulatedSystemScopeAtomics())
21085 return GlobalFPAtomicLegality::EmulatedSystemScope;
21086 } else if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics())
21087 return GlobalFPAtomicLegality::AgentScopeFineGrainedRemoteMemory;
21088
21089 return RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory")
21090 ? GlobalFPAtomicLegality::NoFineGrainedMemory
21091 : GlobalFPAtomicLegality::Illegal;
21092}
21093
21094/// \return Action to perform on AtomicRMWInsts for integer operations.
21095static TargetLowering::AtomicExpansionKind
21096atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
21097 return isAtomicRMWLegalIntTy(Ty: RMW->getType())
21098 ? TargetLowering::AtomicExpansionKind::None
21099 : TargetLowering::AtomicExpansionKind::CmpXChg;
21100}
21101
21102/// Return if a flat address space atomicrmw can access private memory.
21103static bool flatInstrMayAccessPrivate(const Instruction *I) {
21104 const MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_noalias_addrspace);
21105 return !MD ||
21106 !AMDGPU::hasValueInRangeLikeMetadata(MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
21107}
21108
21109static TargetLowering::AtomicExpansionKind
21110getPrivateAtomicExpansionKind(const GCNSubtarget &STI) {
21111 // For GAS, lower to flat atomic.
21112 return STI.hasGloballyAddressableScratch()
21113 ? TargetLowering::AtomicExpansionKind::CustomExpand
21114 : TargetLowering::AtomicExpansionKind::NotAtomic;
21115}
21116
21117TargetLowering::AtomicExpansionKind
21118SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
21119 unsigned AS = RMW->getPointerAddressSpace();
21120 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
21121 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
21122
21123 // 64-bit flat atomics that dynamically reside in private memory will silently
21124 // be dropped.
21125 //
21126 // Note that we will emit a new copy of the original atomic in the expansion,
21127 // which will be incrementally relegalized.
21128 const DataLayout &DL = RMW->getFunction()->getDataLayout();
21129 if (AS == AMDGPUAS::FLAT_ADDRESS &&
21130 DL.getTypeSizeInBits(Ty: RMW->getType()) == 64 &&
21131 flatInstrMayAccessPrivate(I: RMW))
21132 return AtomicExpansionKind::CustomExpand;
21133
21134 GlobalFPAtomicLegality MemLegality = GlobalFPAtomicLegality::Illegal;
21135 AtomicFlushDenormalReason DenormReason = AtomicFlushDenormalReason::Native;
21136 auto ReportHWInst = [&](TargetLowering::AtomicExpansionKind Kind) {
21137 OptimizationRemarkEmitter ORE(RMW->getFunction());
21138 ORE.emit(RemarkBuilder: [&]() {
21139 return emitAtomicRMWLegalRemark(RMW, MemLegality, DenormReason);
21140 });
21141 return Kind;
21142 };
21143
21144 auto SSID = RMW->getSyncScopeID();
21145 bool HasSystemScope =
21146 SSID == SyncScope::System ||
21147 SSID == RMW->getContext().getOrInsertSyncScopeID(SSN: *getAtomicScopeIRString(
21148 T: getTargetMachine().getTargetTriple(), S: AtomicScope::System,
21149 /*OneAddressSpace=*/IsSingleAddressSpace: true));
21150
21151 auto Op = RMW->getOperation();
21152 switch (Op) {
21153 case AtomicRMWInst::Xchg:
21154 // PCIe supports add and xchg for system atomics.
21155 return isAtomicRMWLegalXChgTy(RMW)
21156 ? TargetLowering::AtomicExpansionKind::None
21157 : TargetLowering::AtomicExpansionKind::CmpXChg;
21158 case AtomicRMWInst::Add:
21159 // PCIe supports add and xchg for system atomics.
21160 return atomicSupportedIfLegalIntType(RMW);
21161 case AtomicRMWInst::Sub:
21162 case AtomicRMWInst::And:
21163 case AtomicRMWInst::Or:
21164 case AtomicRMWInst::Xor:
21165 case AtomicRMWInst::Max:
21166 case AtomicRMWInst::Min:
21167 case AtomicRMWInst::UMax:
21168 case AtomicRMWInst::UMin:
21169 case AtomicRMWInst::UIncWrap:
21170 case AtomicRMWInst::UDecWrap:
21171 case AtomicRMWInst::USubCond:
21172 case AtomicRMWInst::USubSat: {
21173 if (Op == AtomicRMWInst::USubCond && !Subtarget->hasCondSubInsts())
21174 return AtomicExpansionKind::CmpXChg;
21175 if (Op == AtomicRMWInst::USubSat) {
21176 // The global and buffer forms predate the LDS and flat ones.
21177 if (!Subtarget->hasSubClampInsts() ||
21178 (AS == AMDGPUAS::LOCAL_ADDRESS &&
21179 !Subtarget->hasAtomicDsCondSubClampInsts()) ||
21180 (AS == AMDGPUAS::FLAT_ADDRESS &&
21181 !Subtarget->hasAtomicCondSubClampFlatInsts()))
21182 return AtomicExpansionKind::CmpXChg;
21183 }
21184 if (Op == AtomicRMWInst::USubCond || Op == AtomicRMWInst::USubSat) {
21185 auto *IT = dyn_cast<IntegerType>(Val: RMW->getType());
21186 if (!IT || IT->getBitWidth() != 32)
21187 return AtomicExpansionKind::CmpXChg;
21188 }
21189
21190 if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
21191 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21192 if (Subtarget->hasEmulatedSystemScopeAtomics())
21193 return atomicSupportedIfLegalIntType(RMW);
21194
21195 // On most subtargets, for atomicrmw operations other than add/xchg,
21196 // whether or not the instructions will behave correctly depends on where
21197 // the address physically resides and what interconnect is used in the
21198 // system configuration. On some some targets the instruction will nop,
21199 // and in others synchronization will only occur at degraded device scope.
21200 //
21201 // If the allocation is known local to the device, the instructions should
21202 // work correctly.
21203 if (RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
21204 return atomicSupportedIfLegalIntType(RMW);
21205
21206 // If fine-grained remote memory works at device scope, we don't need to
21207 // do anything.
21208 if (!HasSystemScope &&
21209 Subtarget->hasAgentScopeFineGrainedRemoteMemoryAtomics())
21210 return atomicSupportedIfLegalIntType(RMW);
21211
21212 // If we are targeting a remote allocated address, it depends what kind of
21213 // allocation the address belongs to.
21214 //
21215 // If the allocation is fine-grained (in host memory, or in PCIe peer
21216 // device memory), the operation will fail depending on the target.
21217 //
21218 // Note fine-grained host memory access does work on APUs or if XGMI is
21219 // used, but we do not know if we are targeting an APU or the system
21220 // configuration from the ISA version/target-cpu.
21221 if (RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
21222 return atomicSupportedIfLegalIntType(RMW);
21223
21224 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
21225 Op == AtomicRMWInst::Xor) {
21226 // Atomic sub/or/xor do not work over PCI express, but atomic add
21227 // does. InstCombine transforms these with 0 to or, so undo that.
21228 // Sub-word types are not selectable and take the cmpxchg expansion.
21229 if (const Constant *ConstVal = dyn_cast<Constant>(Val: RMW->getValOperand());
21230 ConstVal && ConstVal->isNullValue() &&
21231 isAtomicRMWLegalIntTy(Ty: RMW->getType()))
21232 return AtomicExpansionKind::CustomExpand;
21233 }
21234
21235 // If the allocation could be in remote, fine-grained memory, the rmw
21236 // instructions may fail. cmpxchg should work, so emit that. On some
21237 // system configurations, PCIe atomics aren't supported so cmpxchg won't
21238 // even work, so you're out of luck anyway.
21239
21240 // In summary:
21241 //
21242 // Cases that may fail:
21243 // - fine-grained pinned host memory
21244 // - fine-grained migratable host memory
21245 // - fine-grained PCIe peer device
21246 //
21247 // Cases that should work, but may be treated overly conservatively.
21248 // - fine-grained host memory on an APU
21249 // - fine-grained XGMI peer device
21250 return AtomicExpansionKind::CmpXChg;
21251 }
21252
21253 return atomicSupportedIfLegalIntType(RMW);
21254 }
21255 case AtomicRMWInst::FAdd: {
21256 Type *Ty = RMW->getType();
21257
21258 // TODO: Handle REGION_ADDRESS
21259 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
21260 // DS F32 FP atomics do respect the denormal mode, but the rounding mode
21261 // is fixed to round-to-nearest-even.
21262 //
21263 // F64 / PK_F16 / PK_BF16 never flush and are also fixed to
21264 // round-to-nearest-even.
21265 //
21266 // We ignore the rounding mode problem, even in strictfp. The C++ standard
21267 // suggests it is OK if the floating-point mode may not match the calling
21268 // thread.
21269 if (Ty->isFloatTy()) {
21270 return Subtarget->hasLDSFPAtomicAddF32() ? AtomicExpansionKind::None
21271 : AtomicExpansionKind::CmpXChg;
21272 }
21273
21274 if (Ty->isDoubleTy()) {
21275 // Ignores denormal mode, but we don't consider flushing mandatory.
21276 return Subtarget->hasLdsAtomicAddF64() ? AtomicExpansionKind::None
21277 : AtomicExpansionKind::CmpXChg;
21278 }
21279
21280 if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
21281 return AtomicExpansionKind::None;
21282
21283 return AtomicExpansionKind::CmpXChg;
21284 }
21285
21286 // LDS atomics respect the denormal mode from the mode register.
21287 //
21288 // Traditionally f32 global/buffer memory atomics would unconditionally
21289 // flush denormals, but newer targets do not flush. f64/f16/bf16 cases never
21290 // flush.
21291 //
21292 // On targets with flat atomic fadd, denormals would flush depending on
21293 // whether the target address resides in LDS or global memory. We consider
21294 // this flat-maybe-flush as will-flush.
21295 if (Ty->isFloatTy() &&
21296 !Subtarget->hasMemoryAtomicFaddF32DenormalSupport()) {
21297 DenormReason = getAtomicFlushDenormalReason(RMW);
21298 if (DenormReason == AtomicFlushDenormalReason::IEEE)
21299 return AtomicExpansionKind::CmpXChg;
21300 }
21301
21302 MemLegality =
21303 getGlobalMemoryFPAtomicLegality(Subtarget: *Subtarget, RMW, HasSystemScope);
21304 if (MemLegality != GlobalFPAtomicLegality::Illegal) {
21305 if (AS == AMDGPUAS::FLAT_ADDRESS) {
21306 // gfx942, gfx12
21307 if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
21308 return ReportHWInst(AtomicExpansionKind::None);
21309 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
21310 // gfx90a, gfx942, gfx12
21311 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
21312 return ReportHWInst(AtomicExpansionKind::None);
21313
21314 // gfx942, gfx12
21315 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
21316 return ReportHWInst(AtomicExpansionKind::None);
21317 } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21318 // gfx90a, gfx942, gfx12
21319 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
21320 return ReportHWInst(AtomicExpansionKind::None);
21321
21322 // While gfx90a/gfx942 supports v2bf16 for global/flat, it does not for
21323 // buffer. gfx12 does have the buffer version.
21324 if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
21325 return ReportHWInst(AtomicExpansionKind::None);
21326 }
21327
21328 // global and flat atomic fadd f64: gfx90a, gfx942.
21329 if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy())
21330 return ReportHWInst(AtomicExpansionKind::None);
21331
21332 if (AS != AMDGPUAS::FLAT_ADDRESS) {
21333 if (Ty->isFloatTy()) {
21334 // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx942,
21335 // gfx11+.
21336 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21337 return ReportHWInst(AtomicExpansionKind::None);
21338 // global/buffer atomic fadd f32 rtn: gfx90a, gfx942, gfx11+.
21339 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21340 return ReportHWInst(AtomicExpansionKind::None);
21341 } else {
21342 // gfx908
21343 if (RMW->use_empty() &&
21344 Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
21345 isV2F16(Ty))
21346 return ReportHWInst(AtomicExpansionKind::None);
21347 }
21348 }
21349
21350 // flat atomic fadd f32: gfx942, gfx11+.
21351 if (AS == AMDGPUAS::FLAT_ADDRESS && Ty->isFloatTy()) {
21352 if (Subtarget->hasFlatAtomicFaddF32Inst())
21353 return ReportHWInst(AtomicExpansionKind::None);
21354
21355 // If it is in flat address space, and the type is float, we will try to
21356 // expand it, if the target supports global and lds atomic fadd. The
21357 // reason we need that is, in the expansion, we emit the check of
21358 // address space. If it is in global address space, we emit the global
21359 // atomic fadd; if it is in shared address space, we emit the LDS atomic
21360 // fadd.
21361 if (Subtarget->hasLDSFPAtomicAddF32()) {
21362 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21363 return AtomicExpansionKind::CustomExpand;
21364 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21365 return AtomicExpansionKind::CustomExpand;
21366 }
21367 }
21368 }
21369
21370 return AtomicExpansionKind::CmpXChg;
21371 }
21372 case AtomicRMWInst::FMin:
21373 case AtomicRMWInst::FMax: {
21374 Type *Ty = RMW->getType();
21375
21376 // LDS float and double fmin/fmax were always supported.
21377 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
21378 return Ty->isFloatTy() || Ty->isDoubleTy() ? AtomicExpansionKind::None
21379 : AtomicExpansionKind::CmpXChg;
21380 }
21381
21382 MemLegality =
21383 getGlobalMemoryFPAtomicLegality(Subtarget: *Subtarget, RMW, HasSystemScope);
21384 if (MemLegality != GlobalFPAtomicLegality::Illegal) {
21385 // For flat and global cases:
21386 // float, double in gfx7. Manual claims denormal support.
21387 // Removed in gfx8.
21388 // float, double restored in gfx10.
21389 // double removed again in gfx11, so only f32 for gfx11/gfx12.
21390 //
21391 // For gfx9, gfx90a and gfx942 support f64 for global (same as fadd), but
21392 // no f32.
21393 if (AS == AMDGPUAS::FLAT_ADDRESS) {
21394 if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy())
21395 return ReportHWInst(AtomicExpansionKind::None);
21396 if (Subtarget->hasAtomicFMinFMaxF64FlatInsts() && Ty->isDoubleTy())
21397 return ReportHWInst(AtomicExpansionKind::None);
21398 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS) ||
21399 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21400 if (Subtarget->hasAtomicFMinFMaxF32GlobalInsts() && Ty->isFloatTy())
21401 return ReportHWInst(AtomicExpansionKind::None);
21402 if (Subtarget->hasAtomicFMinFMaxF64GlobalInsts() && Ty->isDoubleTy())
21403 return ReportHWInst(AtomicExpansionKind::None);
21404 }
21405 }
21406
21407 return AtomicExpansionKind::CmpXChg;
21408 }
21409 case AtomicRMWInst::Nand:
21410 case AtomicRMWInst::FSub:
21411 default:
21412 return AtomicExpansionKind::CmpXChg;
21413 }
21414
21415 llvm_unreachable("covered atomicrmw op switch");
21416}
21417
21418TargetLowering::AtomicExpansionKind
21419SITargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
21420 return LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
21421 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
21422 : AtomicExpansionKind::None;
21423}
21424
21425TargetLowering::AtomicExpansionKind
21426SITargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
21427 return SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
21428 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
21429 : AtomicExpansionKind::None;
21430}
21431
21432TargetLowering::AtomicExpansionKind
21433SITargetLowering::shouldExpandAtomicCmpXchgInIR(
21434 const AtomicCmpXchgInst *CmpX) const {
21435 unsigned AddrSpace = CmpX->getPointerAddressSpace();
21436 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
21437 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
21438
21439 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS || !flatInstrMayAccessPrivate(I: CmpX))
21440 return AtomicExpansionKind::None;
21441
21442 const DataLayout &DL = CmpX->getDataLayout();
21443
21444 Type *ValTy = CmpX->getNewValOperand()->getType();
21445
21446 // If a 64-bit flat atomic may alias private, we need to avoid using the
21447 // atomic in the private case.
21448 return DL.getTypeSizeInBits(Ty: ValTy) == 64 ? AtomicExpansionKind::CustomExpand
21449 : AtomicExpansionKind::None;
21450}
21451
21452const TargetRegisterClass *
21453SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
21454 const TargetRegisterClass *RC = TargetLoweringBase::getRegClassFor(VT, isDivergent: false);
21455 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
21456 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
21457 return Subtarget->isWave64() ? &AMDGPU::SReg_64RegClass
21458 : &AMDGPU::SReg_32RegClass;
21459 if (!TRI->isSGPRClass(RC) && !isDivergent)
21460 return TRI->getEquivalentSGPRClass(VRC: RC);
21461 if (TRI->isSGPRClass(RC) && isDivergent) {
21462 if (Subtarget->hasGFX90AInsts())
21463 return TRI->getEquivalentAVClass(SRC: RC);
21464 return TRI->getEquivalentVGPRClass(SRC: RC);
21465 }
21466
21467 return RC;
21468}
21469
21470// FIXME: This is a workaround for DivergenceAnalysis not understanding always
21471// uniform values (as produced by the mask results of control flow intrinsics)
21472// used outside of divergent blocks. The phi users need to also be treated as
21473// always uniform.
21474//
21475// FIXME: DA is no longer in-use. Does this still apply to UniformityAnalysis?
21476static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited,
21477 unsigned WaveSize) {
21478 // FIXME: We assume we never cast the mask results of a control flow
21479 // intrinsic.
21480 // Early exit if the type won't be consistent as a compile time hack.
21481 IntegerType *IT = dyn_cast<IntegerType>(Val: V->getType());
21482 if (!IT || IT->getBitWidth() != WaveSize)
21483 return false;
21484
21485 if (!isa<Instruction>(Val: V))
21486 return false;
21487 if (!Visited.insert(Ptr: V).second)
21488 return false;
21489 bool Result = false;
21490 for (const auto *U : V->users()) {
21491 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: U)) {
21492 if (V == U->getOperand(i: 1)) {
21493 switch (Intrinsic->getIntrinsicID()) {
21494 default:
21495 Result = false;
21496 break;
21497 case Intrinsic::amdgcn_if_break:
21498 case Intrinsic::amdgcn_if:
21499 case Intrinsic::amdgcn_else:
21500 Result = true;
21501 break;
21502 }
21503 }
21504 if (V == U->getOperand(i: 0)) {
21505 switch (Intrinsic->getIntrinsicID()) {
21506 default:
21507 Result = false;
21508 break;
21509 case Intrinsic::amdgcn_end_cf:
21510 case Intrinsic::amdgcn_loop:
21511 Result = true;
21512 break;
21513 }
21514 }
21515 } else {
21516 Result = hasCFUser(V: U, Visited, WaveSize);
21517 }
21518 if (Result)
21519 break;
21520 }
21521 return Result;
21522}
21523
21524bool SITargetLowering::requiresUniformRegister(MachineFunction &MF,
21525 const Value *V) const {
21526 if (const CallInst *CI = dyn_cast<CallInst>(Val: V)) {
21527 if (CI->isInlineAsm()) {
21528 // FIXME: This cannot give a correct answer. This should only trigger in
21529 // the case where inline asm returns mixed SGPR and VGPR results, used
21530 // outside the defining block. We don't have a specific result to
21531 // consider, so this assumes if any value is SGPR, the overall register
21532 // also needs to be SGPR.
21533 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
21534 TargetLowering::AsmOperandInfoVector TargetConstraints = ParseConstraints(
21535 DL: MF.getDataLayout(), TRI: Subtarget->getRegisterInfo(), Call: *CI);
21536 for (auto &TC : TargetConstraints) {
21537 if (TC.Type == InlineAsm::isOutput) {
21538 ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
21539 const TargetRegisterClass *RC =
21540 getRegForInlineAsmConstraint(TRI_: SIRI, Constraint: TC.ConstraintCode,
21541 VT: TC.ConstraintVT)
21542 .second;
21543 if (RC && SIRI->isSGPRClass(RC))
21544 return true;
21545 }
21546 }
21547 }
21548 }
21549 SmallPtrSet<const Value *, 16> Visited;
21550 return hasCFUser(V, Visited, WaveSize: Subtarget->getWavefrontSize());
21551}
21552
21553bool SITargetLowering::hasMemSDNodeUser(SDNode *N) const {
21554 for (SDUse &Use : N->uses()) {
21555 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: Use.getUser())) {
21556 if (getBasePtrIndex(N: M) == Use.getOperandNo())
21557 return true;
21558 }
21559 }
21560 return false;
21561}
21562
21563bool SITargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
21564 SDValue N1) const {
21565 if (!N0.hasOneUse())
21566 return false;
21567 // Take care of the opportunity to keep N0 uniform
21568 if (N0->isDivergent() || !N1->isDivergent())
21569 return true;
21570 // Check if we have a good chance to form the memory access pattern with the
21571 // base and offset
21572 return (DAG.isBaseWithConstantOffset(Op: N0) &&
21573 hasMemSDNodeUser(N: *N0->user_begin()));
21574}
21575
21576bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
21577 Register N0, Register N1) const {
21578 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
21579}
21580
21581MachineMemOperand::Flags
21582SITargetLowering::getTargetMMOFlags(const Instruction &I) const {
21583 // Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
21584 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
21585 if (I.getMetadata(Kind: "amdgpu.noclobber"))
21586 Flags |= MONoClobber;
21587 if (I.getMetadata(Kind: "amdgpu.last.use"))
21588 Flags |= MOLastUse;
21589 return Flags;
21590}
21591
21592void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
21593 Instruction *AI) const {
21594 // Given: atomicrmw fadd ptr %addr, float %val ordering
21595 //
21596 // With this expansion we produce the following code:
21597 // [...]
21598 // %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %addr)
21599 // br i1 %is.shared, label %atomicrmw.shared, label %atomicrmw.check.private
21600 //
21601 // atomicrmw.shared:
21602 // %cast.shared = addrspacecast ptr %addr to ptr addrspace(3)
21603 // %loaded.shared = atomicrmw fadd ptr addrspace(3) %cast.shared,
21604 // float %val ordering
21605 // br label %atomicrmw.phi
21606 //
21607 // atomicrmw.check.private:
21608 // %is.private = call i1 @llvm.amdgcn.is.private(ptr %int8ptr)
21609 // br i1 %is.private, label %atomicrmw.private, label %atomicrmw.global
21610 //
21611 // atomicrmw.private:
21612 // %cast.private = addrspacecast ptr %addr to ptr addrspace(5)
21613 // %loaded.private = load float, ptr addrspace(5) %cast.private
21614 // %val.new = fadd float %loaded.private, %val
21615 // store float %val.new, ptr addrspace(5) %cast.private
21616 // br label %atomicrmw.phi
21617 //
21618 // atomicrmw.global:
21619 // %cast.global = addrspacecast ptr %addr to ptr addrspace(1)
21620 // %loaded.global = atomicrmw fadd ptr addrspace(1) %cast.global,
21621 // float %val ordering
21622 // br label %atomicrmw.phi
21623 //
21624 // atomicrmw.phi:
21625 // %loaded.phi = phi float [ %loaded.shared, %atomicrmw.shared ],
21626 // [ %loaded.private, %atomicrmw.private ],
21627 // [ %loaded.global, %atomicrmw.global ]
21628 // br label %atomicrmw.end
21629 //
21630 // atomicrmw.end:
21631 // [...]
21632 //
21633 //
21634 // For 64-bit atomics which may reside in private memory, we perform a simpler
21635 // version that only inserts the private check, and uses the flat operation.
21636
21637 IRBuilder<> Builder(AI);
21638 LLVMContext &Ctx = Builder.getContext();
21639
21640 auto *RMW = dyn_cast<AtomicRMWInst>(Val: AI);
21641 const unsigned PtrOpIdx = RMW ? AtomicRMWInst::getPointerOperandIndex()
21642 : AtomicCmpXchgInst::getPointerOperandIndex();
21643 Value *Addr = AI->getOperand(i: PtrOpIdx);
21644
21645 /// TODO: Only need to check private, then emit flat-known-not private (no
21646 /// need for shared block, or cast to global).
21647 AtomicCmpXchgInst *CX = dyn_cast<AtomicCmpXchgInst>(Val: AI);
21648
21649 Align Alignment;
21650 if (RMW)
21651 Alignment = RMW->getAlign();
21652 else if (CX)
21653 Alignment = CX->getAlign();
21654 else
21655 llvm_unreachable("unhandled atomic operation");
21656
21657 // FullFlatEmulation is true if we need to issue the private, shared, and
21658 // global cases.
21659 //
21660 // If this is false, we are only dealing with the flat-targeting-private case,
21661 // where we only insert a check for private and still use the flat instruction
21662 // for global and shared.
21663
21664 bool FullFlatEmulation =
21665 RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
21666 ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
21667 (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
21668 RMW->getType()->isDoubleTy()));
21669
21670 // If the return value isn't used, do not introduce a false use in the phi.
21671 bool ReturnValueIsUsed = !AI->use_empty();
21672
21673 BasicBlock *BB = Builder.GetInsertBlock();
21674 Function *F = BB->getParent();
21675 BasicBlock *ExitBB =
21676 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
21677 BasicBlock *SharedBB = nullptr;
21678
21679 BasicBlock *CheckPrivateBB = BB;
21680 if (FullFlatEmulation) {
21681 SharedBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.shared", Parent: F, InsertBefore: ExitBB);
21682 CheckPrivateBB =
21683 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.check.private", Parent: F, InsertBefore: ExitBB);
21684 }
21685
21686 BasicBlock *PrivateBB =
21687 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.private", Parent: F, InsertBefore: ExitBB);
21688 BasicBlock *GlobalBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.global", Parent: F, InsertBefore: ExitBB);
21689 BasicBlock *PhiBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.phi", Parent: F, InsertBefore: ExitBB);
21690
21691 std::prev(x: BB->end())->eraseFromParent();
21692 Builder.SetInsertPoint(BB);
21693
21694 Value *LoadedShared = nullptr;
21695 if (FullFlatEmulation) {
21696 Value *IsShared = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_shared,
21697 Args: {Addr}, FMFSource: nullptr, Name: "is.shared");
21698 Builder.CreateCondBr(Cond: IsShared, True: SharedBB, False: CheckPrivateBB);
21699 Builder.SetInsertPoint(SharedBB);
21700 Value *CastToLocal = Builder.CreateAddrSpaceCast(
21701 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS));
21702
21703 Instruction *Clone = AI->clone();
21704 Clone->insertInto(ParentBB: SharedBB, It: SharedBB->end());
21705 Clone->getOperandUse(i: PtrOpIdx).set(CastToLocal);
21706 LoadedShared = Clone;
21707
21708 Builder.CreateBr(Dest: PhiBB);
21709 Builder.SetInsertPoint(CheckPrivateBB);
21710 }
21711
21712 Value *IsPrivate = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_private,
21713 Args: {Addr}, FMFSource: nullptr, Name: "is.private");
21714 Builder.CreateCondBr(Cond: IsPrivate, True: PrivateBB, False: GlobalBB);
21715
21716 Builder.SetInsertPoint(PrivateBB);
21717
21718 Value *CastToPrivate = Builder.CreateAddrSpaceCast(
21719 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::PRIVATE_ADDRESS));
21720
21721 Value *LoadedPrivate;
21722 if (RMW) {
21723 LoadedPrivate = Builder.CreateAlignedLoad(
21724 Ty: RMW->getType(), Ptr: CastToPrivate, Align: RMW->getAlign(), isVolatile: RMW->isVolatile(),
21725 Name: "loaded.private");
21726
21727 Value *NewVal = buildAtomicRMWValue(Op: RMW->getOperation(), Builder,
21728 Loaded: LoadedPrivate, Val: RMW->getValOperand());
21729
21730 Builder.CreateAlignedStore(Val: NewVal, Ptr: CastToPrivate, Align: RMW->getAlign(),
21731 isVolatile: RMW->isVolatile());
21732 } else {
21733 auto [ResultLoad, Equal] = buildCmpXchgValue(
21734 Builder, Ptr: CastToPrivate, Cmp: CX->getCompareOperand(), Val: CX->getNewValOperand(),
21735 Alignment: CX->getAlign(), IsVolatile: CX->isVolatile());
21736
21737 Value *Insert = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CX->getType()),
21738 Val: ResultLoad, Idxs: 0);
21739 LoadedPrivate = Builder.CreateInsertValue(Agg: Insert, Val: Equal, Idxs: 1);
21740 }
21741
21742 Builder.CreateBr(Dest: PhiBB);
21743
21744 Builder.SetInsertPoint(GlobalBB);
21745
21746 // Continue using a flat instruction if we only emitted the check for private.
21747 Instruction *LoadedGlobal = AI;
21748 if (FullFlatEmulation) {
21749 Value *CastToGlobal = Builder.CreateAddrSpaceCast(
21750 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
21751 AI->getOperandUse(i: PtrOpIdx).set(CastToGlobal);
21752 }
21753
21754 AI->removeFromParent();
21755 AI->insertInto(ParentBB: GlobalBB, It: GlobalBB->end());
21756
21757 // The new atomicrmw may go through another round of legalization later.
21758 if (!FullFlatEmulation) {
21759 // We inserted the runtime check already, make sure we do not try to
21760 // re-expand this.
21761 // TODO: Should union with any existing metadata.
21762 MDBuilder MDB(F->getContext());
21763 MDNode *RangeNotPrivate =
21764 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
21765 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
21766 LoadedGlobal->setMetadata(KindID: LLVMContext::MD_noalias_addrspace,
21767 Node: RangeNotPrivate);
21768 }
21769
21770 Builder.CreateBr(Dest: PhiBB);
21771
21772 Builder.SetInsertPoint(PhiBB);
21773
21774 if (ReturnValueIsUsed) {
21775 PHINode *Loaded = Builder.CreatePHI(Ty: AI->getType(), NumReservedValues: 3);
21776 AI->replaceAllUsesWith(V: Loaded);
21777 if (FullFlatEmulation)
21778 Loaded->addIncoming(V: LoadedShared, BB: SharedBB);
21779 Loaded->addIncoming(V: LoadedPrivate, BB: PrivateBB);
21780 Loaded->addIncoming(V: LoadedGlobal, BB: GlobalBB);
21781 Loaded->takeName(V: AI);
21782 }
21783
21784 Builder.CreateBr(Dest: ExitBB);
21785}
21786
21787static void convertScratchAtomicToFlatAtomic(Instruction *I,
21788 unsigned PtrOpIdx) {
21789 Value *PtrOp = I->getOperand(i: PtrOpIdx);
21790 assert(PtrOp->getType()->getPointerAddressSpace() ==
21791 AMDGPUAS::PRIVATE_ADDRESS);
21792
21793 Type *FlatPtr = PointerType::get(C&: I->getContext(), AddressSpace: AMDGPUAS::FLAT_ADDRESS);
21794 Value *ASCast = CastInst::CreatePointerCast(S: PtrOp, Ty: FlatPtr, Name: "scratch.ascast",
21795 InsertBefore: I->getIterator());
21796 I->setOperand(i: PtrOpIdx, Val: ASCast);
21797}
21798
21799void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
21800 AtomicRMWInst::BinOp Op = AI->getOperation();
21801
21802 if (AI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21803 return convertScratchAtomicToFlatAtomic(I: AI, PtrOpIdx: AI->getPointerOperandIndex());
21804
21805 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
21806 Op == AtomicRMWInst::Xor) {
21807 if (const auto *ConstVal = dyn_cast<Constant>(Val: AI->getValOperand());
21808 ConstVal && ConstVal->isNullValue() &&
21809 isAtomicRMWLegalIntTy(Ty: AI->getType())) {
21810 // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
21811 AI->setOperation(AtomicRMWInst::Add);
21812
21813 // We may still need the private-alias-flat handling below.
21814
21815 // TODO: Skip this for cases where we cannot access remote memory.
21816 }
21817 }
21818
21819 // The non-flat expansions should only perform the de-canonicalization of
21820 // identity values.
21821 if (AI->getPointerAddressSpace() != AMDGPUAS::FLAT_ADDRESS)
21822 return;
21823
21824 emitExpandAtomicAddrSpacePredicate(AI);
21825}
21826
21827void SITargetLowering::emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const {
21828 if (CI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21829 return convertScratchAtomicToFlatAtomic(I: CI, PtrOpIdx: CI->getPointerOperandIndex());
21830
21831 emitExpandAtomicAddrSpacePredicate(AI: CI);
21832}
21833
21834void SITargetLowering::emitExpandAtomicLoad(LoadInst *LI) const {
21835 if (LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21836 return convertScratchAtomicToFlatAtomic(I: LI, PtrOpIdx: LI->getPointerOperandIndex());
21837
21838 llvm_unreachable(
21839 "Expand Atomic Load only handles SCRATCH -> FLAT conversion");
21840}
21841
21842void SITargetLowering::emitExpandAtomicStore(StoreInst *SI) const {
21843 if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21844 return convertScratchAtomicToFlatAtomic(I: SI, PtrOpIdx: SI->getPointerOperandIndex());
21845
21846 llvm_unreachable(
21847 "Expand Atomic Store only handles SCRATCH -> FLAT conversion");
21848}
21849