1//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for SI
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUIGroupLP.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPULaneMaskUtils.h"
19#include "AMDGPUMachineInstrs.h"
20#include "AMDGPUMemoryUtils.h"
21#include "AMDGPUSelectionDAGInfo.h"
22#include "AMDGPUTargetMachine.h"
23#include "GCNSubtarget.h"
24#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
25#include "SIMachineFunctionInfo.h"
26#include "SIRegisterInfo.h"
27#include "llvm/ADT/APFloat.h"
28#include "llvm/ADT/APInt.h"
29#include "llvm/ADT/FloatingPointMode.h"
30#include "llvm/ADT/Statistic.h"
31#include "llvm/Analysis/OptimizationRemarkEmitter.h"
32#include "llvm/Analysis/UniformityAnalysis.h"
33#include "llvm/CodeGen/Analysis.h"
34#include "llvm/CodeGen/ByteProvider.h"
35#include "llvm/CodeGen/FunctionLoweringInfo.h"
36#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
37#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
38#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
39#include "llvm/CodeGen/GlobalISel/Utils.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineLoopInfo.h"
43#include "llvm/CodeGen/PseudoSourceValueManager.h"
44#include "llvm/CodeGen/SDPatternMatch.h"
45#include "llvm/IR/DiagnosticInfo.h"
46#include "llvm/IR/IRBuilder.h"
47#include "llvm/IR/IntrinsicInst.h"
48#include "llvm/IR/IntrinsicsAMDGPU.h"
49#include "llvm/IR/IntrinsicsR600.h"
50#include "llvm/IR/MDBuilder.h"
51#include "llvm/Support/AMDGPUAddrSpace.h"
52#include "llvm/Support/KnownBits.h"
53#include "llvm/Support/ModRef.h"
54#include "llvm/TargetParser/AtomicScope.h"
55#include "llvm/Transforms/Utils/LowerAtomic.h"
56#include <optional>
57
58using namespace llvm;
59using namespace llvm::SDPatternMatch;
60
61#define DEBUG_TYPE "si-lower"
62
63STATISTIC(NumTailCalls, "Number of tail calls");
64
65static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF) {
66 return MF.getInfo<SIMachineFunctionInfo>()->getMode().getDenormalFPEnv();
67}
68
69static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
70 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
71 return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
72}
73
74static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF) {
75 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
76 return Info->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign();
77}
78
79static unsigned findFirstFreeSGPR(CCState &CCInfo) {
80 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
81 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
82 if (!CCInfo.isAllocated(Reg: AMDGPU::SGPR0 + Reg)) {
83 return AMDGPU::SGPR0 + Reg;
84 }
85 }
86 llvm_unreachable("Cannot allocate sgpr");
87}
88
89SITargetLowering::SITargetLowering(const TargetMachine &TM,
90 const GCNSubtarget &STI)
91 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI) {
92 addRegisterClass(VT: MVT::i1, RC: &AMDGPU::VReg_1RegClass);
93 addRegisterClass(VT: MVT::i64, RC: &AMDGPU::SReg_64RegClass);
94
95 addRegisterClass(VT: MVT::i32, RC: &AMDGPU::SReg_32RegClass);
96
97 const SIRegisterInfo *TRI = STI.getRegisterInfo();
98 const TargetRegisterClass *V32RegClass =
99 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 32);
100 addRegisterClass(VT: MVT::f32, RC: V32RegClass);
101
102 addRegisterClass(VT: MVT::v2i32, RC: &AMDGPU::SReg_64RegClass);
103
104 const TargetRegisterClass *V64RegClass =
105 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 64);
106
107 addRegisterClass(VT: MVT::f64, RC: V64RegClass);
108 addRegisterClass(VT: MVT::v2f32, RC: V64RegClass);
109 addRegisterClass(VT: MVT::Untyped, RC: V64RegClass);
110
111 addRegisterClass(VT: MVT::v3i32, RC: &AMDGPU::SGPR_96RegClass);
112 addRegisterClass(VT: MVT::v3f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 96));
113
114 addRegisterClass(VT: MVT::v2i64, RC: &AMDGPU::SGPR_128RegClass);
115 addRegisterClass(VT: MVT::v2f64, RC: &AMDGPU::SGPR_128RegClass);
116
117 addRegisterClass(VT: MVT::v4i32, RC: &AMDGPU::SGPR_128RegClass);
118 addRegisterClass(VT: MVT::v4f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 128));
119
120 addRegisterClass(VT: MVT::v5i32, RC: &AMDGPU::SGPR_160RegClass);
121 addRegisterClass(VT: MVT::v5f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 160));
122
123 addRegisterClass(VT: MVT::v6i32, RC: &AMDGPU::SGPR_192RegClass);
124 addRegisterClass(VT: MVT::v6f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
125
126 addRegisterClass(VT: MVT::v3i64, RC: &AMDGPU::SGPR_192RegClass);
127 addRegisterClass(VT: MVT::v3f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
128
129 addRegisterClass(VT: MVT::v7i32, RC: &AMDGPU::SGPR_224RegClass);
130 addRegisterClass(VT: MVT::v7f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 224));
131
132 addRegisterClass(VT: MVT::v8i32, RC: &AMDGPU::SGPR_256RegClass);
133 addRegisterClass(VT: MVT::v8f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
134
135 addRegisterClass(VT: MVT::v4i64, RC: &AMDGPU::SGPR_256RegClass);
136 addRegisterClass(VT: MVT::v4f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
137
138 addRegisterClass(VT: MVT::v9i32, RC: &AMDGPU::SGPR_288RegClass);
139 addRegisterClass(VT: MVT::v9f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 288));
140
141 addRegisterClass(VT: MVT::v10i32, RC: &AMDGPU::SGPR_320RegClass);
142 addRegisterClass(VT: MVT::v10f32,
143 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 320));
144
145 addRegisterClass(VT: MVT::v11i32, RC: &AMDGPU::SGPR_352RegClass);
146 addRegisterClass(VT: MVT::v11f32,
147 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 352));
148
149 addRegisterClass(VT: MVT::v12i32, RC: &AMDGPU::SGPR_384RegClass);
150 addRegisterClass(VT: MVT::v12f32,
151 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 384));
152
153 addRegisterClass(VT: MVT::v16i32, RC: &AMDGPU::SGPR_512RegClass);
154 addRegisterClass(VT: MVT::v16f32,
155 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
156
157 addRegisterClass(VT: MVT::v8i64, RC: &AMDGPU::SGPR_512RegClass);
158 addRegisterClass(VT: MVT::v8f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
159
160 addRegisterClass(VT: MVT::v16i64, RC: &AMDGPU::SGPR_1024RegClass);
161 addRegisterClass(VT: MVT::v16f64,
162 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
163
164 if (Subtarget->has16BitInsts()) {
165 if (Subtarget->useRealTrue16Insts()) {
166 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::VGPR_16RegClass);
167 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::VGPR_16RegClass);
168 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::VGPR_16RegClass);
169 } else {
170 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::SReg_32RegClass);
171 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::SReg_32RegClass);
172 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::SReg_32RegClass);
173 }
174
175 // Unless there are also VOP3P operations, not operations are really legal.
176 addRegisterClass(VT: MVT::v2i16, RC: &AMDGPU::SReg_32RegClass);
177 addRegisterClass(VT: MVT::v2f16, RC: &AMDGPU::SReg_32RegClass);
178 addRegisterClass(VT: MVT::v2bf16, RC: &AMDGPU::SReg_32RegClass);
179 addRegisterClass(VT: MVT::v4i16, RC: &AMDGPU::SReg_64RegClass);
180 addRegisterClass(VT: MVT::v4f16, RC: &AMDGPU::SReg_64RegClass);
181 addRegisterClass(VT: MVT::v4bf16, RC: &AMDGPU::SReg_64RegClass);
182 addRegisterClass(VT: MVT::v8i16, RC: &AMDGPU::SGPR_128RegClass);
183 addRegisterClass(VT: MVT::v8f16, RC: &AMDGPU::SGPR_128RegClass);
184 addRegisterClass(VT: MVT::v8bf16, RC: &AMDGPU::SGPR_128RegClass);
185 addRegisterClass(VT: MVT::v16i16, RC: &AMDGPU::SGPR_256RegClass);
186 addRegisterClass(VT: MVT::v16f16, RC: &AMDGPU::SGPR_256RegClass);
187 addRegisterClass(VT: MVT::v16bf16, RC: &AMDGPU::SGPR_256RegClass);
188 addRegisterClass(VT: MVT::v32i16, RC: &AMDGPU::SGPR_512RegClass);
189 addRegisterClass(VT: MVT::v32f16, RC: &AMDGPU::SGPR_512RegClass);
190 addRegisterClass(VT: MVT::v32bf16, RC: &AMDGPU::SGPR_512RegClass);
191 }
192
193 addRegisterClass(VT: MVT::v32i32, RC: &AMDGPU::VReg_1024RegClass);
194 addRegisterClass(VT: MVT::v32f32,
195 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
196
197 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
198
199 setMinFunctionAlignment(Align(4));
200 setPrefFunctionAlignment(Align(STI.getInstCacheLineSize()));
201
202 // The boolean content concept here is too inflexible. Compares only ever
203 // really produce a 1-bit result. Any copy/extend from these will turn into a
204 // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
205 // it's what most targets use.
206 setBooleanContents(ZeroOrOneBooleanContent);
207 setBooleanVectorContents(ZeroOrOneBooleanContent);
208
209 // We need to custom lower vector stores from local memory
210 setOperationAction(Ops: ISD::LOAD,
211 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
212 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
213 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
214 MVT::i1, MVT::v32i32},
215 Action: Custom);
216
217 setOperationAction(Ops: ISD::STORE,
218 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
219 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
220 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
221 MVT::i1, MVT::v32i32},
222 Action: Custom);
223
224 if (isTypeLegal(VT: MVT::bf16)) {
225 for (unsigned Opc :
226 {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
227 ISD::FREM, ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
228 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FSQRT, ISD::FCBRT,
229 ISD::FSIN, ISD::FCOS, ISD::FPOW, ISD::FPOWI,
230 ISD::FLDEXP, ISD::FFREXP, ISD::FLOG, ISD::FLOG2,
231 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10,
232 ISD::FCEIL, ISD::FTRUNC, ISD::FRINT, ISD::FNEARBYINT,
233 ISD::FROUND, ISD::FROUNDEVEN, ISD::FFLOOR, ISD::FCANONICALIZE,
234 ISD::SETCC}) {
235 setOperationAction(Op: Opc, VT: MVT::bf16, Action: Promote);
236 }
237
238 // Only targets with packed bf16 instructions, e.g. gfx13.
239 if (Subtarget->hasBF16PackedInsts()) {
240 // Don't use Expand for fsub - the DAG combiner will undo fadd+fneg back
241 // to fsub, causing a libcall (which doesn't exist for bf16). Instead,
242 // directly expand to widened v2bf16 operations.
243 setOperationAction(Op: ISD::FSUB, VT: MVT::bf16, Action: Custom);
244 setOperationAction(Ops: {ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}, VT: MVT::bf16,
245 Action: Promote);
246 // Promote scalar operations to a v2bf16 operation with an unused high
247 // lane.
248 for (unsigned Opc :
249 {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMAXNUM, ISD::FMINNUM,
250 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FCANONICALIZE})
251 AddPromotedToType(Opc, OrigVT: MVT::bf16, DestVT: MVT::v2bf16);
252 }
253
254 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Expand);
255
256 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Promote);
257 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::bf16, DestVT: MVT::i16);
258
259 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Legal);
260 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Legal);
261 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Legal);
262
263 // We only need to custom lower because we can't specify an action for bf16
264 // sources.
265 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
266 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
267 }
268
269 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
270 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i16, Action: Expand);
271 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Expand);
272 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i16, Action: Expand);
273 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i16, Action: Expand);
274 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i16, Action: Expand);
275 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
276 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Expand);
277 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i8, Action: Expand);
278 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i8, Action: Expand);
279 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i8, Action: Expand);
280 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
281 setTruncStoreAction(ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Expand);
282 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Expand);
283 setTruncStoreAction(ValVT: MVT::v16i16, MemVT: MVT::v16i8, Action: Expand);
284 setTruncStoreAction(ValVT: MVT::v32i16, MemVT: MVT::v32i8, Action: Expand);
285
286 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
287 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
288 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i8, Action: Expand);
289 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i8, Action: Expand);
290 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i16, Action: Expand);
291 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i32, Action: Expand);
292 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i32, Action: Expand);
293
294 setOperationAction(Ops: ISD::GlobalAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
295
296 // Only for an alloca in the VGPR ("as memory") address space.
297 setOperationAction(Op: ISD::FrameIndex, VT: MVT::i32, Action: Custom);
298 setOperationAction(Ops: ISD::BlockAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
299 setOperationAction(Ops: ISD::ExternalSymbol, VTs: {MVT::i32, MVT::i64}, Action: Custom);
300
301 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Promote);
302 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Custom);
303 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Promote);
304 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::f64, DestVT: MVT::i64);
305
306 setOperationAction(Ops: ISD::FSQRT, VTs: {MVT::f32, MVT::f64}, Action: Custom);
307
308 setOperationAction(Ops: ISD::SELECT_CC,
309 VTs: {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Action: Expand);
310
311 setOperationAction(Op: ISD::SETCC, VT: MVT::i1, Action: Promote);
312 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::v2i1, MVT::v4i1}, Action: Expand);
313 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::i1, DestVT: MVT::i32);
314
315 setOperationAction(Ops: ISD::TRUNCATE,
316 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
317 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
318 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
319 Action: Expand);
320 setOperationAction(Ops: ISD::FP_ROUND,
321 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
322 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
323 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
324 Action: Expand);
325
326 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG,
327 VTs: {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
328 MVT::v3i16, MVT::v4i16, MVT::Other},
329 Action: Custom);
330
331 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
332 setOperationAction(Ops: ISD::BR_CC,
333 VTs: {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Action: Expand);
334
335 setOperationAction(Ops: {ISD::ABS, ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Legal);
336 setOperationAction(Ops: {ISD::UADDO, ISD::USUBO}, VT: MVT::i64, Action: Legal);
337
338 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i32, Action: Legal);
339 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i64, Action: Legal);
340
341 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, VT: MVT::i64,
342 Action: Expand);
343
344 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
345
346 // We only support LOAD/STORE and vector manipulation ops for vectors
347 // with > 4 elements.
348 for (MVT VT :
349 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
350 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
351 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
352 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
353 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
354 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
355 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
356 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
357 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
358 switch (Op) {
359 case ISD::LOAD:
360 case ISD::STORE:
361 case ISD::ATOMIC_LOAD:
362 case ISD::ATOMIC_STORE:
363 case ISD::BUILD_VECTOR:
364 case ISD::BITCAST:
365 case ISD::UNDEF:
366 case ISD::POISON:
367 case ISD::EXTRACT_VECTOR_ELT:
368 case ISD::INSERT_VECTOR_ELT:
369 case ISD::SCALAR_TO_VECTOR:
370 case ISD::IS_FPCLASS:
371 break;
372 case ISD::EXTRACT_SUBVECTOR:
373 case ISD::INSERT_SUBVECTOR:
374 case ISD::CONCAT_VECTORS:
375 setOperationAction(Op, VT, Action: Custom);
376 break;
377 default:
378 setOperationAction(Op, VT, Action: Expand);
379 break;
380 }
381 }
382 }
383
384 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f32, Action: Expand);
385
386 // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
387 // is expanded to avoid having two separate loops in case the index is a VGPR.
388
389 // Most operations are naturally 32-bit vector operations. We only support
390 // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
391 for (MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
392 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
393 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
394
395 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
396 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
397
398 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
399 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
400
401 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
402 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
403 }
404
405 for (MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
406 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
407 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
408
409 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
410 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
411
412 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
413 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
414
415 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
416 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
417 }
418
419 for (MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
420 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
421 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
422
423 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
424 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
425
426 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
427 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
428
429 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
430 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
431 }
432
433 for (MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
434 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
435 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
436
437 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
438 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
439
440 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
441 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
442
443 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
444 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
445 }
446
447 for (MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
448 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
449 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
450
451 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
452 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
453
454 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
455 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
456
457 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
458 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
459 }
460
461 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
462 VTs: {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
463 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
464 Action: Custom);
465
466 if (Subtarget->hasPkMovB32()) {
467 // TODO: 16-bit element vectors should be legal with even aligned elements.
468 // TODO: Can be legal with wider source types than the result with
469 // subregister extracts.
470 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2i32, MVT::v2f32}, Action: Legal);
471 }
472
473 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT: MVT::v2i32, Action: Legal);
474 // Prevent SELECT v2i32 from being implemented with the above bitwise ops and
475 // instead lower to cndmask in SITargetLowering::LowerSELECT().
476 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i32, Action: Custom);
477 // Enable MatchRotate to produce ISD::ROTR, which is later transformed to
478 // alignbit.
479 setOperationAction(Op: ISD::ROTR, VT: MVT::v2i32, Action: Custom);
480
481 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
482 Action: Custom);
483
484 // Avoid stack access for these.
485 // TODO: Generalize to wider i8 vectors.
486 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
487 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
488 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
489 Action: Custom);
490
491 // Deal with vec3 vector operations when widened to vec4.
492 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
493 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
494
495 // Deal with vec5/6/7 vector operations when widened to vec8.
496 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
497 VTs: {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
498 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
499 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
500 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
501 Action: Custom);
502
503 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
504 // and output demarshalling
505 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP, VTs: {MVT::i32, MVT::i64}, Action: Custom);
506
507 // We can't return success/failure, only the old value,
508 // let LLVM add the comparison
509 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VTs: {MVT::i32, MVT::i64},
510 Action: Expand);
511
512 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
513
514 setOperationAction(Ops: ISD::BITREVERSE, VTs: {MVT::i32, MVT::i64}, Action: Legal);
515
516 // FIXME: This should be narrowed to i32, but that only happens if i64 is
517 // illegal.
518 // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
519 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i64, MVT::i32}, Action: Legal);
520
521 // On SI this is s_memtime and s_memrealtime on VI.
522 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
523
524 if (Subtarget->hasSMemRealTime() ||
525 Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
526 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
527 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Custom);
528
529 if (Subtarget->has16BitInsts()) {
530 setOperationAction(Ops: {ISD::FPOW, ISD::FPOWI}, VT: MVT::f16, Action: Promote);
531 setOperationAction(Ops: {ISD::FLOG, ISD::FEXP, ISD::FLOG10}, VT: MVT::f16, Action: Custom);
532 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Legal);
533 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Legal);
534 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f16, Action: Legal);
535 } else {
536 setOperationAction(Op: ISD::FSQRT, VT: MVT::f16, Action: Custom);
537 }
538
539 if (Subtarget->hasMadMacF32Insts())
540 setOperationAction(Op: ISD::FMAD, VT: MVT::f32, Action: Legal);
541
542 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
543 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
544 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
545
546 // We only really have 32-bit BFE instructions (and 16-bit on VI).
547 //
548 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
549 // effort to match them now. We want this to be false for i64 cases when the
550 // extraction isn't restricted to the upper or lower half. Ideally we would
551 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
552 // span the midpoint are probably relatively rare, so don't worry about them
553 // for now.
554 setHasExtractBitsInsn(true);
555
556 // Clamp modifier on add/sub
557 if (Subtarget->hasIntClamp())
558 setOperationAction(Ops: {ISD::UADDSAT, ISD::USUBSAT}, VT: MVT::i32, Action: Legal);
559
560 if (Subtarget->hasAddNoCarryInsts())
561 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT}, VTs: {MVT::i16, MVT::i32},
562 Action: Legal);
563
564 // Do not have s_{min|max}_*f64 instruction f64 will only be lowered to
565 // v_{min|max}_*f64
566 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
567 setOperationAction(
568 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
569 VTs: {MVT::f64, MVT::f32}, Action: Legal);
570 } else {
571 setOperationAction(
572 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
573 VTs: {MVT::f64, MVT::f32}, Action: Custom);
574 // These are really only legal for ieee_mode functions. We should be
575 // avoiding them for functions that don't have ieee_mode enabled, so just
576 // say they are legal.
577 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
578 VTs: {MVT::f64, MVT::f32}, Action: Legal);
579 }
580
581 if (Subtarget->haveRoundOpsF64())
582 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, VT: MVT::f64,
583 Action: Legal);
584 else
585 setOperationAction(Ops: {ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
586 VT: MVT::f64, Action: Custom);
587
588 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
589 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VTs: {MVT::f32, MVT::f64},
590 Action: Legal);
591 setOperationAction(Ops: ISD::FFREXP, VTs: {MVT::f32, MVT::f64}, Action: Custom);
592
593 setOperationAction(Ops: {ISD::FSIN, ISD::FCOS, ISD::FDIV}, VT: MVT::f32, Action: Custom);
594 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Custom);
595
596 setOperationAction(Ops: ISD::BF16_TO_FP, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
597 setOperationAction(Ops: ISD::FP_TO_BF16, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
598
599 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i32,
600 Action: Custom);
601 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i16,
602 Action: Custom);
603 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i1,
604 Action: Custom);
605
606 // Custom lower these because we can't specify a rule based on an illegal
607 // source bf16.
608 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f32, Action: Custom);
609 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f64, Action: Custom);
610
611 if (Subtarget->has16BitInsts()) {
612 setOperationAction(Ops: {ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
613 ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
614 VT: MVT::i16, Action: Legal);
615
616 AddPromotedToType(Opc: ISD::SIGN_EXTEND, OrigVT: MVT::i16, DestVT: MVT::i32);
617
618 setOperationAction(Ops: {ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
619 VT: MVT::i16, Action: Expand);
620
621 setOperationAction(Ops: {ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
622 ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
623 ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
624 ISD::CTPOP},
625 VT: MVT::i16, Action: Promote);
626
627 setOperationAction(Op: ISD::LOAD, VT: MVT::i16, Action: Custom);
628
629 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
630
631 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::i16, Action: Promote);
632 AddPromotedToType(Opc: ISD::FP16_TO_FP, OrigVT: MVT::i16, DestVT: MVT::i32);
633 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::i16, Action: Promote);
634 AddPromotedToType(Opc: ISD::FP_TO_FP16, OrigVT: MVT::i16, DestVT: MVT::i32);
635
636 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i16, Action: Custom);
637 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i32, Action: Custom);
638 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i16, Action: Custom);
639 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i1, Action: Custom);
640
641 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i32, Action: Custom);
642
643 // F16 - Constant Actions.
644 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
645 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
646
647 // F16 - Load/Store Actions.
648 setOperationAction(Op: ISD::LOAD, VT: MVT::f16, Action: Promote);
649 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
650 setOperationAction(Op: ISD::STORE, VT: MVT::f16, Action: Promote);
651 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
652
653 // BF16 - Load/Store Actions.
654 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Promote);
655 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
656 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Promote);
657 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
658
659 if (!Subtarget->useRealTrue16Insts() ||
660 !Subtarget->d16PreservesUnusedBits()) {
661 for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
662 setOperationAction(Op: ISD::ATOMIC_LOAD, VT, Action: Promote);
663 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: VT, DestVT: MVT::i32);
664 }
665 }
666
667 if (!Subtarget->useRealTrue16Insts()) {
668 for (MVT VT : {MVT::i16, MVT::f16, MVT::bf16}) {
669 setOperationAction(Op: ISD::ATOMIC_STORE, VT, Action: Promote);
670 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: VT, DestVT: MVT::i32);
671 }
672 }
673
674 // F16 - VOP1 Actions.
675 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
676 ISD::FSIN, ISD::FROUND},
677 VT: MVT::f16, Action: Custom);
678
679 // BF16 - VOP1 Actions.
680 if (Subtarget->hasBF16TransInsts())
681 setOperationAction(Ops: {ISD::FCOS, ISD::FSIN, ISD::FDIV}, VT: MVT::bf16, Action: Custom);
682
683 // F16 - VOP2 Actions.
684 setOperationAction(Ops: {ISD::BR_CC, ISD::SELECT_CC}, VTs: {MVT::f16, MVT::bf16},
685 Action: Expand);
686 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VT: MVT::f16, Action: Custom);
687 setOperationAction(Op: ISD::FFREXP, VT: MVT::f16, Action: Custom);
688 setOperationAction(Op: ISD::FDIV, VT: MVT::f16, Action: Custom);
689
690 // F16 - VOP3 Actions.
691 setOperationAction(Op: ISD::FMA, VT: MVT::f16, Action: Legal);
692 if (STI.hasMadF16())
693 setOperationAction(Op: ISD::FMAD, VT: MVT::f16, Action: Legal);
694
695 for (MVT VT :
696 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
697 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
698 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
699 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
700 switch (Op) {
701 case ISD::LOAD:
702 case ISD::STORE:
703 case ISD::ATOMIC_LOAD:
704 case ISD::ATOMIC_STORE:
705 case ISD::BUILD_VECTOR:
706 case ISD::BITCAST:
707 case ISD::UNDEF:
708 case ISD::POISON:
709 case ISD::EXTRACT_VECTOR_ELT:
710 case ISD::INSERT_VECTOR_ELT:
711 case ISD::INSERT_SUBVECTOR:
712 case ISD::SCALAR_TO_VECTOR:
713 case ISD::IS_FPCLASS:
714 break;
715 case ISD::EXTRACT_SUBVECTOR:
716 case ISD::CONCAT_VECTORS:
717 case ISD::FSIN:
718 case ISD::FCOS:
719 setOperationAction(Op, VT, Action: Custom);
720 break;
721 default:
722 setOperationAction(Op, VT, Action: Expand);
723 break;
724 }
725 }
726 }
727
728 // v_perm_b32 can handle either of these.
729 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::v2i16}, Action: Legal);
730 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i16, Action: Custom);
731
732 // Legalize vector types for sat conversions to select v_cvt_pk_[iu]16_f32.
733 if (Subtarget->hasVCvtPkIU16F32())
734 setOperationAction(
735 Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT},
736 VTs: {MVT::v2i16, MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16},
737 Action: Custom);
738
739 // XXX - Do these do anything? Vector constants turn into build_vector.
740 setOperationAction(Ops: ISD::Constant, VTs: {MVT::v2i16, MVT::v2f16}, Action: Legal);
741
742 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON},
743 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Legal);
744
745 setOperationAction(Op: ISD::STORE, VT: MVT::v2i16, Action: Promote);
746 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
747 setOperationAction(Op: ISD::STORE, VT: MVT::v2f16, Action: Promote);
748 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
749
750 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i16, Action: Promote);
751 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
752 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f16, Action: Promote);
753 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
754
755 setOperationAction(Op: ISD::AND, VT: MVT::v2i16, Action: Promote);
756 AddPromotedToType(Opc: ISD::AND, OrigVT: MVT::v2i16, DestVT: MVT::i32);
757 setOperationAction(Op: ISD::OR, VT: MVT::v2i16, Action: Promote);
758 AddPromotedToType(Opc: ISD::OR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
759 setOperationAction(Op: ISD::XOR, VT: MVT::v2i16, Action: Promote);
760 AddPromotedToType(Opc: ISD::XOR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
761
762 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i16, Action: Promote);
763 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
764 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f16, Action: Promote);
765 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
766 setOperationAction(Op: ISD::LOAD, VT: MVT::v4bf16, Action: Promote);
767 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
768
769 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
770 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
771 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
772 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
773 setOperationAction(Op: ISD::STORE, VT: MVT::v4bf16, Action: Promote);
774 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
775
776 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i16, Action: Promote);
777 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
778 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f16, Action: Promote);
779 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
780 setOperationAction(Op: ISD::LOAD, VT: MVT::v8bf16, Action: Promote);
781 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
782
783 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
784 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
785 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
786 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
787
788 setOperationAction(Op: ISD::STORE, VT: MVT::v8i16, Action: Promote);
789 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
790 setOperationAction(Op: ISD::STORE, VT: MVT::v8f16, Action: Promote);
791 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
792 setOperationAction(Op: ISD::STORE, VT: MVT::v8bf16, Action: Promote);
793 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
794
795 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i16, Action: Promote);
796 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
797 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f16, Action: Promote);
798 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
799 setOperationAction(Op: ISD::LOAD, VT: MVT::v16bf16, Action: Promote);
800 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
801
802 setOperationAction(Op: ISD::STORE, VT: MVT::v16i16, Action: Promote);
803 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
804 setOperationAction(Op: ISD::STORE, VT: MVT::v16f16, Action: Promote);
805 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
806 setOperationAction(Op: ISD::STORE, VT: MVT::v16bf16, Action: Promote);
807 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
808
809 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i16, Action: Promote);
810 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
811 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f16, Action: Promote);
812 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
813 setOperationAction(Op: ISD::LOAD, VT: MVT::v32bf16, Action: Promote);
814 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
815
816 setOperationAction(Op: ISD::STORE, VT: MVT::v32i16, Action: Promote);
817 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
818 setOperationAction(Op: ISD::STORE, VT: MVT::v32f16, Action: Promote);
819 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
820 setOperationAction(Op: ISD::STORE, VT: MVT::v32bf16, Action: Promote);
821 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
822
823 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
824 VT: MVT::v2i32, Action: Expand);
825 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
826
827 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
828 VT: MVT::v4i32, Action: Expand);
829
830 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
831 VT: MVT::v8i32, Action: Expand);
832
833 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
834 Action: Subtarget->hasVOP3PInsts() ? Legal : Custom);
835
836 setOperationAction(Ops: ISD::FNEG, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
837 // This isn't really legal, but this avoids the legalizer unrolling it (and
838 // allows matching fneg (fabs x) patterns)
839 setOperationAction(Ops: ISD::FABS, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
840
841 // Can do this in one BFI plus a constant materialize.
842 setOperationAction(Ops: ISD::FCOPYSIGN,
843 VTs: {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
844 MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
845 MVT::v32f16, MVT::v32bf16},
846 Action: Custom);
847 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
848 setOperationAction(
849 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
850 VT: MVT::f16, Action: Legal);
851
852 setOperationAction(
853 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
854 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, Action: Custom);
855 } else {
856 setOperationAction(
857 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
858 VT: MVT::f16, Action: Custom);
859
860 setOperationAction(Ops: {ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, VT: MVT::f16,
861 Action: Legal);
862
863 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
864 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
865 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
866 Action: Custom);
867
868 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM},
869 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
870 Action: Expand);
871 }
872
873 for (MVT Vec16 :
874 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
875 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
876 setOperationAction(Ops: {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT,
877 ISD::INSERT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
878 VT: Vec16, Action: Custom);
879 }
880 }
881
882 if (Subtarget->hasVOP3PInsts()) {
883 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
884 ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
885 ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
886 VT: MVT::v2i16, Action: Legal);
887
888 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
889 ISD::FCANONICALIZE},
890 VT: MVT::v2f16, Action: Legal);
891
892 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT,
893 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Custom);
894
895 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
896 VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
897 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
898 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
899 Action: Custom);
900
901 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
902 // Split vector operations.
903 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
904 ISD::MUL, ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN,
905 ISD::UMAX, ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
906 ISD::SSUBSAT},
907 VT, Action: Custom);
908
909 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
910 // Split vector operations.
911 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
912 ISD::FCANONICALIZE},
913 VT, Action: Custom);
914
915 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
916 setOperationAction(
917 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
918 VT: MVT::v2f16, Action: Legal);
919 } else {
920 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE}, VT: MVT::v2f16,
921 Action: Legal);
922
923 setOperationAction(
924 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
925 VTs: {MVT::v2f16, MVT::v4f16}, Action: Custom);
926 }
927 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f16, Action: Custom);
928 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
929 Action: Custom);
930
931 if (Subtarget->hasBF16PackedInsts()) {
932 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMAXNUM, ISD::FMINNUM,
933 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FMA,
934 ISD::FNEG, ISD::FABS, ISD::FCANONICALIZE},
935 VT: MVT::v2bf16, Action: Legal);
936
937 for (MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
938 // Split vector operations.
939 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE,
940 ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM,
941 ISD::FMAXIMUMNUM, ISD::FNEG, ISD::FABS},
942 VT, Action: Custom);
943 }
944
945 if (Subtarget->hasAnyPackedFP32Ops()) {
946 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
947 VT: MVT::v2f32, Action: Legal);
948 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
949 VTs: {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
950 Action: Custom);
951 }
952 if (Subtarget->hasAnyPackedFP64Ops()) {
953 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG,
954 ISD::FCANONICALIZE, ISD::BUILD_VECTOR},
955 VT: MVT::v2f64, Action: Legal);
956 setOperationAction(
957 Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FCANONICALIZE},
958 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Action: Custom);
959
960 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
961 setOperationAction(
962 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
963 VT: MVT::v2f64, Action: Legal);
964
965 setOperationAction(
966 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
967 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Action: Custom);
968 } else {
969 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE}, VT: MVT::v2f64,
970 Action: Legal);
971 setOperationAction(
972 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
973 VT: MVT::v2f64, Action: Custom);
974 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINNUM,
975 ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
976 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64},
977 Action: Custom);
978 }
979 }
980
981 if (Subtarget->hasAnyPackedU64Ops()) {
982 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL, ISD::BUILD_VECTOR},
983 VT: MVT::v2i64, Action: Legal);
984 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL},
985 VTs: {MVT::v4i64, MVT::v8i64, MVT::v16i64, MVT::v32i64},
986 Action: Custom);
987 }
988 }
989
990 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v4f16, Action: Custom);
991
992 if (Subtarget->has16BitInsts()) {
993 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i16, Action: Promote);
994 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2i16, DestVT: MVT::i32);
995 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f16, Action: Promote);
996 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f16, DestVT: MVT::i32);
997 setOperationAction(Op: ISD::SELECT, VT: MVT::v2bf16, Action: Promote);
998 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2bf16, DestVT: MVT::i32);
999 } else {
1000 // Legalization hack.
1001 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v2i16, MVT::v2f16}, Action: Custom);
1002
1003 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v2f16, Action: Custom);
1004 }
1005
1006 setOperationAction(Ops: ISD::SELECT,
1007 VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
1008 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1009 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
1010 MVT::v32f16, MVT::v32bf16},
1011 Action: Custom);
1012
1013 setOperationAction(Ops: {ISD::SMULO, ISD::UMULO}, VT: MVT::i64, Action: Custom);
1014
1015 if (Subtarget->useVMulU64Inst())
1016 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Legal);
1017 else if (Subtarget->hasScalarSMulU64())
1018 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
1019
1020 if (Subtarget->hasMad64_32())
1021 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: MVT::i32, Action: Custom);
1022
1023 if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
1024 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
1025 // Type legalization cannot expand the i128 pointer of a buffer resource
1026 // prefetch; lowerPREFETCH drops the prefetch instead.
1027 setOperationAction(Op: ISD::PREFETCH, VT: MVT::i128, Action: Custom);
1028
1029 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
1030 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM},
1031 VTs: {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Action: Legal);
1032 } else {
1033 // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum
1034 if (Subtarget->hasMinimum3Maximum3F32())
1035 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
1036
1037 if (Subtarget->hasMinimum3Maximum3PKF16()) {
1038 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::v2f16, Action: Legal);
1039
1040 // If only the vector form is available, we need to widen to a vector.
1041 if (!Subtarget->hasMinimum3Maximum3F16())
1042 setOperationPromotedToType(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, OrigVT: MVT::f16,
1043 DestVT: MVT::v2f16);
1044 }
1045 }
1046
1047 if (Subtarget->hasVOP3PInsts()) {
1048 // We want to break these into v2f16 pieces, not scalarize.
1049 setOperationAction(Ops: {ISD::FMINIMUM, ISD::FMAXIMUM},
1050 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
1051 Action: Custom);
1052 }
1053
1054 if (Subtarget->useMinMaxI64Insts())
1055 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i64,
1056 Action: Legal);
1057
1058 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
1059 VTs: {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
1060 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
1061 MVT::i8},
1062 Action: Custom);
1063
1064 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
1065 VTs: {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
1066 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
1067 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
1068 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1069 Action: Custom);
1070
1071 // The s_buffer_load intrinsics accept any result type in IR, but only a few
1072 // of them can be selected. Mark the remaining illegal result types Custom so
1073 // ReplaceNodeResults gets a chance to diagnose them instead of letting the
1074 // type legalizer abort. Its INTRINSIC_WO_CHAIN case dispatches on the
1075 // intrinsic ID, but INTRINSIC_W_CHAIN does not, so remember the types added
1076 // here to keep other chained intrinsics on generic legalization.
1077 for (MVT VT : MVT::all_valuetypes()) {
1078 if (VT.isScalableVector() || isTypeLegal(VT))
1079 continue;
1080 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT, Action: Custom);
1081 if (getOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT) != Custom) {
1082 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT, Action: Custom);
1083 SBufferLoadDiagnosticVTs.set(position: VT.SimpleTy);
1084 }
1085 }
1086
1087 setOperationAction(Ops: ISD::INTRINSIC_VOID,
1088 VTs: {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
1089 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
1090 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1091 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1092 Action: Custom);
1093
1094 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
1095 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
1096 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
1097 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i64, Action: Custom);
1098 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i64, Action: Custom);
1099
1100 // TODO: Could move this to custom lowering, could benefit from combines on
1101 // extract of relevant bits.
1102 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
1103
1104 setOperationAction(Op: ISD::MUL, VT: MVT::i1, Action: Promote);
1105
1106 if (Subtarget->hasBF16ConversionInsts()) {
1107 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND},
1108 VTs: {MVT::bf16, MVT::v2bf16}, Action: Custom);
1109 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Legal);
1110 }
1111
1112 if (Subtarget->hasBF16TransInsts()) {
1113 setOperationAction(Ops: {ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, VT: MVT::bf16, Action: Legal);
1114 }
1115
1116 const bool HasE5M3ConversionInsts =
1117 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
1118 if (Subtarget->hasOCPFP8ConversionInsts() || HasE5M3ConversionInsts) {
1119 setOperationAction(Ops: ISD::CONVERT_FROM_ARBITRARY_FP, VTs: {MVT::f32, MVT::v2f32},
1120 Action: Custom);
1121 setOperationAction(Op: ISD::CONVERT_FROM_ARBITRARY_FP, VT: MVT::v2i8, Action: Custom);
1122
1123 // i8 result promotes to i16, wider vectors split down to v2i8, and v2i8 is
1124 // handled in ReplaceNodeResults before the legalizer splits it per lane.
1125 setOperationAction(Ops: ISD::CONVERT_TO_ARBITRARY_FP, VTs: {MVT::i16, MVT::v2i8},
1126 Action: Custom);
1127 }
1128
1129 if (Subtarget->hasFP8F16ConversionInsts()) {
1130 setOperationAction(Ops: ISD::CONVERT_FROM_ARBITRARY_FP, VTs: {MVT::f16, MVT::v2f16},
1131 Action: Custom);
1132 }
1133
1134 if (Subtarget->hasCvtPkF16F32Inst()) {
1135 setOperationAction(Ops: ISD::FP_ROUND,
1136 VTs: {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
1137 Action: Custom);
1138 }
1139
1140 setTargetDAGCombine({ISD::ADD,
1141 ISD::PTRADD,
1142 ISD::SUB,
1143 ISD::MUL,
1144 ISD::FADD,
1145 ISD::FSUB,
1146 ISD::FDIV,
1147 ISD::FMUL,
1148 ISD::FMINNUM,
1149 ISD::FMAXNUM,
1150 ISD::FMINNUM_IEEE,
1151 ISD::FMAXNUM_IEEE,
1152 ISD::FMINIMUM,
1153 ISD::FMAXIMUM,
1154 ISD::FMINIMUMNUM,
1155 ISD::FMAXIMUMNUM,
1156 ISD::FMA,
1157 ISD::ABS,
1158 ISD::SMIN,
1159 ISD::SMAX,
1160 ISD::UMIN,
1161 ISD::UMAX,
1162 ISD::SETCC,
1163 ISD::SELECT,
1164 ISD::SMIN,
1165 ISD::SMAX,
1166 ISD::UMIN,
1167 ISD::UMAX,
1168 ISD::USUBSAT,
1169 ISD::UADDSAT,
1170 ISD::AND,
1171 ISD::OR,
1172 ISD::XOR,
1173 ISD::SHL,
1174 ISD::SRL,
1175 ISD::SRA,
1176 ISD::FSHR,
1177 ISD::SINT_TO_FP,
1178 ISD::UINT_TO_FP,
1179 ISD::FCANONICALIZE,
1180 ISD::SCALAR_TO_VECTOR,
1181 ISD::ZERO_EXTEND,
1182 ISD::SIGN_EXTEND,
1183 ISD::SIGN_EXTEND_INREG,
1184 ISD::ANY_EXTEND,
1185 ISD::EXTRACT_VECTOR_ELT,
1186 ISD::INSERT_VECTOR_ELT,
1187 ISD::FCOPYSIGN});
1188
1189 if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
1190 setTargetDAGCombine(ISD::FP_ROUND);
1191
1192 // All memory operations. Some folding on the pointer operand is done to help
1193 // matching the constant offsets in the addressing modes.
1194 setTargetDAGCombine({ISD::LOAD,
1195 ISD::STORE,
1196 ISD::ATOMIC_LOAD,
1197 ISD::ATOMIC_STORE,
1198 ISD::ATOMIC_CMP_SWAP,
1199 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
1200 ISD::ATOMIC_SWAP,
1201 ISD::ATOMIC_LOAD_ADD,
1202 ISD::ATOMIC_LOAD_SUB,
1203 ISD::ATOMIC_LOAD_AND,
1204 ISD::ATOMIC_LOAD_OR,
1205 ISD::ATOMIC_LOAD_XOR,
1206 ISD::ATOMIC_LOAD_NAND,
1207 ISD::ATOMIC_LOAD_MIN,
1208 ISD::ATOMIC_LOAD_MAX,
1209 ISD::ATOMIC_LOAD_UMIN,
1210 ISD::ATOMIC_LOAD_UMAX,
1211 ISD::ATOMIC_LOAD_FADD,
1212 ISD::ATOMIC_LOAD_FMIN,
1213 ISD::ATOMIC_LOAD_FMAX,
1214 ISD::ATOMIC_LOAD_UINC_WRAP,
1215 ISD::ATOMIC_LOAD_UDEC_WRAP,
1216 ISD::ATOMIC_LOAD_USUB_COND,
1217 ISD::ATOMIC_LOAD_USUB_SAT,
1218 ISD::INTRINSIC_VOID,
1219 ISD::INTRINSIC_W_CHAIN});
1220
1221 // FIXME: In other contexts we pretend this is a per-function property.
1222 setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
1223
1224 setSchedulingPreference(Sched::RegPressure);
1225}
1226
1227const GCNSubtarget *SITargetLowering::getSubtarget() const { return Subtarget; }
1228
1229ArrayRef<MCPhysReg> SITargetLowering::getRoundingControlRegisters() const {
1230 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1231 return RCRegs;
1232}
1233
1234//===----------------------------------------------------------------------===//
1235// TargetLowering queries
1236//===----------------------------------------------------------------------===//
1237
1238// v_mad_mix* support a conversion from f16 to f32.
1239//
1240// There is only one special case when denormals are enabled we don't currently,
1241// where this is OK to use.
1242bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
1243 EVT DestVT, EVT SrcVT) const {
1244 return DestVT.getScalarType() == MVT::f32 &&
1245 ((((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
1246 (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
1247 SrcVT.getScalarType() == MVT::f16) ||
1248 (Opcode == ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
1249 SrcVT.getScalarType() == MVT::bf16)) &&
1250 // TODO: This probably only requires no input flushing?
1251 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
1252}
1253
1254bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
1255 LLT DestTy, LLT SrcTy) const {
1256 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
1257 (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
1258 DestTy.getScalarSizeInBits() == 32 &&
1259 SrcTy.getScalarSizeInBits() == 16 &&
1260 // TODO: This probably only requires no input flushing?
1261 denormalModeIsFlushAllF32(MF: *MI.getMF());
1262}
1263
1264bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
1265 // SI has some legal vector types, but no legal vector operations. Say no
1266 // shuffles are legal in order to prefer scalarizing some vector operations.
1267 return false;
1268}
1269
1270MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1271 CallingConv::ID CC,
1272 EVT VT) const {
1273 if (CC == CallingConv::AMDGPU_KERNEL)
1274 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1275
1276 if (VT.isVector()) {
1277 EVT ScalarVT = VT.getScalarType();
1278 unsigned Size = ScalarVT.getSizeInBits();
1279 if (Size == 16) {
1280 return Subtarget->has16BitInsts()
1281 ? MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), NumElements: 2)
1282 : MVT::i32;
1283 }
1284
1285 if (Size < 16)
1286 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
1287 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
1288 }
1289
1290 if (!Subtarget->has16BitInsts() && VT.getSizeInBits() == 16)
1291 return MVT::i32;
1292
1293 if (VT.getSizeInBits() > 32)
1294 return MVT::i32;
1295
1296 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1297}
1298
1299unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1300 CallingConv::ID CC,
1301 EVT VT) const {
1302 if (CC == CallingConv::AMDGPU_KERNEL)
1303 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1304
1305 if (VT.isVector()) {
1306 unsigned NumElts = VT.getVectorNumElements();
1307 EVT ScalarVT = VT.getScalarType();
1308 unsigned Size = ScalarVT.getSizeInBits();
1309
1310 // FIXME: Should probably promote 8-bit vectors to i16.
1311 if (Size == 16)
1312 return (NumElts + 1) / 2;
1313
1314 if (Size <= 32)
1315 return NumElts;
1316
1317 if (Size > 32)
1318 return NumElts * ((Size + 31) / 32);
1319 } else if (VT.getSizeInBits() > 32)
1320 return (VT.getSizeInBits() + 31) / 32;
1321
1322 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1323}
1324
1325unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
1326 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1327 unsigned &NumIntermediates, MVT &RegisterVT) const {
1328 if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
1329 unsigned NumElts = VT.getVectorNumElements();
1330 EVT ScalarVT = VT.getScalarType();
1331 unsigned Size = ScalarVT.getSizeInBits();
1332 // FIXME: We should fix the ABI to be the same on targets without 16-bit
1333 // support, but unless we can properly handle 3-vectors, it will be still be
1334 // inconsistent.
1335 if (Size == 16) {
1336 MVT SimpleIntermediateVT =
1337 MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), EC: ElementCount::getFixed(MinVal: 2));
1338 IntermediateVT = SimpleIntermediateVT;
1339 RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
1340 NumIntermediates = (NumElts + 1) / 2;
1341 return (NumElts + 1) / 2;
1342 }
1343
1344 if (Size == 32) {
1345 RegisterVT = ScalarVT.getSimpleVT();
1346 IntermediateVT = RegisterVT;
1347 NumIntermediates = NumElts;
1348 return NumIntermediates;
1349 }
1350
1351 if (Size < 16 && Subtarget->has16BitInsts()) {
1352 // FIXME: Should probably form v2i16 pieces
1353 RegisterVT = MVT::i16;
1354 IntermediateVT = ScalarVT;
1355 NumIntermediates = NumElts;
1356 return NumIntermediates;
1357 }
1358
1359 if (Size != 16 && Size <= 32) {
1360 RegisterVT = MVT::i32;
1361 IntermediateVT = ScalarVT;
1362 NumIntermediates = NumElts;
1363 return NumIntermediates;
1364 }
1365
1366 if (Size > 32) {
1367 RegisterVT = MVT::i32;
1368 IntermediateVT = RegisterVT;
1369 NumIntermediates = NumElts * ((Size + 31) / 32);
1370 return NumIntermediates;
1371 }
1372 }
1373
1374 return TargetLowering::getVectorTypeBreakdownForCallingConv(
1375 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1376}
1377
1378static EVT memVTFromLoadIntrData(const SITargetLowering &TLI,
1379 const DataLayout &DL, Type *Ty,
1380 unsigned MaxNumLanes) {
1381 assert(MaxNumLanes != 0);
1382
1383 LLVMContext &Ctx = Ty->getContext();
1384 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1385 unsigned NumElts = std::min(a: MaxNumLanes, b: VT->getNumElements());
1386 return EVT::getVectorVT(Context&: Ctx, VT: TLI.getValueType(DL, Ty: VT->getElementType()),
1387 NumElements: NumElts);
1388 }
1389
1390 return TLI.getValueType(DL, Ty);
1391}
1392
1393// Peek through TFE struct returns to only use the data size.
1394static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI,
1395 const DataLayout &DL, Type *Ty,
1396 unsigned MaxNumLanes) {
1397 auto *ST = dyn_cast<StructType>(Val: Ty);
1398 if (!ST)
1399 return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
1400
1401 // TFE intrinsics return an aggregate type.
1402 assert(ST->getNumContainedTypes() == 2 &&
1403 ST->getContainedType(1)->isIntegerTy(32));
1404 return memVTFromLoadIntrData(TLI, DL, Ty: ST->getContainedType(i: 0), MaxNumLanes);
1405}
1406
1407/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its
1408/// in-memory representation. This return value is a custom type because there
1409/// is no MVT::i160 and adding one breaks integer promotion logic. While this
1410/// could cause issues during codegen, these address space 7 pointers will be
1411/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer
1412/// in order to allow pre-codegen passes that query TargetTransformInfo, often
1413/// for cost modeling, to work. (This also sets us up decently for doing the
1414/// buffer lowering in GlobalISel if SelectionDAG ever goes away.)
1415MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
1416 if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
1417 return MVT::amdgpuBufferFatPointer;
1418 if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1419 DL.getPointerSizeInBits(AS) == 192)
1420 return MVT::amdgpuBufferStridedPointer;
1421 return AMDGPUTargetLowering::getPointerTy(DL, AS);
1422}
1423/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
1424/// v8i32 when padding is added.
1425/// The in-memory representation of a p9 is {p8, i32, i32}, which is
1426/// also v8i32 with padding.
1427MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
1428 if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
1429 DL.getPointerSizeInBits(AS) == 160) ||
1430 (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1431 DL.getPointerSizeInBits(AS) == 192))
1432 return MVT::v8i32;
1433 return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
1434}
1435
1436static unsigned getIntrMemWidth(unsigned IntrID) {
1437 switch (IntrID) {
1438 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1439 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1440 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1441 return 8;
1442 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1443 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1444 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1445 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1446 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1447 case Intrinsic::amdgcn_flat_load_monitor_b32:
1448 case Intrinsic::amdgcn_global_load_monitor_b32:
1449 return 32;
1450 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1451 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1452 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1453 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1454 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1455 case Intrinsic::amdgcn_flat_load_monitor_b64:
1456 case Intrinsic::amdgcn_global_load_monitor_b64:
1457 return 64;
1458 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1459 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1460 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1461 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
1462 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
1463 case Intrinsic::amdgcn_flat_load_monitor_b128:
1464 case Intrinsic::amdgcn_global_load_monitor_b128:
1465 return 128;
1466 default:
1467 llvm_unreachable("Unknown width");
1468 }
1469}
1470
1471static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI,
1472 unsigned ArgIdx) {
1473 Value *OrderingArg = CI.getArgOperand(i: ArgIdx);
1474 unsigned Ord = cast<ConstantInt>(Val: OrderingArg)->getZExtValue();
1475 switch (AtomicOrderingCABI(Ord)) {
1476 case AtomicOrderingCABI::acquire:
1477 return AtomicOrdering::Acquire;
1478 break;
1479 case AtomicOrderingCABI::release:
1480 return AtomicOrdering::Release;
1481 break;
1482 case AtomicOrderingCABI::seq_cst:
1483 return AtomicOrdering::SequentiallyConsistent;
1484 break;
1485 default:
1486 return AtomicOrdering::Monotonic;
1487 }
1488}
1489
1490static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
1491 MDNode *ScopeMD = cast<MDNode>(
1492 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ArgIdx))->getMetadata());
1493 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1494 return CI.getContext().getOrInsertSyncScopeID(SSN: Scope);
1495}
1496
1497void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
1498 const CallBase &CI,
1499 MachineFunction &MF,
1500 unsigned IntrID) const {
1501 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
1502 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
1503 Flags |= MachineMemOperand::MOInvariant;
1504 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
1505 Flags |= MachineMemOperand::MONonTemporal;
1506 Flags |= getTargetMMOFlags(I: CI);
1507
1508 if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
1509 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
1510 AttributeSet Attr =
1511 Intrinsic::getFnAttributes(C&: CI.getContext(), id: (Intrinsic::ID)IntrID);
1512 MemoryEffects ME = Attr.getMemoryEffects();
1513 if (ME.doesNotAccessMemory())
1514 return;
1515
1516 const AMDGPU::BufferIntrinsicInfo *BufInfo =
1517 AMDGPU::lookupBufferIntrinsic(Intr: IntrID);
1518 bool IsSPrefetch = BufInfo && BufInfo->Kind == AMDGPU::BufferKindPrefetch;
1519 std::optional<unsigned> AuxArg =
1520 BufInfo ? BufInfo->getAuxArg() : CI.arg_size() - 1;
1521 if (AuxArg) {
1522 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: *AuxArg));
1523 if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
1524 Flags |= MachineMemOperand::MOVolatile;
1525 }
1526
1527 Flags |= MachineMemOperand::MODereferenceable;
1528
1529 IntrinsicInfo Info;
1530 // TODO: Should images get their own address space?
1531 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1532
1533 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
1534 if (RsrcIntr->IsImage) {
1535 const AMDGPU::ImageDimIntrinsicInfo *Intr =
1536 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID);
1537 BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Intr->BaseOpcode);
1538 Info.align.reset();
1539 }
1540
1541 Value *RsrcArg = CI.getArgOperand(i: RsrcIntr->RsrcArg);
1542 if (auto *RsrcPtrTy = dyn_cast<PointerType>(Val: RsrcArg->getType())) {
1543 if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
1544 // We conservatively set the memory operand of a buffer intrinsic to the
1545 // base resource pointer, so that we can access alias information about
1546 // those pointers. Cases like "this points at the same value
1547 // but with a different offset" are handled in
1548 // areMemAccessesTriviallyDisjoint.
1549 Info.ptrVal = RsrcArg;
1550 }
1551
1552 if (ME.onlyReadsMemory()) {
1553 if (RsrcIntr->IsImage) {
1554 unsigned MaxNumLanes = 4;
1555
1556 if (!BaseOpcode->Gather4) {
1557 // If this isn't a gather, we may have excess loaded elements in the
1558 // IR type. Check the dmask for the real number of elements loaded.
1559 unsigned DMask =
1560 cast<ConstantInt>(Val: CI.getArgOperand(i: 0))->getZExtValue();
1561 MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1562 }
1563
1564 Info.memVT = memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(),
1565 Ty: CI.getType(), MaxNumLanes);
1566 } else {
1567 Info.memVT =
1568 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1569 MaxNumLanes: std::numeric_limits<unsigned>::max());
1570 }
1571
1572 // FIXME: What does alignment mean for an image?
1573 Info.opc = ISD::INTRINSIC_W_CHAIN;
1574 Info.flags = Flags | MachineMemOperand::MOLoad;
1575 } else if (ME.onlyWritesMemory()) {
1576 Info.opc = ISD::INTRINSIC_VOID;
1577
1578 Type *DataTy = CI.getArgOperand(i: 0)->getType();
1579 if (RsrcIntr->IsImage) {
1580 unsigned DMask = cast<ConstantInt>(Val: CI.getArgOperand(i: 1))->getZExtValue();
1581 unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1582 Info.memVT = memVTFromLoadIntrData(TLI: *this, DL: MF.getDataLayout(), Ty: DataTy,
1583 MaxNumLanes: DMaskLanes);
1584 } else
1585 Info.memVT = getValueType(DL: MF.getDataLayout(), Ty: DataTy);
1586
1587 Info.flags = Flags | MachineMemOperand::MOStore;
1588 } else {
1589 // Atomic, NoReturn Sampler or prefetch
1590 Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
1591 : ISD::INTRINSIC_W_CHAIN;
1592
1593 if (BufInfo && BufInfo->Kind == AMDGPU::BufferKindLoadToLDS) {
1594 unsigned Width =
1595 cast<ConstantInt>(Val: CI.getArgOperand(i: *BufInfo->getSizeArg()))
1596 ->getZExtValue();
1597
1598 // Entry 0: Load from buffer.
1599 // Don't set an offset, since the pointer value always represents the
1600 // base of the buffer.
1601 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1602 Info.flags = Flags | MachineMemOperand::MOLoad;
1603 Infos.push_back(Elt: Info);
1604
1605 // Entry 1: Store to LDS.
1606 // Instruction offset is applied, and an additional per-lane offset
1607 // which we simulate using a larger memory type.
1608 Info.memVT = EVT::getIntegerVT(
1609 Context&: CI.getContext(), BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1610 Info.ptrVal = CI.getArgOperand(i: *BufInfo->getLDSPtrArg());
1611 Info.offset =
1612 cast<ConstantInt>(Val: CI.getArgOperand(i: *BufInfo->getImmOffsetArg()))
1613 ->getZExtValue();
1614 Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
1615 Info.flags = Flags | MachineMemOperand::MOStore;
1616 Infos.push_back(Elt: Info);
1617 return;
1618 }
1619 if (BufInfo && BufInfo->Kind == AMDGPU::BufferKindAtomicLoad) {
1620 Info.memVT =
1621 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1622 MaxNumLanes: std::numeric_limits<unsigned>::max());
1623 Info.flags = Flags | MachineMemOperand::MOLoad;
1624 Infos.push_back(Elt: Info);
1625 return;
1626 }
1627
1628 Info.flags = Flags | MachineMemOperand::MOLoad;
1629 if (!IsSPrefetch)
1630 Info.flags |= MachineMemOperand::MOStore;
1631
1632 if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
1633 // Fake memory access type for no return sampler intrinsics
1634 Info.memVT = MVT::i32;
1635 } else {
1636 // XXX - Should this be volatile without known ordering?
1637 Info.flags |= MachineMemOperand::MOVolatile;
1638 Info.memVT = MVT::getVT(Ty: CI.getArgOperand(i: 0)->getType());
1639 }
1640 }
1641 Infos.push_back(Elt: Info);
1642 return;
1643 }
1644
1645 IntrinsicInfo Info;
1646 switch (IntrID) {
1647 case Intrinsic::amdgcn_ds_ordered_add:
1648 case Intrinsic::amdgcn_ds_ordered_swap: {
1649 Info.opc = ISD::INTRINSIC_W_CHAIN;
1650 Info.memVT = MVT::getVT(Ty: CI.getType());
1651 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1652 Info.align.reset();
1653 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1654
1655 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 4));
1656 if (!Vol->isZero())
1657 Info.flags |= MachineMemOperand::MOVolatile;
1658
1659 Infos.push_back(Elt: Info);
1660 return;
1661 }
1662 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1663 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1664 Info.opc = ISD::INTRINSIC_W_CHAIN;
1665 Info.memVT = MVT::getVT(Ty: CI.getOperand(i_nocapture: 0)->getType());
1666 Info.ptrVal = nullptr;
1667 Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
1668 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1669 Infos.push_back(Elt: Info);
1670 return;
1671 }
1672 case Intrinsic::amdgcn_ds_append:
1673 case Intrinsic::amdgcn_ds_consume: {
1674 Info.opc = ISD::INTRINSIC_W_CHAIN;
1675 Info.memVT = MVT::getVT(Ty: CI.getType());
1676 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1677 Info.align.reset();
1678 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1679
1680 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
1681 if (!Vol->isZero())
1682 Info.flags |= MachineMemOperand::MOVolatile;
1683
1684 Infos.push_back(Elt: Info);
1685 return;
1686 }
1687 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1688 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
1689 Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
1690 ? ISD::INTRINSIC_W_CHAIN
1691 : ISD::INTRINSIC_VOID;
1692 Info.memVT = MVT::getVT(Ty: CI.getType());
1693 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1694 Info.memVT = MVT::i64;
1695 Info.size = 8;
1696 Info.align.reset();
1697 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1698 Info.order = AtomicOrdering::Monotonic;
1699 Infos.push_back(Elt: Info);
1700 return;
1701 }
1702 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
1703 case Intrinsic::amdgcn_image_bvh_intersect_ray:
1704 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
1705 Info.opc = ISD::INTRINSIC_W_CHAIN;
1706 Info.memVT =
1707 MVT::getVT(Ty: IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
1708 ? CI.getType()
1709 : cast<StructType>(Val: CI.getType())
1710 ->getElementType(N: 0)); // XXX: what is correct VT?
1711
1712 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1713 Info.align.reset();
1714 Info.flags = Flags | MachineMemOperand::MOLoad |
1715 MachineMemOperand::MODereferenceable;
1716 Infos.push_back(Elt: Info);
1717 return;
1718 }
1719 case Intrinsic::amdgcn_global_atomic_fmin_num:
1720 case Intrinsic::amdgcn_global_atomic_fmax_num:
1721 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1722 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1723 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
1724 Info.opc = ISD::INTRINSIC_W_CHAIN;
1725 Info.memVT = MVT::getVT(Ty: CI.getType());
1726 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1727 Info.align.reset();
1728 Info.flags =
1729 Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1730 MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile;
1731 Infos.push_back(Elt: Info);
1732 return;
1733 }
1734 case Intrinsic::amdgcn_cluster_load_b32:
1735 case Intrinsic::amdgcn_cluster_load_b64:
1736 case Intrinsic::amdgcn_cluster_load_b128:
1737 case Intrinsic::amdgcn_ds_load_tr6_b96:
1738 case Intrinsic::amdgcn_ds_load_tr4_b64:
1739 case Intrinsic::amdgcn_ds_load_tr8_b64:
1740 case Intrinsic::amdgcn_ds_load_tr16_b128:
1741 case Intrinsic::amdgcn_global_load_tr6_b96:
1742 case Intrinsic::amdgcn_global_load_tr4_b64:
1743 case Intrinsic::amdgcn_global_load_tr_b64:
1744 case Intrinsic::amdgcn_global_load_tr_b128:
1745 case Intrinsic::amdgcn_ds_read_tr4_b64:
1746 case Intrinsic::amdgcn_ds_read_tr6_b96:
1747 case Intrinsic::amdgcn_ds_read_tr8_b64:
1748 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1749 Info.opc = ISD::INTRINSIC_W_CHAIN;
1750 Info.memVT = MVT::getVT(Ty: CI.getType());
1751 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1752 Info.align.reset();
1753 Info.flags = Flags | MachineMemOperand::MOLoad;
1754 Infos.push_back(Elt: Info);
1755 return;
1756 }
1757 case Intrinsic::amdgcn_flat_load_monitor_b32:
1758 case Intrinsic::amdgcn_flat_load_monitor_b64:
1759 case Intrinsic::amdgcn_flat_load_monitor_b128:
1760 case Intrinsic::amdgcn_global_load_monitor_b32:
1761 case Intrinsic::amdgcn_global_load_monitor_b64:
1762 case Intrinsic::amdgcn_global_load_monitor_b128: {
1763 Info.opc = ISD::INTRINSIC_W_CHAIN;
1764 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1765 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1766 Info.align.reset();
1767 Info.flags = MachineMemOperand::MOLoad;
1768 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1769 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1770 Infos.push_back(Elt: Info);
1771 return;
1772 }
1773 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1774 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1775 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
1776 Info.opc = ISD::INTRINSIC_W_CHAIN;
1777 Info.memVT = MVT::getVT(Ty: CI.getType());
1778 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1779 Info.align.reset();
1780 Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
1781 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1782 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1783 Infos.push_back(Elt: Info);
1784 return;
1785 }
1786 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1787 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1788 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
1789 Info.opc = ISD::INTRINSIC_VOID;
1790 Info.memVT = MVT::getVT(Ty: CI.getArgOperand(i: 1)->getType());
1791 Info.ptrVal = CI.getArgOperand(i: 0);
1792 Info.align.reset();
1793 Info.flags = (MachineMemOperand::MOStore | MOCooperative);
1794 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 2);
1795 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 3);
1796 Infos.push_back(Elt: Info);
1797 return;
1798 }
1799 case Intrinsic::amdgcn_ds_gws_init:
1800 case Intrinsic::amdgcn_ds_gws_barrier:
1801 case Intrinsic::amdgcn_ds_gws_sema_v:
1802 case Intrinsic::amdgcn_ds_gws_sema_br:
1803 case Intrinsic::amdgcn_ds_gws_sema_p:
1804 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1805 Info.opc = ISD::INTRINSIC_VOID;
1806
1807 const GCNTargetMachine &TM =
1808 static_cast<const GCNTargetMachine &>(getTargetMachine());
1809
1810 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1811 Info.ptrVal = MFI->getGWSPSV(TM);
1812
1813 // This is an abstract access, but we need to specify a type and size.
1814 Info.memVT = MVT::i32;
1815 Info.size = 4;
1816 Info.align = Align(4);
1817
1818 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1819 Info.flags = Flags | MachineMemOperand::MOLoad;
1820 else
1821 Info.flags = Flags | MachineMemOperand::MOStore;
1822 Infos.push_back(Elt: Info);
1823 return;
1824 }
1825 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1826 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1827 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1828 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1829 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1830 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1831 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1832 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
1833 // Entry 0: Load from source (global/flat).
1834 Info.opc = ISD::INTRINSIC_VOID;
1835 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1836 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1837 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1838 Info.flags = Flags | MachineMemOperand::MOLoad;
1839 Infos.push_back(Elt: Info);
1840
1841 // Entry 1: Store to LDS (same offset).
1842 Info.flags = Flags | MachineMemOperand::MOStore;
1843 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1844 Infos.push_back(Elt: Info);
1845 return;
1846 }
1847 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1848 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1849 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1850 case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
1851 // Entry 0: Load from LDS.
1852 Info.opc = ISD::INTRINSIC_VOID;
1853 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1854 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1855 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1856 Info.flags = Flags | MachineMemOperand::MOLoad;
1857 Infos.push_back(Elt: Info);
1858
1859 // Entry 1: Store to global (same offset).
1860 Info.flags = Flags | MachineMemOperand::MOStore;
1861 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1862 Infos.push_back(Elt: Info);
1863 return;
1864 }
1865 case Intrinsic::amdgcn_av_load_b8:
1866 case Intrinsic::amdgcn_av_load_b16:
1867 case Intrinsic::amdgcn_av_load_b32:
1868 case Intrinsic::amdgcn_av_load_b64:
1869 case Intrinsic::amdgcn_av_load_b128:
1870 case Intrinsic::amdgcn_av_store_b8:
1871 case Intrinsic::amdgcn_av_store_b16:
1872 case Intrinsic::amdgcn_av_store_b32:
1873 case Intrinsic::amdgcn_av_store_b64:
1874 case Intrinsic::amdgcn_av_store_b128: {
1875 bool IsStore = CI.getType()->isVoidTy();
1876 Type *DataTy = IsStore ? CI.getArgOperand(i: 1)->getType() : CI.getType();
1877 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
1878 Info.memVT = MVT::getVT(Ty: DataTy);
1879 Info.ptrVal = CI.getArgOperand(i: 0);
1880 Info.align = Align(Info.memVT.getStoreSize());
1881 Info.flags |=
1882 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
1883 // Pretend to be atomic so that SIMemoryLegalizer::expandStore sets cache
1884 // flags appropriately.
1885 Info.order = AtomicOrdering::Monotonic;
1886
1887 LLVMContext &Ctx = CI.getContext();
1888 unsigned ScopeIdx = CI.arg_size() - 1;
1889 MDNode *ScopeMD = cast<MDNode>(
1890 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ScopeIdx))->getMetadata());
1891 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1892 Info.ssid = Ctx.getOrInsertSyncScopeID(SSN: Scope);
1893 Infos.push_back(Elt: Info);
1894 return;
1895 }
1896 case Intrinsic::amdgcn_load_to_lds:
1897 case Intrinsic::amdgcn_load_async_to_lds:
1898 case Intrinsic::amdgcn_global_load_lds:
1899 case Intrinsic::amdgcn_global_load_async_lds: {
1900 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1901 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1902 bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
1903 if (IsVolatile)
1904 Flags |= MachineMemOperand::MOVolatile;
1905
1906 // Entry 0: Load from source (global/flat).
1907 Info.opc = ISD::INTRINSIC_VOID;
1908 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1909 Info.ptrVal = CI.getArgOperand(i: 0); // Source pointer
1910 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 3))->getSExtValue();
1911 Info.flags = Flags | MachineMemOperand::MOLoad;
1912 Infos.push_back(Elt: Info);
1913
1914 // Entry 1: Store to LDS.
1915 // Same offset from the instruction, but an additional per-lane offset is
1916 // added. Represent that using a wider memory type.
1917 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(),
1918 BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1919 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1920 Info.flags = Flags | MachineMemOperand::MOStore;
1921 Infos.push_back(Elt: Info);
1922 return;
1923 }
1924 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
1925 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
1926 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
1927 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1928 Info.opc = ISD::INTRINSIC_W_CHAIN;
1929
1930 const GCNTargetMachine &TM =
1931 static_cast<const GCNTargetMachine &>(getTargetMachine());
1932
1933 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1934 Info.ptrVal = MFI->getGWSPSV(TM);
1935
1936 // This is an abstract access, but we need to specify a type and size.
1937 Info.memVT = MVT::i32;
1938 Info.size = 4;
1939 Info.align = Align(4);
1940
1941 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1942 Infos.push_back(Elt: Info);
1943 return;
1944 }
1945 case Intrinsic::amdgcn_s_prefetch_data:
1946 case Intrinsic::amdgcn_s_prefetch_inst:
1947 case Intrinsic::amdgcn_flat_prefetch:
1948 case Intrinsic::amdgcn_global_prefetch: {
1949 Info.opc = ISD::INTRINSIC_VOID;
1950 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: 8);
1951 Info.ptrVal = CI.getArgOperand(i: 0);
1952 Info.flags = Flags | MachineMemOperand::MOLoad;
1953 Infos.push_back(Elt: Info);
1954 return;
1955 }
1956 default:
1957 return;
1958 }
1959}
1960
1961bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
1962 SmallVectorImpl<Value *> &Ops,
1963 Type *&AccessTy) const {
1964 Value *Ptr = nullptr;
1965 switch (II->getIntrinsicID()) {
1966 case Intrinsic::amdgcn_cluster_load_b128:
1967 case Intrinsic::amdgcn_cluster_load_b64:
1968 case Intrinsic::amdgcn_cluster_load_b32:
1969 case Intrinsic::amdgcn_ds_append:
1970 case Intrinsic::amdgcn_ds_consume:
1971 case Intrinsic::amdgcn_ds_load_tr8_b64:
1972 case Intrinsic::amdgcn_ds_load_tr16_b128:
1973 case Intrinsic::amdgcn_ds_load_tr4_b64:
1974 case Intrinsic::amdgcn_ds_load_tr6_b96:
1975 case Intrinsic::amdgcn_ds_read_tr4_b64:
1976 case Intrinsic::amdgcn_ds_read_tr6_b96:
1977 case Intrinsic::amdgcn_ds_read_tr8_b64:
1978 case Intrinsic::amdgcn_ds_read_tr16_b64:
1979 case Intrinsic::amdgcn_ds_ordered_add:
1980 case Intrinsic::amdgcn_ds_ordered_swap:
1981 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1982 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
1983 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1984 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1985 case Intrinsic::amdgcn_global_atomic_fmax_num:
1986 case Intrinsic::amdgcn_global_atomic_fmin_num:
1987 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1988 case Intrinsic::amdgcn_global_load_tr_b64:
1989 case Intrinsic::amdgcn_global_load_tr_b128:
1990 case Intrinsic::amdgcn_global_load_tr4_b64:
1991 case Intrinsic::amdgcn_global_load_tr6_b96:
1992 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1993 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1994 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1995 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1996 case Intrinsic::amdgcn_av_load_b8:
1997 case Intrinsic::amdgcn_av_load_b16:
1998 case Intrinsic::amdgcn_av_load_b32:
1999 case Intrinsic::amdgcn_av_load_b64:
2000 case Intrinsic::amdgcn_av_load_b128:
2001 case Intrinsic::amdgcn_av_store_b8:
2002 case Intrinsic::amdgcn_av_store_b16:
2003 case Intrinsic::amdgcn_av_store_b32:
2004 case Intrinsic::amdgcn_av_store_b64:
2005 case Intrinsic::amdgcn_av_store_b128:
2006 Ptr = II->getArgOperand(i: 0);
2007 break;
2008 case Intrinsic::amdgcn_load_to_lds:
2009 case Intrinsic::amdgcn_load_async_to_lds:
2010 case Intrinsic::amdgcn_global_load_lds:
2011 case Intrinsic::amdgcn_global_load_async_lds:
2012 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
2013 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
2014 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
2015 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
2016 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
2017 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
2018 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
2019 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
2020 Ptr = II->getArgOperand(i: 1);
2021 break;
2022 default:
2023 return false;
2024 }
2025 AccessTy = II->getType();
2026 Ops.push_back(Elt: Ptr);
2027 return true;
2028}
2029
2030bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM,
2031 unsigned AddrSpace) const {
2032 if (!Subtarget->hasFlatInstOffsets()) {
2033 // Flat instructions do not have offsets, and only have the register
2034 // address.
2035 return AM.BaseOffs == 0 && AM.Scale == 0;
2036 }
2037
2038 using AMDGPU::FlatAddrSpace;
2039 FlatAddrSpace FlatVariant =
2040 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ? FlatAddrSpace::FlatGlobal
2041 : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? FlatAddrSpace::FlatScratch
2042 : FlatAddrSpace::FLAT;
2043
2044 return AM.Scale == 0 &&
2045 (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
2046 Offset: AM.BaseOffs, AddrSpace, FlatVariant));
2047}
2048
2049bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
2050 if (Subtarget->hasFlatGlobalInsts())
2051 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS);
2052
2053 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
2054 // Assume the we will use FLAT for all global memory accesses
2055 // on VI.
2056 // FIXME: This assumption is currently wrong. On VI we still use
2057 // MUBUF instructions for the r + i addressing mode. As currently
2058 // implemented, the MUBUF instructions only work on buffer < 4GB.
2059 // It may be possible to support > 4GB buffers with MUBUF instructions,
2060 // by setting the stride value in the resource descriptor which would
2061 // increase the size limit to (stride * 4GB). However, this is risky,
2062 // because it has never been validated.
2063 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2064 }
2065
2066 return isLegalMUBUFAddressingMode(AM);
2067}
2068
2069bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
2070 // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
2071 // additionally can do r + r + i with addr64. 32-bit has more addressing
2072 // mode options. Depending on the resource constant, it can also do
2073 // (i64 r0) + (i32 r1) * (i14 i).
2074 //
2075 // Private arrays end up using a scratch buffer most of the time, so also
2076 // assume those use MUBUF instructions. Scratch loads / stores are currently
2077 // implemented as mubuf instructions with offen bit set, so slightly
2078 // different than the normal addr64.
2079 const SIInstrInfo *TII = Subtarget->getInstrInfo();
2080 if (!TII->isLegalMUBUFImmOffset(Imm: AM.BaseOffs))
2081 return false;
2082
2083 // FIXME: Since we can split immediate into soffset and immediate offset,
2084 // would it make sense to allow any immediate?
2085
2086 switch (AM.Scale) {
2087 case 0: // r + i or just i, depending on HasBaseReg.
2088 return true;
2089 case 1:
2090 return true; // We have r + r or r + i.
2091 case 2:
2092 if (AM.HasBaseReg) {
2093 // Reject 2 * r + r.
2094 return false;
2095 }
2096
2097 // Allow 2 * r as r + r
2098 // Or 2 * r + i is allowed as r + r + i.
2099 return true;
2100 default: // Don't allow n * r
2101 return false;
2102 }
2103}
2104
2105bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
2106 const AddrMode &AM, Type *Ty,
2107 unsigned AS,
2108 Instruction *I) const {
2109 // No global is ever allowed as a base.
2110 if (AM.BaseGV)
2111 return false;
2112
2113 if (AS == AMDGPUAS::GLOBAL_ADDRESS)
2114 return isLegalGlobalAddressingMode(AM);
2115
2116 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
2117 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
2118 AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
2119 AS == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2120 // If the offset isn't a multiple of 4, it probably isn't going to be
2121 // correctly aligned.
2122 // FIXME: Can we get the real alignment here?
2123 if (AM.BaseOffs % 4 != 0)
2124 return isLegalMUBUFAddressingMode(AM);
2125
2126 if (!Subtarget->hasScalarSubwordLoads()) {
2127 // There are no SMRD extloads, so if we have to do a small type access we
2128 // will use a MUBUF load.
2129 // FIXME?: We also need to do this if unaligned, but we don't know the
2130 // alignment here.
2131 if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
2132 return isLegalGlobalAddressingMode(AM);
2133 }
2134
2135 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
2136 // SMRD instructions have an 8-bit, dword offset on SI.
2137 if (!isUInt<8>(x: AM.BaseOffs / 4))
2138 return false;
2139 } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
2140 // On CI+, this can also be a 32-bit literal constant offset. If it fits
2141 // in 8-bits, it can use a smaller encoding.
2142 if (!isUInt<32>(x: AM.BaseOffs / 4))
2143 return false;
2144 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
2145 // On VI, these use the SMEM format and the offset is 20-bit in bytes.
2146 if (!isUInt<20>(x: AM.BaseOffs))
2147 return false;
2148 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
2149 // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
2150 // for S_BUFFER_* instructions).
2151 if (!isInt<21>(x: AM.BaseOffs))
2152 return false;
2153 } else {
2154 // On GFX12, all offsets are signed 24-bit in bytes.
2155 if (!isInt<24>(x: AM.BaseOffs))
2156 return false;
2157 }
2158
2159 if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
2160 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
2161 AM.BaseOffs < 0) {
2162 // Scalar (non-buffer) loads can only use a negative offset if
2163 // soffset+offset is non-negative. Since the compiler can only prove that
2164 // in a few special cases, it is safer to claim that negative offsets are
2165 // not supported.
2166 return false;
2167 }
2168
2169 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2170 return true;
2171
2172 if (AM.Scale == 1 && AM.HasBaseReg)
2173 return true;
2174
2175 return false;
2176 }
2177
2178 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
2179 return Subtarget->hasFlatScratchEnabled()
2180 ? isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS)
2181 : isLegalMUBUFAddressingMode(AM);
2182
2183 if (AS == AMDGPUAS::LOCAL_ADDRESS ||
2184 (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
2185 // Basic, single offset DS instructions allow a 16-bit unsigned immediate
2186 // field.
2187 // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
2188 // an 8-bit dword offset but we don't know the alignment here.
2189 if (!isUInt<16>(x: AM.BaseOffs))
2190 return false;
2191
2192 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2193 return true;
2194
2195 if (AM.Scale == 1 && AM.HasBaseReg)
2196 return true;
2197
2198 return false;
2199 }
2200
2201 if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
2202 // For an unknown address space, this usually means that this is for some
2203 // reason being used for pure arithmetic, and not based on some addressing
2204 // computation. We don't have instructions that compute pointers with any
2205 // addressing modes, so treat them as having no offset like flat
2206 // instructions.
2207 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2208 }
2209
2210 // Assume a user alias of global for unknown address spaces.
2211 return isLegalGlobalAddressingMode(AM);
2212}
2213
2214bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
2215 const MachineFunction &MF) const {
2216 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
2217 return (MemVT.getSizeInBits() <= 4 * 32);
2218 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
2219 unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
2220 return (MemVT.getSizeInBits() <= MaxPrivateBits);
2221 }
2222 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
2223 return (MemVT.getSizeInBits() <= 2 * 32);
2224 return true;
2225}
2226
2227bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
2228 unsigned Size, unsigned AddrSpace, Align Alignment,
2229 MachineMemOperand::Flags Flags, unsigned *IsFast) const {
2230 if (IsFast)
2231 *IsFast = 0;
2232
2233 if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
2234 AddrSpace == AMDGPUAS::REGION_ADDRESS) {
2235 // Check if alignment requirements for ds_read/write instructions are
2236 // disabled.
2237 if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
2238 return false;
2239
2240 Align RequiredAlignment(
2241 PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))); // Natural alignment.
2242 if (Subtarget->hasLDSMisalignedBugInWGPMode() && Size > 32 &&
2243 Alignment < RequiredAlignment)
2244 return false;
2245
2246 // Either, the alignment requirements are "enabled", or there is an
2247 // unaligned LDS access related hardware bug though alignment requirements
2248 // are "disabled". In either case, we need to check for proper alignment
2249 // requirements.
2250 //
2251 switch (Size) {
2252 case 64:
2253 // SI has a hardware bug in the LDS / GDS bounds checking: if the base
2254 // address is negative, then the instruction is incorrectly treated as
2255 // out-of-bounds even if base + offsets is in bounds. Split vectorized
2256 // loads here to avoid emitting ds_read2_b32. We may re-combine the
2257 // load later in the SILoadStoreOptimizer.
2258 if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
2259 return false;
2260
2261 // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
2262 // can do a 4 byte aligned, 8 byte access in a single operation using
2263 // ds_read2/write2_b32 with adjacent offsets.
2264 RequiredAlignment = Align(4);
2265
2266 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2267 // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
2268 // ds_write2_b32 depending on the alignment. In either case with either
2269 // alignment there is no faster way of doing this.
2270
2271 // The numbers returned here and below are not additive, it is a 'speed
2272 // rank'. They are just meant to be compared to decide if a certain way
2273 // of lowering an operation is faster than another. For that purpose
2274 // naturally aligned operation gets it bitsize to indicate that "it
2275 // operates with a speed comparable to N-bit wide load". With the full
2276 // alignment ds128 is slower than ds96 for example. If underaligned it
2277 // is comparable to a speed of a single dword access, which would then
2278 // mean 32 < 128 and it is faster to issue a wide load regardless.
2279 // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
2280 // wider load which will not be aligned anymore the latter is slower.
2281 if (IsFast)
2282 *IsFast = (Alignment >= RequiredAlignment) ? 64
2283 : (Alignment < Align(4)) ? 32
2284 : 1;
2285 return true;
2286 }
2287
2288 break;
2289 case 96:
2290 if (!Subtarget->hasDS96AndDS128())
2291 return false;
2292
2293 // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
2294 // gfx8 and older.
2295
2296 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2297 // Naturally aligned access is fastest. However, also report it is Fast
2298 // if memory is aligned less than DWORD. A narrow load or store will be
2299 // be equally slow as a single ds_read_b96/ds_write_b96, but there will
2300 // be more of them, so overall we will pay less penalty issuing a single
2301 // instruction.
2302
2303 // See comment on the values above.
2304 if (IsFast)
2305 *IsFast = (Alignment >= RequiredAlignment) ? 96
2306 : (Alignment < Align(4)) ? 32
2307 : 1;
2308 return true;
2309 }
2310
2311 break;
2312 case 128:
2313 if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
2314 return false;
2315
2316 // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
2317 // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a
2318 // single operation using ds_read2/write2_b64.
2319 RequiredAlignment = Align(8);
2320
2321 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2322 // Naturally aligned access is fastest. However, also report it is Fast
2323 // if memory is aligned less than DWORD. A narrow load or store will be
2324 // be equally slow as a single ds_read_b128/ds_write_b128, but there
2325 // will be more of them, so overall we will pay less penalty issuing a
2326 // single instruction.
2327
2328 // See comment on the values above.
2329 if (IsFast)
2330 *IsFast = (Alignment >= RequiredAlignment) ? 128
2331 : (Alignment < Align(4)) ? 32
2332 : 1;
2333 return true;
2334 }
2335
2336 break;
2337 default:
2338 if (Size > 32)
2339 return false;
2340
2341 break;
2342 }
2343
2344 // See comment on the values above.
2345 // Note that we have a single-dword or sub-dword here, so if underaligned
2346 // it is a slowest possible access, hence returned value is 0.
2347 if (IsFast)
2348 *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
2349
2350 return Alignment >= RequiredAlignment ||
2351 Subtarget->hasUnalignedDSAccessEnabled();
2352 }
2353
2354 // FIXME: We have to be conservative here and assume that flat operations
2355 // will access scratch. If we had access to the IR function, then we
2356 // could determine if any private memory was used in the function.
2357 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
2358 AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
2359 bool AlignedBy4 = Alignment >= Align(4);
2360 if (Subtarget->hasUnalignedScratchAccessEnabled()) {
2361 if (IsFast)
2362 *IsFast = AlignedBy4 ? Size : 1;
2363 return true;
2364 }
2365
2366 if (IsFast)
2367 *IsFast = AlignedBy4;
2368
2369 return AlignedBy4;
2370 }
2371
2372 // So long as they are correct, wide global memory operations perform better
2373 // than multiple smaller memory ops -- even when misaligned
2374 if (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace)) {
2375 if (IsFast)
2376 *IsFast = Size;
2377
2378 return Alignment >= Align(4) ||
2379 Subtarget->hasUnalignedBufferAccessEnabled();
2380 }
2381
2382 // Ensure robust out-of-bounds guarantees for buffer accesses are met when the
2383 // "amdgpu.buffer.oob.mode" module flag has not enabled relaxed untyped-buffer
2384 // OOB semantics. Normally hardware will ensure proper
2385 // out-of-bounds behavior, but in the edge case where an access starts
2386 // out-of-bounds and then enters in-bounds, the entire access would be treated
2387 // as out-of-bounds. Prevent misaligned memory accesses by requiring the
2388 // natural alignment of buffer accesses.
2389 if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
2390 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
2391 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2392 if (!Subtarget->hasRelaxedBufferOOBMode() &&
2393 Alignment < Align(PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))))
2394 return false;
2395 }
2396
2397 // Smaller than dword value must be aligned.
2398 if (Size < 32)
2399 return false;
2400
2401 // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
2402 // byte-address are ignored, thus forcing Dword alignment.
2403 // This applies to private, global, and constant memory.
2404 if (IsFast)
2405 *IsFast = 1;
2406
2407 return Size >= 32 && Alignment >= Align(4);
2408}
2409
2410bool SITargetLowering::allowsMisalignedMemoryAccesses(
2411 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2412 unsigned *IsFast) const {
2413 return allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace,
2414 Alignment, Flags, IsFast);
2415}
2416
2417bool SITargetLowering::allowsMemoryAccess(LLVMContext &Context,
2418 const DataLayout &DL, EVT VT,
2419 unsigned AddrSpace, Align Alignment,
2420 MachineMemOperand::Flags Flags,
2421 unsigned *IsFast) const {
2422 // Sub-dword accesses aren't lowered yet, so combines must not narrow an
2423 // access to one.
2424 if (AddrSpace == AMDGPUAS::VGPR && VT.getSizeInBits() < 32)
2425 return false;
2426 return TargetLowering::allowsMemoryAccess(Context, DL, VT, AddrSpace,
2427 Alignment, Flags, Fast: IsFast);
2428}
2429
2430EVT SITargetLowering::getOptimalMemOpType(
2431 LLVMContext &Context, const MemOp &Op,
2432 const AttributeList &FuncAttributes) const {
2433 // FIXME: Should account for address space here.
2434
2435 // The default fallback uses the private pointer size as a guess for a type to
2436 // use. Make sure we switch these to 64-bit accesses.
2437
2438 if (Op.size() >= 16 &&
2439 Op.isDstAligned(AlignCheck: Align(4))) // XXX: Should only do for global
2440 return MVT::v4i32;
2441
2442 if (Op.size() >= 8 && Op.isDstAligned(AlignCheck: Align(4)))
2443 return MVT::v2i32;
2444
2445 // Use the default.
2446 return MVT::Other;
2447}
2448
2449bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
2450 const MemSDNode *MemNode = cast<MemSDNode>(Val: N);
2451 return MemNode->getMemOperand()->getFlags() & MONoClobber;
2452}
2453
2454bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
2455 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
2456 AS == AMDGPUAS::PRIVATE_ADDRESS;
2457}
2458
2459bool SITargetLowering::isFreeAddrSpaceCast(const DataLayout &DL, unsigned SrcAS,
2460 unsigned DestAS) const {
2461 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
2462 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2463 Subtarget->hasGloballyAddressableScratch()) {
2464 // Flat -> private requires subtracting src_flat_scratch_base_lo.
2465 return false;
2466 }
2467
2468 // Flat -> private/local is a simple truncate.
2469 // Flat -> global is no-op
2470 return true;
2471 }
2472
2473 const GCNTargetMachine &TM =
2474 static_cast<const GCNTargetMachine &>(getTargetMachine());
2475 return TM.isNoopAddrSpaceCast(DL, SrcAS, DestAS);
2476}
2477
2478TargetLoweringBase::LegalizeTypeAction
2479SITargetLowering::getPreferredVectorAction(MVT VT) const {
2480 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2481 VT.getScalarType().bitsLE(VT: MVT::i16))
2482 return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
2483 return TargetLoweringBase::getPreferredVectorAction(VT);
2484}
2485
2486bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2487 Type *Ty) const {
2488 // FIXME: Could be smarter if called for vector constants.
2489 return true;
2490}
2491
2492TargetLowering::ExtractSubvectorCost
2493SITargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2494 unsigned Index) const {
2495 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2496 return ExtractSubvectorCost::Expensive;
2497
2498 // TODO: Add more cases that are cheap.
2499 if (Index == 0)
2500 return ExtractSubvectorCost::Free;
2501 return ExtractSubvectorCost::Expensive;
2502}
2503
2504bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
2505 // TODO: This should be more aggressive, particular for 16-bit element
2506 // vectors. However there are some mixed improvements and regressions.
2507 EVT EltTy = VT.getVectorElementType();
2508 unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
2509 return EltTy.getSizeInBits() % MinAlign == 0;
2510}
2511
2512bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
2513 if (Subtarget->has16BitInsts() && VT == MVT::i16) {
2514 switch (Op) {
2515 case ISD::LOAD:
2516 case ISD::STORE:
2517 return true;
2518 default:
2519 return false;
2520 }
2521 }
2522
2523 // SimplifySetCC uses this function to determine whether or not it should
2524 // create setcc with i1 operands. We don't have instructions for i1 setcc.
2525 if (VT == MVT::i1 && Op == ISD::SETCC)
2526 return false;
2527
2528 if (VT.getScalarType() == MVT::i64 &&
2529 (Op == ISD::SINT_TO_FP || Op == ISD::UINT_TO_FP))
2530 return false;
2531
2532 // The DAGCombiner replaces a conversion from i1 with a select of the two
2533 // exact results. The select avoids an f32 conversion plus rounding for
2534 // f16/bf16, a 64-bit conversion for f64 and a missing libcall for f128, and
2535 // is the same single v_cndmask for f32.
2536 if (VT == MVT::i1 && (Op == ISD::SINT_TO_FP || Op == ISD::UINT_TO_FP))
2537 return false;
2538
2539 return TargetLowering::isTypeDesirableForOp(Op, VT);
2540}
2541
2542bool SITargetLowering::isTypeDesirableForOp(SDNode *N, EVT VT) const {
2543 // Do not convert uniform loads to 16-bit.
2544 // Uniform 16-bit loads are legalized to i16 = trunc (zextload i16->i32)
2545 // to match subword load patterns. Allowing conversion back to a 16-bit
2546 // load would create an infinite loop.
2547 if (Subtarget->hasScalarSubwordLoads() && N->getOpcode() == ISD::LOAD &&
2548 !VT.isVector() && VT.getSizeInBits() == 16) {
2549 auto *Load = dyn_cast<LoadSDNode>(Val: N);
2550 if (Load && isUniformLoad(Load)) {
2551 return false;
2552 }
2553 }
2554
2555 return isTypeDesirableForOp(Op: N->getOpcode(), VT);
2556}
2557
2558bool SITargetLowering::isUniformLoad(const LoadSDNode *Load) const {
2559 const MachineMemOperand *MMO = Load->getMemOperand();
2560
2561 // FIXME: We ought to able able to take the direct isDivergent result. We
2562 // cannot rely on the MMO for a uniformity check, and should stop using
2563 // it. This is a hack for 2 ways that the IR divergence analysis is superior
2564 // to the DAG divergence: Recognizing shift-of-workitem-id as always
2565 // uniform, and isSingleLaneExecution. These should be handled in the DAG
2566 // version, and then this can be dropped.
2567 if (Load->isDivergent() && !AMDGPU::isUniformMMO(MMO))
2568 return false;
2569
2570 return MMO->getSize().hasValue() &&
2571 Load->getAlign() >=
2572 Align(std::min(a: MMO->getSize().getValue().getKnownMinValue(),
2573 b: uint64_t(4))) &&
2574 (MMO->isInvariant() ||
2575 (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2576 Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
2577 (Load->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2578 Load->isSimple() && isMemOpHasNoClobberedMemOperand(N: Load)));
2579}
2580
2581MachinePointerInfo
2582SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
2583 // This isn't really a constant pool but close enough.
2584 MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
2585 PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
2586 return PtrInfo;
2587}
2588
2589SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
2590 const SDLoc &SL,
2591 SDValue Chain,
2592 uint64_t Offset) const {
2593 const DataLayout &DL = DAG.getDataLayout();
2594 MachineFunction &MF = DAG.getMachineFunction();
2595 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2596 MVT PtrVT = getPointerTy(DL, AS: AMDGPUAS::CONSTANT_ADDRESS);
2597
2598 auto [InputPtrReg, RC, ArgTy] =
2599 Info->getPreloadedValue(Value: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
2600
2601 // We may not have the kernarg segment argument if we have no kernel
2602 // arguments.
2603 if (!InputPtrReg)
2604 return DAG.getConstant(Val: Offset, DL: SL, VT: PtrVT);
2605
2606 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
2607 SDValue BasePtr = DAG.getCopyFromReg(
2608 Chain, dl: SL, Reg: MRI.getLiveInVirtReg(PReg: InputPtrReg->getRegister()), VT: PtrVT);
2609
2610 return DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
2611}
2612
2613SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
2614 const SDLoc &SL) const {
2615 uint64_t Offset =
2616 getImplicitParameterOffset(MF: DAG.getMachineFunction(), Param: FIRST_IMPLICIT);
2617 return lowerKernArgParameterPtr(DAG, SL, Chain: DAG.getEntryNode(), Offset);
2618}
2619
2620SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
2621 const SDLoc &SL) const {
2622
2623 Function &F = DAG.getMachineFunction().getFunction();
2624 std::optional<uint32_t> KnownSize =
2625 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
2626 if (KnownSize.has_value())
2627 return DAG.getConstant(Val: *KnownSize, DL: SL, VT: MVT::i32);
2628 return SDValue();
2629}
2630
2631SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
2632 const SDLoc &SL, SDValue Val,
2633 bool Signed,
2634 const ISD::InputArg *Arg) const {
2635 // First, if it is a widened vector, narrow it.
2636 if (VT.isVector() &&
2637 VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
2638 EVT NarrowedVT =
2639 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(),
2640 NumElements: VT.getVectorNumElements());
2641 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: NarrowedVT, N1: Val,
2642 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
2643 }
2644
2645 // Then convert the vector elements or scalar value.
2646 if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && VT.bitsLT(VT: MemVT)) {
2647 unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
2648 Val = DAG.getNode(Opcode: Opc, DL: SL, VT: MemVT, N1: Val, N2: DAG.getValueType(VT));
2649 }
2650
2651 if (MemVT.isFloatingPoint()) {
2652 if (VT.isFloatingPoint()) {
2653 Val = getFPExtOrFPRound(DAG, Op: Val, DL: SL, VT);
2654 } else {
2655 assert(!MemVT.isVector());
2656 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
2657 SDValue Cast = DAG.getBitcast(VT: IntVT, V: Val);
2658 Val = DAG.getAnyExtOrTrunc(Op: Cast, DL: SL, VT);
2659 }
2660 } else if (Signed)
2661 Val = DAG.getSExtOrTrunc(Op: Val, DL: SL, VT);
2662 else
2663 Val = DAG.getZExtOrTrunc(Op: Val, DL: SL, VT);
2664
2665 return Val;
2666}
2667
2668SDValue SITargetLowering::lowerKernargMemParameter(
2669 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2670 uint64_t Offset, Align Alignment, bool Signed,
2671 const ISD::InputArg *Arg) const {
2672
2673 MachinePointerInfo PtrInfo =
2674 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
2675
2676 // Try to avoid using an extload by loading earlier than the argument address,
2677 // and extracting the relevant bits. The load should hopefully be merged with
2678 // the previous argument.
2679 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
2680 // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
2681 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
2682 int64_t OffsetDiff = Offset - AlignDownOffset;
2683
2684 EVT IntVT = MemVT.changeTypeToInteger();
2685
2686 // TODO: If we passed in the base kernel offset we could have a better
2687 // alignment than 4, but we don't really need it.
2688 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset: AlignDownOffset);
2689 SDValue Load = DAG.getLoad(VT: MVT::i32, dl: SL, Chain, Ptr,
2690 PtrInfo: PtrInfo.getWithOffset(O: AlignDownOffset), Alignment: Align(4),
2691 MMOFlags: MachineMemOperand::MODereferenceable |
2692 MachineMemOperand::MOInvariant);
2693
2694 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL: SL, VT: MVT::i32);
2695 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Load, N2: ShiftAmt);
2696
2697 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: IntVT, Operand: Extract);
2698 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MemVT, Operand: ArgVal);
2699 ArgVal = convertArgType(DAG, VT, MemVT, SL, Val: ArgVal, Signed, Arg);
2700
2701 return DAG.getMergeValues(Ops: {ArgVal, Load.getValue(R: 1)}, dl: SL);
2702 }
2703
2704 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2705 SDValue Load = DAG.getLoad(
2706 VT: MemVT, dl: SL, Chain, Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
2707 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2708
2709 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Val: Load, Signed, Arg);
2710 return DAG.getMergeValues(Ops: {Val, Load.getValue(R: 1)}, dl: SL);
2711}
2712
2713/// Coerce an argument which was passed in a different ABI type to the original
2714/// expected value type.
2715SDValue SITargetLowering::convertABITypeToValueType(SelectionDAG &DAG,
2716 SDValue Val,
2717 CCValAssign &VA,
2718 const SDLoc &SL) const {
2719 EVT ValVT = VA.getValVT();
2720
2721 // If this is an 8 or 16-bit value, it is really passed promoted
2722 // to 32 bits. Insert an assert[sz]ext to capture this, then
2723 // truncate to the right size.
2724 switch (VA.getLocInfo()) {
2725 case CCValAssign::Full:
2726 return Val;
2727 case CCValAssign::BCvt:
2728 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ValVT, Operand: Val);
2729 case CCValAssign::SExt:
2730 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: SL, VT: VA.getLocVT(), N1: Val,
2731 N2: DAG.getValueType(ValVT));
2732 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2733 case CCValAssign::ZExt:
2734 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: VA.getLocVT(), N1: Val,
2735 N2: DAG.getValueType(ValVT));
2736 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2737 case CCValAssign::AExt:
2738 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT.changeTypeToInteger(), Operand: Val);
2739 return DAG.getBitcast(VT: ValVT, V: Val);
2740 default:
2741 llvm_unreachable("Unknown loc info!");
2742 }
2743}
2744
2745SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG,
2746 CCValAssign &VA, const SDLoc &SL,
2747 SDValue Chain,
2748 const ISD::InputArg &Arg) const {
2749 MachineFunction &MF = DAG.getMachineFunction();
2750 MachineFrameInfo &MFI = MF.getFrameInfo();
2751
2752 if (Arg.Flags.isByVal()) {
2753 unsigned Size = Arg.Flags.getByValSize();
2754 int FrameIdx = MFI.CreateFixedObject(Size, SPOffset: VA.getLocMemOffset(), IsImmutable: false);
2755 return DAG.getFrameIndex(FI: FrameIdx, VT: MVT::i32);
2756 }
2757
2758 unsigned ArgOffset = VA.getLocMemOffset();
2759 unsigned ArgSize = VA.getValVT().getStoreSize();
2760
2761 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: true);
2762
2763 // Create load nodes to retrieve arguments from the stack.
2764 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
2765 MachinePointerInfo PtrInfo =
2766 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2767 if (VA.getLocInfo() == CCValAssign::AExt)
2768 return DAG.getLoad(VT: VA.getValVT(), dl: SL, Chain, Ptr: FIN, PtrInfo);
2769
2770 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
2771 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
2772 MVT MemVT = VA.getValVT();
2773
2774 switch (VA.getLocInfo()) {
2775 default:
2776 break;
2777 case CCValAssign::BCvt:
2778 MemVT = VA.getLocVT();
2779 break;
2780 case CCValAssign::SExt:
2781 ExtType = ISD::SEXTLOAD;
2782 break;
2783 case CCValAssign::ZExt:
2784 ExtType = ISD::ZEXTLOAD;
2785 break;
2786 }
2787
2788 SDValue ArgValue =
2789 DAG.getExtLoad(ExtType, dl: SL, VT: VA.getLocVT(), Chain, Ptr: FIN, PtrInfo, MemVT);
2790
2791 SDValue ConvertedVal = convertABITypeToValueType(DAG, Val: ArgValue, VA, SL);
2792 if (ConvertedVal == ArgValue)
2793 return ConvertedVal;
2794
2795 return DAG.getMergeValues(Ops: {ConvertedVal, ArgValue.getValue(R: 1)}, dl: SL);
2796}
2797
2798SDValue SITargetLowering::lowerWorkGroupId(
2799 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2800 AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
2801 AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
2802 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
2803 if (!Subtarget->hasClusters())
2804 return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2805
2806 // Clusters are supported. Return the global position in the grid. If clusters
2807 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
2808
2809 // WorkGroupIdXYZ = ClusterId == 0 ?
2810 // ClusterIdXYZ :
2811 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
2812 SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2813 SDLoc SL(ClusterIdXYZ);
2814 SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
2815 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT);
2816 SDValue ClusterSizeXYZ = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterMaxIdXYZ, N2: One);
2817 SDValue ClusterWorkGroupIdXYZ =
2818 getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
2819 SDValue GlobalIdXYZ =
2820 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterWorkGroupIdXYZ,
2821 N2: DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: ClusterIdXYZ, N2: ClusterSizeXYZ));
2822
2823 switch (MFI.getClusterDims().getKind()) {
2824 case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
2825 case AMDGPU::ClusterDimsAttr::Kind::VariableDims:
2826 return GlobalIdXYZ;
2827 case AMDGPU::ClusterDimsAttr::Kind::NoCluster:
2828 return ClusterIdXYZ;
2829 case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
2830 using namespace AMDGPU::Hwreg;
2831 SDValue ClusterIdField =
2832 DAG.getTargetConstant(Val: HwregEncoding::encode(Values: ID_IB_STS2, Values: 6, Values: 4), DL: SL, VT);
2833 SDNode *GetReg =
2834 DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT, Op1: ClusterIdField);
2835 SDValue ClusterId(GetReg, 0);
2836 SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT);
2837 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SL, VT, N1: ClusterId, N2: Zero, N3: ClusterIdXYZ,
2838 N4: GlobalIdXYZ, N5: DAG.getCondCode(Cond: ISD::SETEQ));
2839 }
2840 }
2841
2842 llvm_unreachable("nothing should reach here");
2843}
2844
2845SDValue SITargetLowering::getPreloadedValue(
2846 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2847 AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
2848 const ArgDescriptor *Reg = nullptr;
2849 const TargetRegisterClass *RC = nullptr;
2850 LLT Ty;
2851
2852 CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
2853 const ArgDescriptor WorkGroupIDX =
2854 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP9);
2855 // If GridZ is not programmed in an entry function then the hardware will set
2856 // it to all zeros, so there is no need to mask the GridY value in the low
2857 // order bits.
2858 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
2859 Reg: AMDGPU::TTMP7,
2860 Mask: AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
2861 const ArgDescriptor WorkGroupIDZ =
2862 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP7, Mask: 0xFFFF0000u);
2863 const ArgDescriptor ClusterWorkGroupIDX =
2864 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000000Fu);
2865 const ArgDescriptor ClusterWorkGroupIDY =
2866 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000000F0u);
2867 const ArgDescriptor ClusterWorkGroupIDZ =
2868 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00000F00u);
2869 const ArgDescriptor ClusterWorkGroupMaxIDX =
2870 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000F000u);
2871 const ArgDescriptor ClusterWorkGroupMaxIDY =
2872 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000F0000u);
2873 const ArgDescriptor ClusterWorkGroupMaxIDZ =
2874 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00F00000u);
2875 const ArgDescriptor ClusterWorkGroupMaxFlatID =
2876 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0F000000u);
2877
2878 auto LoadConstant = [&](unsigned N) {
2879 return DAG.getConstant(Val: N, DL: SDLoc(), VT);
2880 };
2881
2882 if (Subtarget->hasArchitectedSGPRs() &&
2883 (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx ||
2884 AMDGPU::isChainCC(CC) || CC == CallingConv::AMDGPU_Gfx_WholeWave)) {
2885 AMDGPU::ClusterDimsAttr ClusterDims = MFI.getClusterDims();
2886 bool HasFixedDims = ClusterDims.isFixedDims();
2887
2888 switch (PVID) {
2889 case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
2890 Reg = &WorkGroupIDX;
2891 RC = &AMDGPU::SReg_32RegClass;
2892 Ty = LLT::scalar(SizeInBits: 32);
2893 break;
2894 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
2895 Reg = &WorkGroupIDY;
2896 RC = &AMDGPU::SReg_32RegClass;
2897 Ty = LLT::scalar(SizeInBits: 32);
2898 break;
2899 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
2900 Reg = &WorkGroupIDZ;
2901 RC = &AMDGPU::SReg_32RegClass;
2902 Ty = LLT::scalar(SizeInBits: 32);
2903 break;
2904 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
2905 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
2906 return LoadConstant(0);
2907 Reg = &ClusterWorkGroupIDX;
2908 RC = &AMDGPU::SReg_32RegClass;
2909 Ty = LLT::scalar(SizeInBits: 32);
2910 break;
2911 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
2912 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
2913 return LoadConstant(0);
2914 Reg = &ClusterWorkGroupIDY;
2915 RC = &AMDGPU::SReg_32RegClass;
2916 Ty = LLT::scalar(SizeInBits: 32);
2917 break;
2918 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
2919 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
2920 return LoadConstant(0);
2921 Reg = &ClusterWorkGroupIDZ;
2922 RC = &AMDGPU::SReg_32RegClass;
2923 Ty = LLT::scalar(SizeInBits: 32);
2924 break;
2925 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
2926 if (HasFixedDims)
2927 return LoadConstant(ClusterDims.getDims()[0] - 1);
2928 Reg = &ClusterWorkGroupMaxIDX;
2929 RC = &AMDGPU::SReg_32RegClass;
2930 Ty = LLT::scalar(SizeInBits: 32);
2931 break;
2932 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
2933 if (HasFixedDims)
2934 return LoadConstant(ClusterDims.getDims()[1] - 1);
2935 Reg = &ClusterWorkGroupMaxIDY;
2936 RC = &AMDGPU::SReg_32RegClass;
2937 Ty = LLT::scalar(SizeInBits: 32);
2938 break;
2939 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
2940 if (HasFixedDims)
2941 return LoadConstant(ClusterDims.getDims()[2] - 1);
2942 Reg = &ClusterWorkGroupMaxIDZ;
2943 RC = &AMDGPU::SReg_32RegClass;
2944 Ty = LLT::scalar(SizeInBits: 32);
2945 break;
2946 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
2947 Reg = &ClusterWorkGroupMaxFlatID;
2948 RC = &AMDGPU::SReg_32RegClass;
2949 Ty = LLT::scalar(SizeInBits: 32);
2950 break;
2951 default:
2952 break;
2953 }
2954 }
2955
2956 if (!Reg)
2957 std::tie(args&: Reg, args&: RC, args&: Ty) = MFI.getPreloadedValue(Value: PVID);
2958 if (!Reg) {
2959 if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
2960 // It's possible for a kernarg intrinsic call to appear in a kernel with
2961 // no allocated segment, in which case we do not add the user sgpr
2962 // argument, so just return null.
2963 return DAG.getConstant(Val: 0, DL: SDLoc(), VT);
2964 }
2965
2966 // It's undefined behavior if a function marked with the amdgpu-no-*
2967 // attributes uses the corresponding intrinsic.
2968 return DAG.getPOISON(VT);
2969 }
2970
2971 return loadInputValue(DAG, RC, VT, SL: SDLoc(DAG.getEntryNode()), Arg: *Reg);
2972}
2973
2974static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
2975 CallingConv::ID CallConv,
2976 ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
2977 FunctionType *FType,
2978 SIMachineFunctionInfo *Info) {
2979 for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
2980 const ISD::InputArg *Arg = &Ins[I];
2981
2982 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2983 "vector type argument should have been split");
2984
2985 // First check if it's a PS input addr.
2986 if (CallConv == CallingConv::AMDGPU_PS && !Arg->Flags.isInReg() &&
2987 PSInputNum <= 15) {
2988 bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(Index: PSInputNum);
2989
2990 // Inconveniently only the first part of the split is marked as isSplit,
2991 // so skip to the end. We only want to increment PSInputNum once for the
2992 // entire split argument.
2993 if (Arg->Flags.isSplit()) {
2994 while (!Arg->Flags.isSplitEnd()) {
2995 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2996 "unexpected vector split in ps argument type");
2997 if (!SkipArg)
2998 Splits.push_back(Elt: *Arg);
2999 Arg = &Ins[++I];
3000 }
3001 }
3002
3003 if (SkipArg) {
3004 // We can safely skip PS inputs.
3005 Skipped.set(Arg->getOrigArgIndex());
3006 ++PSInputNum;
3007 continue;
3008 }
3009
3010 Info->markPSInputAllocated(Index: PSInputNum);
3011 if (Arg->Used)
3012 Info->markPSInputEnabled(Index: PSInputNum);
3013
3014 ++PSInputNum;
3015 }
3016
3017 Splits.push_back(Elt: *Arg);
3018 }
3019}
3020
3021// Allocate special inputs passed in VGPRs.
3022void SITargetLowering::allocateSpecialEntryInputVGPRs(
3023 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3024 SIMachineFunctionInfo &Info) const {
3025 const LLT I32 = LLT::integer(SizeInBits: 32);
3026 MachineRegisterInfo &MRI = MF.getRegInfo();
3027
3028 if (Info.hasWorkItemIDX()) {
3029 Register Reg = AMDGPU::VGPR0;
3030 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
3031
3032 CCInfo.AllocateReg(Reg);
3033 unsigned Mask =
3034 (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
3035 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3036 }
3037
3038 if (Info.hasWorkItemIDY()) {
3039 assert(Info.hasWorkItemIDX());
3040 if (Subtarget->hasPackedTID()) {
3041 Info.setWorkItemIDY(
3042 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 10));
3043 } else {
3044 unsigned Reg = AMDGPU::VGPR1;
3045 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
3046
3047 CCInfo.AllocateReg(Reg);
3048 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
3049 }
3050 }
3051
3052 if (Info.hasWorkItemIDZ()) {
3053 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
3054 if (Subtarget->hasPackedTID()) {
3055 Info.setWorkItemIDZ(
3056 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 20));
3057 } else {
3058 unsigned Reg = AMDGPU::VGPR2;
3059 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: I32);
3060
3061 CCInfo.AllocateReg(Reg);
3062 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
3063 }
3064 }
3065}
3066
3067static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
3068 const TargetRegisterClass *RC,
3069 unsigned NumArgRegs) {
3070 ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
3071 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgSGPRs);
3072 if (RegIdx == ArgSGPRs.size())
3073 report_fatal_error(reason: "ran out of SGPRs for arguments");
3074
3075 unsigned Reg = ArgSGPRs[RegIdx];
3076 Reg = CCInfo.AllocateReg(Reg);
3077 assert(Reg != AMDGPU::NoRegister);
3078
3079 MachineFunction &MF = CCInfo.getMachineFunction();
3080 MF.addLiveIn(PReg: Reg, RC);
3081 return ArgDescriptor::createRegister(Reg);
3082}
3083
3084// If this has a fixed position, we still should allocate the register in the
3085// CCInfo state. Technically we could get away with this for values passed
3086// outside of the normal argument range.
3087static void allocateFixedSGPRInputImpl(CCState &CCInfo,
3088 const TargetRegisterClass *RC,
3089 MCRegister Reg) {
3090 Reg = CCInfo.AllocateReg(Reg);
3091 assert(Reg != AMDGPU::NoRegister);
3092 MachineFunction &MF = CCInfo.getMachineFunction();
3093 MF.addLiveIn(PReg: Reg, RC);
3094}
3095
3096static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
3097 if (Arg) {
3098 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass,
3099 Reg: Arg.getRegister());
3100 } else
3101 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass, NumArgRegs: 32);
3102}
3103
3104static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
3105 if (Arg) {
3106 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass,
3107 Reg: Arg.getRegister());
3108 } else
3109 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass, NumArgRegs: 16);
3110}
3111
3112/// Allocate implicit function VGPR arguments in fixed registers.
3113void SITargetLowering::allocateSpecialInputVGPRsFixed(
3114 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3115 SIMachineFunctionInfo &Info) const {
3116 Register Reg = CCInfo.AllocateReg(Reg: AMDGPU::VGPR31);
3117 if (!Reg)
3118 report_fatal_error(reason: "failed to allocate VGPR for implicit arguments");
3119
3120 const unsigned Mask = 0x3ff;
3121 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3122 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask: Mask << 10));
3123 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask: Mask << 20));
3124}
3125
3126void SITargetLowering::allocateSpecialInputSGPRs(
3127 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3128 SIMachineFunctionInfo &Info) const {
3129 auto &ArgInfo = Info.getArgInfo();
3130 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3131
3132 // TODO: Unify handling with private memory pointers.
3133 if (UserSGPRInfo.hasDispatchPtr())
3134 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchPtr);
3135
3136 if (UserSGPRInfo.hasQueuePtr())
3137 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.QueuePtr);
3138
3139 // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
3140 // constant offset from the kernarg segment.
3141 if (Info.hasImplicitArgPtr())
3142 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.ImplicitArgPtr);
3143
3144 if (UserSGPRInfo.hasDispatchID())
3145 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchID);
3146
3147 // flat_scratch_init is not applicable for non-kernel functions.
3148
3149 if (Info.hasWorkGroupIDX())
3150 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDX);
3151
3152 if (Info.hasWorkGroupIDY())
3153 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDY);
3154
3155 if (Info.hasWorkGroupIDZ())
3156 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDZ);
3157
3158 if (Info.hasLDSKernelId())
3159 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.LDSKernelId);
3160}
3161
3162// Allocate special inputs passed in user SGPRs.
3163void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
3164 MachineFunction &MF,
3165 const SIRegisterInfo &TRI,
3166 SIMachineFunctionInfo &Info) const {
3167 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3168 if (UserSGPRInfo.hasImplicitBufferPtr()) {
3169 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
3170 MF.addLiveIn(PReg: ImplicitBufferPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3171 CCInfo.AllocateReg(Reg: ImplicitBufferPtrReg);
3172 }
3173
3174 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
3175 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
3176 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
3177 MF.addLiveIn(PReg: PrivateSegmentBufferReg, RC: &AMDGPU::SGPR_128RegClass);
3178 CCInfo.AllocateReg(Reg: PrivateSegmentBufferReg);
3179 }
3180
3181 if (UserSGPRInfo.hasDispatchPtr()) {
3182 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
3183 MF.addLiveIn(PReg: DispatchPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3184 CCInfo.AllocateReg(Reg: DispatchPtrReg);
3185 }
3186
3187 if (UserSGPRInfo.hasQueuePtr()) {
3188 Register QueuePtrReg = Info.addQueuePtr(TRI);
3189 MF.addLiveIn(PReg: QueuePtrReg, RC: &AMDGPU::SGPR_64RegClass);
3190 CCInfo.AllocateReg(Reg: QueuePtrReg);
3191 }
3192
3193 if (UserSGPRInfo.hasKernargSegmentPtr()) {
3194 MachineRegisterInfo &MRI = MF.getRegInfo();
3195 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
3196 CCInfo.AllocateReg(Reg: InputPtrReg);
3197
3198 Register VReg = MF.addLiveIn(PReg: InputPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3199 MRI.setType(VReg, Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
3200 }
3201
3202 if (UserSGPRInfo.hasDispatchID()) {
3203 Register DispatchIDReg = Info.addDispatchID(TRI);
3204 MF.addLiveIn(PReg: DispatchIDReg, RC: &AMDGPU::SGPR_64RegClass);
3205 CCInfo.AllocateReg(Reg: DispatchIDReg);
3206 }
3207
3208 if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
3209 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
3210 MF.addLiveIn(PReg: FlatScratchInitReg, RC: &AMDGPU::SGPR_64RegClass);
3211 CCInfo.AllocateReg(Reg: FlatScratchInitReg);
3212 }
3213
3214 if (UserSGPRInfo.hasPrivateSegmentSize()) {
3215 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
3216 MF.addLiveIn(PReg: PrivateSegmentSizeReg, RC: &AMDGPU::SGPR_32RegClass);
3217 CCInfo.AllocateReg(Reg: PrivateSegmentSizeReg);
3218 }
3219
3220 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
3221 // these from the dispatch pointer.
3222}
3223
3224// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
3225// sequential starting from the first argument.
3226void SITargetLowering::allocatePreloadKernArgSGPRs(
3227 CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
3228 const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
3229 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
3230 Function &F = MF.getFunction();
3231 unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
3232 GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
3233 bool InPreloadSequence = true;
3234 unsigned InIdx = 0;
3235 bool AlignedForImplictArgs = false;
3236 unsigned ImplicitArgOffset = 0;
3237 for (auto &Arg : F.args()) {
3238 if (!InPreloadSequence || !Arg.hasInRegAttr())
3239 break;
3240
3241 unsigned ArgIdx = Arg.getArgNo();
3242 // Don't preload non-original args or parts not in the current preload
3243 // sequence.
3244 if (InIdx < Ins.size() &&
3245 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
3246 break;
3247
3248 for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
3249 Ins[InIdx].getOrigArgIndex() == ArgIdx;
3250 InIdx++) {
3251 assert(ArgLocs[ArgIdx].isMemLoc());
3252 auto &ArgLoc = ArgLocs[InIdx];
3253 const Align KernelArgBaseAlign = Align(16);
3254 unsigned ArgOffset = ArgLoc.getLocMemOffset();
3255 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset: ArgOffset);
3256 unsigned NumAllocSGPRs =
3257 alignTo(Value: ArgLoc.getLocVT().getFixedSizeInBits(), Align: 32) / 32;
3258
3259 // Fix alignment for hidden arguments.
3260 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument")) {
3261 if (!AlignedForImplictArgs) {
3262 ImplicitArgOffset =
3263 alignTo(Size: LastExplicitArgOffset,
3264 A: Subtarget->getAlignmentForImplicitArgPtr()) -
3265 LastExplicitArgOffset;
3266 AlignedForImplictArgs = true;
3267 }
3268 ArgOffset += ImplicitArgOffset;
3269 }
3270
3271 // Arg is preloaded into the previous SGPR.
3272 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
3273 assert(InIdx >= 1 && "No previous SGPR");
3274 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
3275 Elt: Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
3276 continue;
3277 }
3278
3279 unsigned Padding = ArgOffset - LastExplicitArgOffset;
3280 unsigned PaddingSGPRs = alignTo(Value: Padding, Align: 4) / 4;
3281 // Check for free user SGPRs for preloading.
3282 if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
3283 InPreloadSequence = false;
3284 break;
3285 }
3286
3287 // Preload this argument.
3288 const TargetRegisterClass *RC =
3289 TRI.getSGPRClassForBitWidth(BitWidth: NumAllocSGPRs * 32);
3290 SmallVectorImpl<MCRegister> *PreloadRegs =
3291 Info.addPreloadedKernArg(TRI, RC, AllocSizeDWord: NumAllocSGPRs, KernArgIdx: InIdx, PaddingSGPRs);
3292
3293 if (PreloadRegs->size() > 1)
3294 RC = &AMDGPU::SGPR_32RegClass;
3295 for (auto &Reg : *PreloadRegs) {
3296 assert(Reg);
3297 MF.addLiveIn(PReg: Reg, RC);
3298 CCInfo.AllocateReg(Reg);
3299 }
3300
3301 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
3302 }
3303 }
3304}
3305
3306void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
3307 const SIRegisterInfo &TRI,
3308 SIMachineFunctionInfo &Info) const {
3309 // Always allocate this last since it is a synthetic preload.
3310 if (Info.hasLDSKernelId()) {
3311 Register Reg = Info.addLDSKernelId();
3312 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3313 CCInfo.AllocateReg(Reg);
3314 }
3315}
3316
3317// Allocate special input registers that are initialized per-wave.
3318void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF,
3319 SIMachineFunctionInfo &Info,
3320 CallingConv::ID CallConv,
3321 bool IsShader) const {
3322 bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
3323 if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
3324 // Note: user SGPRs are handled by the front-end for graphics shaders
3325 // Pad up the used user SGPRs with dead inputs.
3326
3327 // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
3328 // before enabling architected SGPRs for workgroup IDs.
3329 assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
3330
3331 unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
3332 // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
3333 // rely on it to reach 16 since if we end up having no stack usage, it will
3334 // not really be added.
3335 unsigned NumRequiredSystemSGPRs =
3336 Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
3337 Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
3338 for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
3339 Register Reg = Info.addReservedUserSGPR();
3340 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3341 CCInfo.AllocateReg(Reg);
3342 }
3343 }
3344
3345 if (!HasArchitectedSGPRs) {
3346 if (Info.hasWorkGroupIDX()) {
3347 Register Reg = Info.addWorkGroupIDX();
3348 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3349 CCInfo.AllocateReg(Reg);
3350 }
3351
3352 if (Info.hasWorkGroupIDY()) {
3353 Register Reg = Info.addWorkGroupIDY();
3354 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3355 CCInfo.AllocateReg(Reg);
3356 }
3357
3358 if (Info.hasWorkGroupIDZ()) {
3359 Register Reg = Info.addWorkGroupIDZ();
3360 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3361 CCInfo.AllocateReg(Reg);
3362 }
3363 }
3364
3365 if (Info.hasWorkGroupInfo()) {
3366 Register Reg = Info.addWorkGroupInfo();
3367 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3368 CCInfo.AllocateReg(Reg);
3369 }
3370
3371 if (Info.hasPrivateSegmentWaveByteOffset()) {
3372 // Scratch wave offset passed in system SGPR.
3373 unsigned PrivateSegmentWaveByteOffsetReg;
3374
3375 if (IsShader) {
3376 PrivateSegmentWaveByteOffsetReg =
3377 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
3378
3379 // This is true if the scratch wave byte offset doesn't have a fixed
3380 // location.
3381 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
3382 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
3383 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
3384 }
3385 } else
3386 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
3387
3388 MF.addLiveIn(PReg: PrivateSegmentWaveByteOffsetReg, RC: &AMDGPU::SGPR_32RegClass);
3389 CCInfo.AllocateReg(Reg: PrivateSegmentWaveByteOffsetReg);
3390 }
3391
3392 assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
3393 Info.getNumPreloadedSGPRs() >= 16);
3394}
3395
3396static void reservePrivateMemoryRegs(const TargetMachine &TM,
3397 MachineFunction &MF,
3398 const SIRegisterInfo &TRI,
3399 SIMachineFunctionInfo &Info) {
3400 // Now that we've figured out where the scratch register inputs are, see if
3401 // should reserve the arguments and use them directly.
3402 MachineFrameInfo &MFI = MF.getFrameInfo();
3403 bool HasStackObjects = MFI.hasStackObjects();
3404 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3405
3406 // Record that we know we have non-spill stack objects so we don't need to
3407 // check all stack objects later.
3408 if (HasStackObjects)
3409 Info.setHasNonSpillStackObjects(true);
3410
3411 // Everything live out of a block is spilled with fast regalloc, so it's
3412 // almost certain that spilling will be required.
3413 if (TM.getOptLevel() == CodeGenOptLevel::None)
3414 HasStackObjects = true;
3415
3416 // For now assume stack access is needed in any callee functions, so we need
3417 // the scratch registers to pass in.
3418 bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
3419
3420 if (!ST.hasFlatScratchEnabled()) {
3421 if (RequiresStackAccess && ST.isAmdHsaOrMesa(F: MF.getFunction())) {
3422 // If we have stack objects, we unquestionably need the private buffer
3423 // resource. For the Code Object V2 ABI, this will be the first 4 user
3424 // SGPR inputs. We can reserve those and use them directly.
3425
3426 Register PrivateSegmentBufferReg =
3427 Info.getPreloadedReg(Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
3428 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
3429 } else {
3430 unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
3431 // We tentatively reserve the last registers (skipping the last registers
3432 // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
3433 // we'll replace these with the ones immediately after those which were
3434 // really allocated. In the prologue copies will be inserted from the
3435 // argument to these reserved registers.
3436
3437 // Without HSA, relocations are used for the scratch pointer and the
3438 // buffer resource setup is always inserted in the prologue. Scratch wave
3439 // offset is still in an input SGPR.
3440 Info.setScratchRSrcReg(ReservedBufferReg);
3441 }
3442 }
3443
3444 MachineRegisterInfo &MRI = MF.getRegInfo();
3445
3446 // For entry functions we have to set up the stack pointer if we use it,
3447 // whereas non-entry functions get this "for free". This means there is no
3448 // intrinsic advantage to using S32 over S34 in cases where we do not have
3449 // calls but do need a frame pointer (i.e. if we are requested to have one
3450 // because frame pointer elimination is disabled). To keep things simple we
3451 // only ever use S32 as the call ABI stack pointer, and so using it does not
3452 // imply we need a separate frame pointer.
3453 //
3454 // Try to use s32 as the SP, but move it if it would interfere with input
3455 // arguments. This won't work with calls though.
3456 //
3457 // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
3458 // registers.
3459 if (!MRI.isLiveIn(Reg: AMDGPU::SGPR32)) {
3460 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
3461 } else {
3462 assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
3463
3464 if (MFI.hasCalls())
3465 report_fatal_error(reason: "call in graphics shader with too many input SGPRs");
3466
3467 for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
3468 if (!MRI.isLiveIn(Reg)) {
3469 Info.setStackPtrOffsetReg(Reg);
3470 break;
3471 }
3472 }
3473
3474 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
3475 report_fatal_error(reason: "failed to find register for SP");
3476 }
3477
3478 // hasFP should be accurate for entry functions even before the frame is
3479 // finalized, because it does not rely on the known stack size, only
3480 // properties like whether variable sized objects are present.
3481 if (ST.getFrameLowering()->hasFP(MF)) {
3482 Info.setFrameOffsetReg(AMDGPU::SGPR33);
3483 }
3484}
3485
3486bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
3487 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
3488 return !Info->isEntryFunction();
3489}
3490
3491void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {}
3492
3493void SITargetLowering::insertCopiesSplitCSR(
3494 MachineBasicBlock *Entry,
3495 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
3496 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3497
3498 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
3499 if (!IStart)
3500 return;
3501
3502 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3503 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
3504 MachineBasicBlock::iterator MBBI = Entry->begin();
3505 for (const MCPhysReg *I = IStart; *I; ++I) {
3506 const TargetRegisterClass *RC = nullptr;
3507 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3508 RC = &AMDGPU::SGPR_64RegClass;
3509 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3510 RC = &AMDGPU::SGPR_32RegClass;
3511 else
3512 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3513
3514 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
3515 // Create copy from CSR to a virtual register.
3516 Entry->addLiveIn(PhysReg: *I);
3517 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
3518 .addReg(RegNo: *I);
3519
3520 // Insert the copy-back instructions right before the terminator.
3521 for (auto *Exit : Exits)
3522 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
3523 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
3524 .addReg(RegNo: NewVR);
3525 }
3526}
3527
3528SDValue SITargetLowering::LowerFormalArguments(
3529 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3530 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3531 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3532 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3533
3534 MachineFunction &MF = DAG.getMachineFunction();
3535 const Function &Fn = MF.getFunction();
3536 FunctionType *FType = MF.getFunction().getFunctionType();
3537 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3538 bool IsError = false;
3539
3540 if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CC: CallConv)) {
3541 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3542 Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()));
3543 IsError = true;
3544 }
3545
3546 SmallVector<ISD::InputArg, 16> Splits;
3547 SmallVector<CCValAssign, 16> ArgLocs;
3548 BitVector Skipped(Fn.arg_size());
3549 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3550 *DAG.getContext());
3551
3552 bool IsGraphics = AMDGPU::isGraphics(CC: CallConv);
3553 bool IsKernel = AMDGPU::isKernel(CC: CallConv);
3554 bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC: CallConv);
3555
3556 if (IsGraphics) {
3557 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
3558 assert(!UserSGPRInfo.hasDispatchPtr() &&
3559 !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
3560 !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
3561 !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
3562 (void)UserSGPRInfo;
3563 if (!Subtarget->hasFlatScratchEnabled())
3564 assert(!UserSGPRInfo.hasFlatScratchInit());
3565 if ((CallConv != CallingConv::AMDGPU_CS &&
3566 CallConv != CallingConv::AMDGPU_Gfx &&
3567 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) ||
3568 !Subtarget->hasArchitectedSGPRs())
3569 assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
3570 !Info->hasWorkGroupIDZ());
3571 }
3572
3573 bool IsWholeWaveFunc = Info->isWholeWaveFunction();
3574
3575 if (CallConv == CallingConv::AMDGPU_PS) {
3576 processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
3577
3578 // At least one interpolation mode must be enabled or else the GPU will
3579 // hang.
3580 //
3581 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
3582 // set PSInputAddr, the user wants to enable some bits after the compilation
3583 // based on run-time states. Since we can't know what the final PSInputEna
3584 // will look like, so we shouldn't do anything here and the user should take
3585 // responsibility for the correct programming.
3586 //
3587 // Otherwise, the following restrictions apply:
3588 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
3589 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
3590 // enabled too.
3591 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
3592 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(Index: 11))) {
3593 CCInfo.AllocateReg(Reg: AMDGPU::VGPR0);
3594 CCInfo.AllocateReg(Reg: AMDGPU::VGPR1);
3595 Info->markPSInputAllocated(Index: 0);
3596 Info->markPSInputEnabled(Index: 0);
3597 }
3598 if (Subtarget->isAmdPalOS()) {
3599 // For isAmdPalOS, the user does not enable some bits after compilation
3600 // based on run-time states; the register values being generated here are
3601 // the final ones set in hardware. Therefore we need to apply the
3602 // workaround to PSInputAddr and PSInputEnable together. (The case where
3603 // a bit is set in PSInputAddr but not PSInputEnable is where the
3604 // frontend set up an input arg for a particular interpolation mode, but
3605 // nothing uses that input arg. Really we should have an earlier pass
3606 // that removes such an arg.)
3607 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
3608 if ((PsInputBits & 0x7F) == 0 ||
3609 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
3610 Info->markPSInputEnabled(Index: llvm::countr_zero(Val: Info->getPSInputAddr()));
3611 }
3612 } else if (IsKernel) {
3613 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
3614 } else {
3615 Splits.append(in_start: IsWholeWaveFunc ? std::next(x: Ins.begin()) : Ins.begin(),
3616 in_end: Ins.end());
3617 }
3618
3619 if (IsKernel)
3620 analyzeFormalArgumentsCompute(State&: CCInfo, Ins);
3621
3622 if (IsEntryFunc) {
3623 allocateSpecialEntryInputVGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3624 allocateHSAUserSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3625 if (IsKernel && Subtarget->hasKernargPreload())
3626 allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, TRI: *TRI, Info&: *Info);
3627
3628 allocateLDSKernelId(CCInfo, MF, TRI: *TRI, Info&: *Info);
3629 } else if (!IsGraphics) {
3630 // For the fixed ABI, pass workitem IDs in the last argument register.
3631 allocateSpecialInputVGPRsFixed(CCInfo, MF, TRI: *TRI, Info&: *Info);
3632
3633 // FIXME: Sink this into allocateSpecialInputSGPRs
3634 if (!Subtarget->hasFlatScratchEnabled())
3635 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
3636
3637 allocateSpecialInputSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3638 }
3639
3640 if (!IsKernel) {
3641 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg: isVarArg);
3642 CCInfo.AnalyzeFormalArguments(Ins: Splits, Fn: AssignFn);
3643
3644 // This assumes the registers are allocated by CCInfo in ascending order
3645 // with no gaps.
3646 Info->setNumWaveDispatchSGPRs(
3647 CCInfo.getFirstUnallocated(Regs: AMDGPU::SGPR_32RegClass.getRegisters()));
3648 Info->setNumWaveDispatchVGPRs(
3649 CCInfo.getFirstUnallocated(Regs: AMDGPU::VGPR_32RegClass.getRegisters()));
3650 } else if (Info->getNumKernargPreloadedSGPRs()) {
3651 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
3652 }
3653
3654 SmallVector<SDValue, 16> Chains;
3655
3656 if (IsWholeWaveFunc) {
3657 SDValue Setup = DAG.getNode(Opcode: AMDGPUISD::WHOLE_WAVE_SETUP, DL,
3658 ResultTys: {MVT::i1, MVT::Other}, Ops: Chain);
3659 InVals.push_back(Elt: Setup.getValue(R: 0));
3660 Chains.push_back(Elt: Setup.getValue(R: 1));
3661 }
3662
3663 // FIXME: This is the minimum kernel argument alignment. We should improve
3664 // this to the maximum alignment of the arguments.
3665 //
3666 // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
3667 // kern arg offset.
3668 const Align KernelArgBaseAlign = Align(16);
3669
3670 for (unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.size(), ArgIdx = 0; i != e;
3671 ++i) {
3672 const ISD::InputArg &Arg = Ins[i];
3673 if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) {
3674 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
3675 continue;
3676 }
3677
3678 CCValAssign &VA = ArgLocs[ArgIdx++];
3679 MVT VT = VA.getLocVT();
3680
3681 if (IsEntryFunc && VA.isMemLoc()) {
3682 VT = Ins[i].VT;
3683 EVT MemVT = VA.getLocVT();
3684
3685 const uint64_t Offset = VA.getLocMemOffset();
3686 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset);
3687
3688 if (Arg.Flags.isByRef()) {
3689 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain, Offset);
3690
3691 const GCNTargetMachine &TM =
3692 static_cast<const GCNTargetMachine &>(getTargetMachine());
3693 if (!TM.isNoopAddrSpaceCast(DL: DAG.getDataLayout(),
3694 SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3695 DestAS: Arg.Flags.getPointerAddrSpace())) {
3696 Ptr = DAG.getAddrSpaceCast(dl: DL, VT, Ptr, SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3697 DestAS: Arg.Flags.getPointerAddrSpace());
3698 }
3699
3700 InVals.push_back(Elt: Ptr);
3701 continue;
3702 }
3703
3704 SDValue NewArg;
3705 if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(Val: i)) {
3706 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
3707 // In this case the argument is packed into the previous preload SGPR.
3708 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
3709 int64_t OffsetDiff = Offset - AlignDownOffset;
3710 EVT IntVT = MemVT.changeTypeToInteger();
3711
3712 const SIMachineFunctionInfo *Info =
3713 MF.getInfo<SIMachineFunctionInfo>();
3714 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3715 Register Reg =
3716 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs[0];
3717
3718 assert(Reg);
3719 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3720 SDValue Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3721
3722 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL, VT: MVT::i32);
3723 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Copy, N2: ShiftAmt);
3724
3725 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Extract);
3726 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MemVT, Operand: ArgVal);
3727 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: ArgVal,
3728 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3729
3730 NewArg = DAG.getMergeValues(Ops: {NewArg, Copy.getValue(R: 1)}, dl: DL);
3731 } else {
3732 const SIMachineFunctionInfo *Info =
3733 MF.getInfo<SIMachineFunctionInfo>();
3734 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3735 const SmallVectorImpl<MCRegister> &PreloadRegs =
3736 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs;
3737
3738 SDValue Copy;
3739 if (PreloadRegs.size() == 1) {
3740 Register VReg = MRI.getLiveInVirtReg(PReg: PreloadRegs[0]);
3741 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VReg);
3742 NewArg = DAG.getCopyFromReg(
3743 Chain, dl: DL, Reg: VReg,
3744 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
3745 BitWidth: TRI->getRegSizeInBits(RC: *RC)));
3746
3747 } else {
3748 // If the kernarg alignment does not match the alignment of the SGPR
3749 // tuple RC that can accommodate this argument, it will be built up
3750 // via copies from from the individual SGPRs that the argument was
3751 // preloaded to.
3752 SmallVector<SDValue, 4> Elts;
3753 for (auto Reg : PreloadRegs) {
3754 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3755 Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3756 Elts.push_back(Elt: Copy);
3757 }
3758 NewArg =
3759 DAG.getBuildVector(VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
3760 NumElements: PreloadRegs.size()),
3761 DL, Ops: Elts);
3762 }
3763
3764 // If the argument was preloaded to multiple consecutive 32-bit
3765 // registers because of misalignment between addressable SGPR tuples
3766 // and the argument size, we can still assume that because of kernarg
3767 // segment alignment restrictions that NewArg's size is the same as
3768 // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
3769 // truncate since we cannot preload to less than a single SGPR and the
3770 // MemVT may be smaller.
3771 EVT MemVTInt =
3772 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
3773 if (MemVT.bitsLT(VT: NewArg.getSimpleValueType()))
3774 NewArg = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVTInt, Operand: NewArg);
3775
3776 NewArg = DAG.getBitcast(VT: MemVT, V: NewArg);
3777 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: NewArg,
3778 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3779 NewArg = DAG.getMergeValues(Ops: {NewArg, Chain}, dl: DL);
3780 }
3781 } else {
3782 // Hidden arguments that are in the kernel signature must be preloaded
3783 // to user SGPRs. Print a diagnostic error if a hidden argument is in
3784 // the argument list and is not preloaded.
3785 if (Arg.isOrigArg()) {
3786 Argument *OrigArg = Fn.getArg(i: Arg.getOrigArgIndex());
3787 if (OrigArg->hasAttribute(Kind: "amdgpu-hidden-argument")) {
3788 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3789 *OrigArg->getParent(),
3790 "hidden argument in kernel signature was not preloaded",
3791 DL.getDebugLoc()));
3792 }
3793 }
3794
3795 NewArg =
3796 lowerKernargMemParameter(DAG, VT, MemVT, SL: DL, Chain, Offset,
3797 Alignment, Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3798 }
3799 Chains.push_back(Elt: NewArg.getValue(R: 1));
3800
3801 auto *ParamTy =
3802 dyn_cast<PointerType>(Val: FType->getParamType(i: Ins[i].getOrigArgIndex()));
3803 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
3804 ParamTy &&
3805 (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
3806 ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
3807 // On SI local pointers are just offsets into LDS, so they are always
3808 // less than 16-bits. On CI and newer they could potentially be
3809 // real pointers, so we can't guarantee their size.
3810 NewArg = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: NewArg.getValueType(), N1: NewArg,
3811 N2: DAG.getValueType(MVT::i16));
3812 }
3813
3814 InVals.push_back(Elt: NewArg);
3815 continue;
3816 }
3817 if (!IsEntryFunc && VA.isMemLoc()) {
3818 SDValue Val = lowerStackParameter(DAG, VA, SL: DL, Chain, Arg);
3819 InVals.push_back(Elt: Val);
3820 if (!Arg.Flags.isByVal())
3821 Chains.push_back(Elt: Val.getValue(R: 1));
3822 continue;
3823 }
3824
3825 assert(VA.isRegLoc() && "Parameter must be in a register!");
3826
3827 Register Reg = VA.getLocReg();
3828 const TargetRegisterClass *RC = nullptr;
3829 if (AMDGPU::VGPR_32RegClass.contains(Reg))
3830 RC = &AMDGPU::VGPR_32RegClass;
3831 else if (AMDGPU::SGPR_32RegClass.contains(Reg))
3832 RC = &AMDGPU::SGPR_32RegClass;
3833 else
3834 llvm_unreachable("Unexpected register class in LowerFormalArguments!");
3835
3836 Reg = MF.addLiveIn(PReg: Reg, RC);
3837 SDValue Val = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT);
3838 if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
3839 // FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
3840 // they will read physical regs before any side effect instructions.
3841 SDValue ReadFirstLane =
3842 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3843 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
3844 N1: ReadFirstLane, N2: Val);
3845 }
3846
3847 if (Arg.Flags.isSRet()) {
3848 // The return object should be reasonably addressable.
3849 Val = annotateStackObjectPointer(Ptr: Val, DAG, DL,
3850 Alignment: Arg.Flags.getNonZeroMemAlign());
3851 }
3852
3853 Val = convertABITypeToValueType(DAG, Val, VA, SL: DL);
3854 InVals.push_back(Elt: Val);
3855 }
3856
3857 // Start adding system SGPRs.
3858 if (IsEntryFunc)
3859 allocateSystemSGPRs(CCInfo, MF, Info&: *Info, CallConv, IsShader: IsGraphics);
3860
3861 unsigned StackArgSize = CCInfo.getStackSize();
3862 Info->setBytesInStackArgArea(StackArgSize);
3863
3864 return Chains.empty() ? Chain
3865 : DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3866}
3867
3868// TODO: If return values can't fit in registers, we should return as many as
3869// possible in registers before passing on stack.
3870bool SITargetLowering::CanLowerReturn(
3871 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
3872 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
3873 const Type *RetTy) const {
3874 // Replacing returns with sret/stack usage doesn't make sense for shaders.
3875 // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
3876 // for shaders. Vector types should be explicitly handled by CC.
3877 if (AMDGPU::isEntryFunctionCC(CC: CallConv))
3878 return true;
3879
3880 SmallVector<CCValAssign, 16> RVLocs;
3881 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3882 if (!CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg)))
3883 return false;
3884
3885 // We must use the stack if return would require unavailable registers.
3886 unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
3887 unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
3888 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3889 if (CCInfo.isAllocated(Reg: AMDGPU::VGPR_32RegClass.getRegister(i)))
3890 return false;
3891
3892 return true;
3893}
3894
3895SDValue
3896SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3897 bool isVarArg,
3898 const SmallVectorImpl<ISD::OutputArg> &Outs,
3899 const SmallVectorImpl<SDValue> &OutVals,
3900 const SDLoc &DL, SelectionDAG &DAG) const {
3901 MachineFunction &MF = DAG.getMachineFunction();
3902 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3903 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3904
3905 if (AMDGPU::isKernel(CC: CallConv)) {
3906 return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
3907 OutVals, DL, DAG);
3908 }
3909
3910 bool IsShader = AMDGPU::isShader(CC: CallConv);
3911
3912 Info->setIfReturnsVoid(Outs.empty());
3913 bool IsWaveEnd = Info->returnsVoid() && IsShader;
3914
3915 // CCValAssign - represent the assignment of the return value to a location.
3916 SmallVector<CCValAssign, 48> RVLocs;
3917
3918 // CCState - Info about the registers and stack slots.
3919 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3920 *DAG.getContext());
3921
3922 // Analyze outgoing return values.
3923 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg: isVarArg));
3924
3925 SDValue Glue;
3926 SmallVector<SDValue, 48> RetOps;
3927 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
3928
3929 SDValue ReadFirstLane =
3930 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3931 // Copy the result values into the output registers.
3932 for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
3933 ++I, ++RealRVLocIdx) {
3934 CCValAssign &VA = RVLocs[I];
3935 assert(VA.isRegLoc() && "Can only return in registers!");
3936 // TODO: Partially return in registers if return values don't fit.
3937 SDValue Arg = OutVals[RealRVLocIdx];
3938
3939 // Copied from other backends.
3940 switch (VA.getLocInfo()) {
3941 case CCValAssign::Full:
3942 break;
3943 case CCValAssign::BCvt:
3944 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
3945 break;
3946 case CCValAssign::SExt:
3947 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3948 break;
3949 case CCValAssign::ZExt:
3950 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3951 break;
3952 case CCValAssign::AExt:
3953 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3954 break;
3955 default:
3956 llvm_unreachable("Unknown loc info!");
3957 }
3958 if (TRI->isSGPRPhysReg(Reg: VA.getLocReg()))
3959 Arg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Arg.getValueType(),
3960 N1: ReadFirstLane, N2: Arg);
3961 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Arg, Glue);
3962 Glue = Chain.getValue(R: 1);
3963 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3964 }
3965
3966 // FIXME: Does sret work properly?
3967 if (!Info->isEntryFunction()) {
3968 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3969 const MCPhysReg *I =
3970 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3971 if (I) {
3972 for (; *I; ++I) {
3973 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3974 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i64));
3975 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3976 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3977 else
3978 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3979 }
3980 }
3981 }
3982
3983 // Update chain and glue.
3984 RetOps[0] = Chain;
3985 if (Glue.getNode())
3986 RetOps.push_back(Elt: Glue);
3987
3988 unsigned Opc = AMDGPUISD::ENDPGM;
3989 if (!IsWaveEnd)
3990 Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
3991 : IsShader ? AMDGPUISD::RETURN_TO_EPILOG
3992 : AMDGPUISD::RET_GLUE;
3993 return DAG.getNode(Opcode: Opc, DL, VT: MVT::Other, Ops: RetOps);
3994}
3995
3996SDValue SITargetLowering::LowerCallResult(
3997 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
3998 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3999 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
4000 SDValue ThisVal) const {
4001 CCAssignFn *RetCC = CCAssignFnForReturn(CC: CallConv, IsVarArg);
4002
4003 // Assign locations to each value returned by this call.
4004 SmallVector<CCValAssign, 16> RVLocs;
4005 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
4006 *DAG.getContext());
4007 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC);
4008
4009 // Copy all of the result registers out of their specified physreg.
4010 for (CCValAssign VA : RVLocs) {
4011 SDValue Val;
4012
4013 if (VA.isRegLoc()) {
4014 Val =
4015 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
4016 Chain = Val.getValue(R: 1);
4017 InGlue = Val.getValue(R: 2);
4018 } else if (VA.isMemLoc()) {
4019 report_fatal_error(reason: "TODO: return values in memory");
4020 } else
4021 llvm_unreachable("unknown argument location type");
4022
4023 switch (VA.getLocInfo()) {
4024 case CCValAssign::Full:
4025 break;
4026 case CCValAssign::BCvt:
4027 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
4028 break;
4029 case CCValAssign::ZExt:
4030 Val = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Val,
4031 N2: DAG.getValueType(VA.getValVT()));
4032 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4033 break;
4034 case CCValAssign::SExt:
4035 Val = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Val,
4036 N2: DAG.getValueType(VA.getValVT()));
4037 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4038 break;
4039 case CCValAssign::AExt:
4040 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
4041 break;
4042 default:
4043 llvm_unreachable("Unknown loc info!");
4044 }
4045
4046 InVals.push_back(Elt: Val);
4047 }
4048
4049 return Chain;
4050}
4051
4052// Add code to pass special inputs required depending on used features separate
4053// from the explicit user arguments present in the IR.
4054void SITargetLowering::passSpecialInputs(
4055 CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info,
4056 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
4057 SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const {
4058 // If we don't have a call site, this was a call inserted by
4059 // legalization. These can never use special inputs.
4060 if (!CLI.CB)
4061 return;
4062
4063 SelectionDAG &DAG = CLI.DAG;
4064 const SDLoc &DL = CLI.DL;
4065 const Function &F = DAG.getMachineFunction().getFunction();
4066
4067 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
4068 const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
4069
4070 const AMDGPUFunctionArgInfo &CalleeArgInfo =
4071 AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
4072
4073 // TODO: Unify with private memory register handling. This is complicated by
4074 // the fact that at least in kernels, the input argument is not necessarily
4075 // in the same location as the input.
4076 // clang-format off
4077 static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
4078 std::array<StringLiteral, 2>> ImplicitAttrs[] = {
4079 {AMDGPUFunctionArgInfo::DISPATCH_PTR, {"amdgpu-no-dispatch-ptr", ""}},
4080 {AMDGPUFunctionArgInfo::QUEUE_PTR, {"amdgpu-no-queue-ptr", ""}},
4081 {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, {"amdgpu-no-implicitarg-ptr", ""}},
4082 {AMDGPUFunctionArgInfo::DISPATCH_ID, {"amdgpu-no-dispatch-id", ""}},
4083 {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"}},
4084 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"}},
4085 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"}},
4086 {AMDGPUFunctionArgInfo::LDS_KERNEL_ID, {"amdgpu-no-lds-kernel-id", ""}},
4087 };
4088 // clang-format on
4089
4090 for (auto [InputID, Attrs] : ImplicitAttrs) {
4091 // If the callee does not use the attribute value, skip copying the value.
4092 if (all_of(Range&: Attrs, P: [&](StringRef Attr) {
4093 return Attr.empty() || CLI.CB->hasFnAttr(Kind: Attr);
4094 }))
4095 continue;
4096
4097 const auto [OutgoingArg, ArgRC, ArgTy] =
4098 CalleeArgInfo.getPreloadedValue(Value: InputID);
4099 if (!OutgoingArg)
4100 continue;
4101
4102 const auto [IncomingArg, IncomingArgRC, Ty] =
4103 CallerArgInfo.getPreloadedValue(Value: InputID);
4104 assert(IncomingArgRC == ArgRC);
4105
4106 // All special arguments are ints for now.
4107 EVT ArgVT = TRI->getSpillSize(RC: *ArgRC) == 8 ? MVT::i64 : MVT::i32;
4108 SDValue InputReg;
4109
4110 if (IncomingArg) {
4111 InputReg = loadInputValue(DAG, RC: ArgRC, VT: ArgVT, SL: DL, Arg: *IncomingArg);
4112 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
4113 // The implicit arg ptr is special because it doesn't have a corresponding
4114 // input for kernels, and is computed from the kernarg segment pointer.
4115 InputReg = getImplicitArgPtr(DAG, SL: DL);
4116 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
4117 std::optional<uint32_t> Id =
4118 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
4119 if (Id.has_value()) {
4120 InputReg = DAG.getConstant(Val: *Id, DL, VT: ArgVT);
4121 } else {
4122 InputReg = DAG.getPOISON(VT: ArgVT);
4123 }
4124 } else {
4125 // We may have proven the input wasn't needed, although the ABI is
4126 // requiring it. We just need to allocate the register appropriately.
4127 InputReg = DAG.getPOISON(VT: ArgVT);
4128 }
4129
4130 if (OutgoingArg->isRegister()) {
4131 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4132 if (!CCInfo.AllocateReg(Reg: OutgoingArg->getRegister()))
4133 report_fatal_error(reason: "failed to allocate implicit input argument");
4134 } else {
4135 unsigned SpecialArgOffset =
4136 CCInfo.AllocateStack(Size: ArgVT.getStoreSize(), Alignment: Align(4));
4137 SDValue ArgStore =
4138 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4139 MemOpChains.push_back(Elt: ArgStore);
4140 }
4141 }
4142
4143 // Pack workitem IDs into a single register or pass it as is if already
4144 // packed.
4145
4146 auto [OutgoingArg, ArgRC, Ty] =
4147 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
4148 if (!OutgoingArg)
4149 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4150 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
4151 if (!OutgoingArg)
4152 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4153 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
4154 if (!OutgoingArg)
4155 return;
4156
4157 const ArgDescriptor *IncomingArgX = std::get<0>(
4158 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X));
4159 const ArgDescriptor *IncomingArgY = std::get<0>(
4160 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
4161 const ArgDescriptor *IncomingArgZ = std::get<0>(
4162 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
4163
4164 SDValue InputReg;
4165 SDLoc SL;
4166
4167 const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-x");
4168 const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-y");
4169 const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-z");
4170
4171 // If incoming ids are not packed we need to pack them.
4172 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
4173 NeedWorkItemIDX) {
4174 if (Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 0) != 0) {
4175 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgX);
4176 } else {
4177 InputReg = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
4178 }
4179 }
4180
4181 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
4182 NeedWorkItemIDY && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 1) != 0) {
4183 SDValue Y = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgY);
4184 Y = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Y,
4185 N2: DAG.getShiftAmountConstant(Val: 10, VT: MVT::i32, DL: SL));
4186 InputReg = InputReg.getNode()
4187 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Y)
4188 : Y;
4189 }
4190
4191 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
4192 NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 2) != 0) {
4193 SDValue Z = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgZ);
4194 Z = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Z,
4195 N2: DAG.getShiftAmountConstant(Val: 20, VT: MVT::i32, DL: SL));
4196 InputReg = InputReg.getNode()
4197 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Z)
4198 : Z;
4199 }
4200
4201 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
4202 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
4203 // We're in a situation where the outgoing function requires the workitem
4204 // ID, but the calling function does not have it (e.g a graphics function
4205 // calling a C calling convention function). This is illegal, but we need
4206 // to produce something.
4207 InputReg = DAG.getPOISON(VT: MVT::i32);
4208 } else {
4209 // Workitem ids are already packed, any of present incoming arguments
4210 // will carry all required fields.
4211 ArgDescriptor IncomingArg =
4212 ArgDescriptor::createArg(Arg: IncomingArgX ? *IncomingArgX
4213 : IncomingArgY ? *IncomingArgY
4214 : *IncomingArgZ,
4215 Mask: ~0u);
4216 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: IncomingArg);
4217 }
4218 }
4219
4220 if (OutgoingArg->isRegister()) {
4221 if (InputReg)
4222 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4223
4224 CCInfo.AllocateReg(Reg: OutgoingArg->getRegister());
4225 } else {
4226 unsigned SpecialArgOffset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
4227 if (InputReg) {
4228 SDValue ArgStore =
4229 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4230 MemOpChains.push_back(Elt: ArgStore);
4231 }
4232 }
4233}
4234
4235bool SITargetLowering::isEligibleForTailCallOptimization(
4236 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
4237 const SmallVectorImpl<ISD::OutputArg> &Outs,
4238 const SmallVectorImpl<SDValue> &OutVals,
4239 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4240 if (AMDGPU::isChainCC(CC: CalleeCC))
4241 return true;
4242
4243 if (!AMDGPU::mayTailCallThisCC(CC: CalleeCC))
4244 return false;
4245
4246 // For a divergent call target, we need to do a waterfall loop over the
4247 // possible callees which precludes us from using a simple jump.
4248 if (Callee->isDivergent())
4249 return false;
4250
4251 MachineFunction &MF = DAG.getMachineFunction();
4252 const Function &CallerF = MF.getFunction();
4253 CallingConv::ID CallerCC = CallerF.getCallingConv();
4254 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
4255 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
4256
4257 // Kernels aren't callable, and don't have a live in return address so it
4258 // doesn't make sense to do a tail call with entry functions.
4259 if (!CallerPreserved)
4260 return false;
4261
4262 bool CCMatch = CallerCC == CalleeCC;
4263
4264 if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
4265 if (AMDGPU::canGuaranteeTCO(CC: CalleeCC) && CCMatch)
4266 return true;
4267 return false;
4268 }
4269
4270 // TODO: Can we handle var args?
4271 if (IsVarArg)
4272 return false;
4273
4274 for (const Argument &Arg : CallerF.args()) {
4275 if (Arg.hasByValAttr())
4276 return false;
4277 }
4278
4279 LLVMContext &Ctx = *DAG.getContext();
4280
4281 // Check that the call results are passed in the same way.
4282 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C&: Ctx, Ins,
4283 CalleeFn: CCAssignFnForCall(CC: CalleeCC, IsVarArg),
4284 CallerFn: CCAssignFnForCall(CC: CallerCC, IsVarArg)))
4285 return false;
4286
4287 // The callee has to preserve all registers the caller needs to preserve.
4288 if (!CCMatch) {
4289 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
4290 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
4291 return false;
4292 }
4293
4294 // Nothing more to check if the callee is taking no arguments.
4295 if (Outs.empty())
4296 return true;
4297
4298 SmallVector<CCValAssign, 16> ArgLocs;
4299 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
4300
4301 // FIXME: We are not allocating special input registers, so we will be
4302 // deciding based on incorrect register assignments.
4303 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CalleeCC, IsVarArg));
4304
4305 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
4306 // If the stack arguments for this call do not fit into our own save area then
4307 // the call cannot be made tail.
4308 // TODO: Is this really necessary?
4309 if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
4310 return false;
4311
4312 for (const auto &[CCVA, ArgVal] : zip_equal(t&: ArgLocs, u: OutVals)) {
4313 // FIXME: What about inreg arguments that end up passed in memory?
4314 if (!CCVA.isRegLoc())
4315 continue;
4316
4317 // If we are passing an argument in an SGPR, and the value is divergent,
4318 // this call requires a waterfall loop.
4319 if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(Reg: CCVA.getLocReg())) {
4320 LLVM_DEBUG(
4321 dbgs() << "Cannot tail call due to divergent outgoing argument in "
4322 << printReg(CCVA.getLocReg(), TRI) << '\n');
4323 return false;
4324 }
4325 }
4326
4327 const MachineRegisterInfo &MRI = MF.getRegInfo();
4328 return parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals);
4329}
4330
4331bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
4332 if (!CI->isTailCall())
4333 return false;
4334
4335 const Function *ParentFn = CI->getFunction();
4336 if (AMDGPU::isEntryFunctionCC(CC: ParentFn->getCallingConv()))
4337 return false;
4338 return true;
4339}
4340
4341namespace {
4342// Chain calls have special arguments that we need to handle. These are
4343// tagging along at the end of the arguments list(s), after the SGPR and VGPR
4344// arguments (index 0 and 1 respectively).
4345enum ChainCallArgIdx {
4346 Exec = 2,
4347 Flags,
4348 NumVGPRs,
4349 FallbackExec,
4350 FallbackCallee
4351};
4352} // anonymous namespace
4353
4354// The wave scratch offset register is used as the global base pointer.
4355SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
4356 SmallVectorImpl<SDValue> &InVals) const {
4357 CallingConv::ID CallConv = CLI.CallConv;
4358 bool IsChainCallConv = AMDGPU::isChainCC(CC: CallConv);
4359
4360 SelectionDAG &DAG = CLI.DAG;
4361
4362 const SDLoc &DL = CLI.DL;
4363 SDValue Chain = CLI.Chain;
4364 SDValue Callee = CLI.Callee;
4365
4366 llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
4367 bool UsesDynamicVGPRs = false;
4368 if (IsChainCallConv) {
4369 // The last arguments should be the value that we need to put in EXEC,
4370 // followed by the flags and any other arguments with special meanings.
4371 // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so
4372 // we don't treat them like the "real" arguments.
4373 auto RequestedExecIt =
4374 llvm::find_if(Range&: CLI.Outs, P: [](const ISD::OutputArg &Arg) {
4375 return Arg.OrigArgIndex == 2;
4376 });
4377 assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC");
4378
4379 size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin();
4380 CLI.OutVals.erase(CS: CLI.OutVals.begin() + SpecialArgsBeginIdx,
4381 CE: CLI.OutVals.end());
4382 CLI.Outs.erase(CS: RequestedExecIt, CE: CLI.Outs.end());
4383
4384 assert(CLI.Outs.back().OrigArgIndex < 2 &&
4385 "Haven't popped all the special args");
4386
4387 TargetLowering::ArgListEntry RequestedExecArg =
4388 CLI.Args[ChainCallArgIdx::Exec];
4389 if (!RequestedExecArg.Ty->isIntegerTy(BitWidth: Subtarget->getWavefrontSize()))
4390 return lowerUnhandledCall(CLI, InVals, Reason: "Invalid value for EXEC");
4391
4392 // Convert constants into TargetConstants, so they become immediate operands
4393 // instead of being selected into S_MOV.
4394 auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) {
4395 if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Val&: Arg.Node)) {
4396 ChainCallSpecialArgs.push_back(Elt: DAG.getTargetConstant(
4397 Val: ArgNode->getAPIntValue(), DL, VT: ArgNode->getValueType(ResNo: 0)));
4398 } else
4399 ChainCallSpecialArgs.push_back(Elt: Arg.Node);
4400 };
4401
4402 PushNodeOrTargetConstant(RequestedExecArg);
4403
4404 // Process any other special arguments depending on the value of the flags.
4405 TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags];
4406
4407 const APInt &FlagsValue = cast<ConstantSDNode>(Val&: Flags.Node)->getAPIntValue();
4408 if (FlagsValue.isZero()) {
4409 if (CLI.Args.size() > ChainCallArgIdx::Flags + 1)
4410 return lowerUnhandledCall(CLI, InVals,
4411 Reason: "no additional args allowed if flags == 0");
4412 } else if (FlagsValue.isOneBitSet(BitNo: 0)) {
4413 if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
4414 return lowerUnhandledCall(CLI, InVals, Reason: "expected 3 additional args");
4415 }
4416
4417 if (!Subtarget->isWave32()) {
4418 return lowerUnhandledCall(
4419 CLI, InVals, Reason: "dynamic VGPR mode is only supported for wave32");
4420 }
4421
4422 UsesDynamicVGPRs = true;
4423 std::for_each(first: CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
4424 last: CLI.Args.end(), f: PushNodeOrTargetConstant);
4425 }
4426 }
4427
4428 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
4429 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
4430 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
4431 bool &IsTailCall = CLI.IsTailCall;
4432 bool IsVarArg = CLI.IsVarArg;
4433 bool IsSibCall = false;
4434 MachineFunction &MF = DAG.getMachineFunction();
4435
4436 if (Callee.isUndef() || isNullConstant(V: Callee)) {
4437 if (!CLI.IsTailCall) {
4438 for (ISD::InputArg &Arg : CLI.Ins)
4439 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
4440 }
4441
4442 return Chain;
4443 }
4444
4445 if (IsVarArg) {
4446 return lowerUnhandledCall(CLI, InVals,
4447 Reason: "unsupported call to variadic function ");
4448 }
4449
4450 if (!CLI.CB)
4451 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported libcall legalization");
4452
4453 if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
4454 return lowerUnhandledCall(CLI, InVals,
4455 Reason: "unsupported required tail call to function ");
4456 }
4457
4458 if (IsTailCall) {
4459 IsTailCall = isEligibleForTailCallOptimization(Callee, CalleeCC: CallConv, IsVarArg,
4460 Outs, OutVals, Ins, DAG);
4461 if (!IsTailCall &&
4462 ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
4463 report_fatal_error(reason: "failed to perform tail call elimination on a call "
4464 "site marked musttail or on llvm.amdgcn.cs.chain");
4465 }
4466
4467 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4468
4469 // A sibling call is one where we're under the usual C ABI and not planning
4470 // to change that but can still do a tail call:
4471 if (!TailCallOpt && IsTailCall)
4472 IsSibCall = true;
4473
4474 if (IsTailCall)
4475 ++NumTailCalls;
4476 }
4477
4478 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4479 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4480 SmallVector<SDValue, 8> MemOpChains;
4481
4482 // Analyze operands of the call, assigning locations to each operand.
4483 SmallVector<CCValAssign, 16> ArgLocs;
4484 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
4485 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg);
4486
4487 if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CC: CallConv) &&
4488 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {
4489 // With a fixed ABI, allocate fixed registers before user arguments.
4490 passSpecialInputs(CLI, CCInfo, Info: *Info, RegsToPass, MemOpChains, Chain);
4491 }
4492
4493 // Mark the scratch resource descriptor as allocated so the CC analysis
4494 // does not assign user arguments to these registers, matching the callee.
4495 if (!Subtarget->hasFlatScratchEnabled()) {
4496 CCInfo.AllocateReg(
4497 Reg: AMDGPUFunctionArgInfo::getPrivateSegmentBufferReg(CC: CallConv));
4498 }
4499
4500 CCInfo.AnalyzeCallOperands(Outs, Fn: AssignFn);
4501
4502 // Get a count of how many bytes are to be pushed on the stack.
4503 unsigned NumBytes = CCInfo.getStackSize();
4504
4505 if (IsSibCall) {
4506 // Since we're not changing the ABI to make this a tail call, the memory
4507 // operands are already available in the caller's incoming argument space.
4508 NumBytes = 0;
4509 }
4510
4511 // FPDiff is the byte offset of the call's argument area from the callee's.
4512 // Stores to callee stack arguments will be placed in FixedStackSlots offset
4513 // by this amount for a tail call. In a sibling call it must be 0 because the
4514 // caller will deallocate the entire stack and the callee still expects its
4515 // arguments to begin at SP+0. Completely unused for non-tail calls.
4516 int32_t FPDiff = 0;
4517 MachineFrameInfo &MFI = MF.getFrameInfo();
4518 auto *TRI = Subtarget->getRegisterInfo();
4519
4520 // Adjust the stack pointer for the new arguments...
4521 // These operations are automatically eliminated by the prolog/epilog pass
4522 if (!IsSibCall)
4523 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL);
4524
4525 if (!IsSibCall || IsChainCallConv) {
4526 if (!Subtarget->hasFlatScratchEnabled()) {
4527 SmallVector<SDValue, 4> CopyFromChains;
4528
4529 // In the HSA case, this should be an identity copy.
4530 SDValue ScratchRSrcReg =
4531 DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
4532 RegsToPass.emplace_back(
4533 Args: AMDGPUFunctionArgInfo::getPrivateSegmentBufferReg(CC: CallConv),
4534 Args&: ScratchRSrcReg);
4535 CopyFromChains.push_back(Elt: ScratchRSrcReg.getValue(R: 1));
4536 Chain = DAG.getTokenFactor(DL, Vals&: CopyFromChains);
4537 }
4538 }
4539
4540 const unsigned NumSpecialInputs = RegsToPass.size();
4541
4542 MVT PtrVT = MVT::i32;
4543
4544 // Walk the register/memloc assignments, inserting copies/loads.
4545 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4546 CCValAssign &VA = ArgLocs[i];
4547 SDValue Arg = OutVals[i];
4548
4549 // Promote the value if needed.
4550 switch (VA.getLocInfo()) {
4551 case CCValAssign::Full:
4552 break;
4553 case CCValAssign::BCvt:
4554 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
4555 break;
4556 case CCValAssign::ZExt:
4557 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4558 break;
4559 case CCValAssign::SExt:
4560 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4561 break;
4562 case CCValAssign::AExt:
4563 Arg = DAG.getBitcast(VT: VA.getValVT().changeTypeToInteger(), V: Arg);
4564 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4565 break;
4566 case CCValAssign::FPExt:
4567 Arg = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4568 break;
4569 default:
4570 llvm_unreachable("Unknown loc info!");
4571 }
4572
4573 if (VA.isRegLoc()) {
4574 RegsToPass.push_back(Elt: std::pair(VA.getLocReg(), Arg));
4575 } else {
4576 assert(VA.isMemLoc());
4577
4578 SDValue DstAddr;
4579 MachinePointerInfo DstInfo;
4580
4581 unsigned LocMemOffset = VA.getLocMemOffset();
4582 int32_t Offset = LocMemOffset;
4583
4584 SDValue PtrOff = DAG.getConstant(Val: Offset, DL, VT: PtrVT);
4585 MaybeAlign Alignment;
4586
4587 if (IsTailCall) {
4588 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4589 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
4590 : VA.getValVT().getStoreSize();
4591
4592 // FIXME: We can have better than the minimum byval required alignment.
4593 Alignment =
4594 Flags.isByVal()
4595 ? Flags.getNonZeroByValAlign()
4596 : commonAlignment(A: Subtarget->getStackAlignment(), Offset);
4597
4598 Offset = Offset + FPDiff;
4599 int FI = MFI.CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
4600
4601 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
4602 DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
4603
4604 // Make sure any stack arguments overlapping with where we're storing
4605 // are loaded before this eventual operation. Otherwise they'll be
4606 // clobbered.
4607
4608 // FIXME: Why is this really necessary? This seems to just result in a
4609 // lot of code to copy the stack and write them back to the same
4610 // locations, which are supposed to be immutable?
4611 Chain = addTokenForArgument(Chain, DAG, MFI, ClobberedFI: FI);
4612 } else {
4613 // Stores to the argument stack area are relative to the stack pointer.
4614 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getStackPtrOffsetReg(),
4615 VT: MVT::i32);
4616 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SP, N2: PtrOff);
4617 DstInfo = MachinePointerInfo::getStack(MF, Offset: LocMemOffset);
4618 Alignment =
4619 commonAlignment(A: Subtarget->getStackAlignment(), Offset: LocMemOffset);
4620 }
4621
4622 if (Outs[i].Flags.isByVal()) {
4623 SDValue SizeNode =
4624 DAG.getConstant(Val: Outs[i].Flags.getByValSize(), DL, VT: MVT::i32);
4625 SDValue Cpy =
4626 DAG.getMemcpy(Chain, dl: DL, Dst: DstAddr, Src: Arg, Size: SizeNode,
4627 DstAlign: Outs[i].Flags.getNonZeroByValAlign(),
4628 SrcAlign: Outs[i].Flags.getNonZeroByValAlign(),
4629 /*isVol = */ false, /*AlwaysInline = */ true,
4630 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: DstInfo,
4631 SrcPtrInfo: MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
4632
4633 MemOpChains.push_back(Elt: Cpy);
4634 } else {
4635 SDValue Store =
4636 DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo, Alignment);
4637 MemOpChains.push_back(Elt: Store);
4638 }
4639 }
4640 }
4641
4642 if (!MemOpChains.empty())
4643 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
4644
4645 SDValue ReadFirstLaneID =
4646 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4647
4648 SDValue TokenGlue;
4649 if (CLI.ConvergenceControlToken) {
4650 TokenGlue = DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL, VT: MVT::Glue,
4651 Operand: CLI.ConvergenceControlToken);
4652 }
4653
4654 // Build a sequence of copy-to-reg nodes chained together with token chain
4655 // and flag operands which copy the outgoing args into the appropriate regs.
4656 SDValue InGlue;
4657
4658 unsigned ArgIdx = 0;
4659 for (auto [Reg, Val] : RegsToPass) {
4660 if (ArgIdx++ >= NumSpecialInputs &&
4661 (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
4662 // For chain calls, the inreg arguments are required to be
4663 // uniform. Speculatively Insert a readfirstlane in case we cannot prove
4664 // they are uniform.
4665 //
4666 // For other calls, if an inreg arguments is known to be uniform,
4667 // speculatively insert a readfirstlane in case it is in a VGPR.
4668 //
4669 // FIXME: We need to execute this in a waterfall loop if it is a divergent
4670 // value, so let that continue to produce invalid code.
4671
4672 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
4673 if (TokenGlue)
4674 ReadfirstlaneArgs.push_back(Elt: TokenGlue);
4675 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
4676 Ops: ReadfirstlaneArgs);
4677 }
4678
4679 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: Val, Glue: InGlue);
4680 InGlue = Chain.getValue(R: 1);
4681 }
4682
4683 // We don't usually want to end the call-sequence here because we would tidy
4684 // the frame up *after* the call, however in the ABI-changing tail-call case
4685 // we've carefully laid out the parameters so that when sp is reset they'll be
4686 // in the correct location.
4687 if (IsTailCall && !IsSibCall) {
4688 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL);
4689 InGlue = Chain.getValue(R: 1);
4690 }
4691
4692 std::vector<SDValue> Ops({Chain});
4693
4694 // Add a redundant copy of the callee global which will not be legalized, as
4695 // we need direct access to the callee later.
4696 if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
4697 const GlobalValue *GV = GSD->getGlobal();
4698 Ops.push_back(x: Callee);
4699 Ops.push_back(x: DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64));
4700 } else {
4701 if (IsTailCall) {
4702 // isEligibleForTailCallOptimization considered whether the call target is
4703 // divergent, but we may still end up with a uniform value in a VGPR.
4704 // Insert a readfirstlane just in case.
4705 SDValue ReadFirstLaneID =
4706 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4707
4708 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
4709 if (TokenGlue)
4710 ReadfirstlaneArgs.push_back(Elt: TokenGlue); // Wire up convergence token.
4711 Callee = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Callee.getValueType(),
4712 Ops: ReadfirstlaneArgs);
4713 }
4714
4715 Ops.push_back(x: Callee);
4716 Ops.push_back(x: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
4717 }
4718
4719 if (IsTailCall) {
4720 // Each tail call may have to adjust the stack by a different amount, so
4721 // this information must travel along with the operation for eventual
4722 // consumption by emitEpilogue.
4723 Ops.push_back(x: DAG.getTargetConstant(Val: FPDiff, DL, VT: MVT::i32));
4724 }
4725
4726 if (IsChainCallConv)
4727 llvm::append_range(C&: Ops, R&: ChainCallSpecialArgs);
4728
4729 // Add argument registers to the end of the list so that they are known live
4730 // into the call.
4731 for (auto &[Reg, Val] : RegsToPass)
4732 Ops.push_back(x: DAG.getRegister(Reg, VT: Val.getValueType()));
4733
4734 // Add a register mask operand representing the call-preserved registers.
4735 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
4736 assert(Mask && "Missing call preserved mask for calling convention");
4737 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
4738
4739 if (SDValue Token = CLI.ConvergenceControlToken) {
4740 SmallVector<SDValue, 2> GlueOps;
4741 GlueOps.push_back(Elt: Token);
4742 if (InGlue)
4743 GlueOps.push_back(Elt: InGlue);
4744
4745 InGlue = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::CONVERGENCECTRL_GLUE, dl: DL,
4746 VT: MVT::Glue, Ops: GlueOps),
4747 0);
4748 }
4749
4750 if (InGlue)
4751 Ops.push_back(x: InGlue);
4752
4753 // If we're doing a tall call, use a TC_RETURN here rather than an
4754 // actual call instruction.
4755 if (IsTailCall) {
4756 MFI.setHasTailCall();
4757 unsigned OPC = AMDGPUISD::TC_RETURN;
4758 switch (CallConv) {
4759 case CallingConv::AMDGPU_Gfx:
4760 OPC = AMDGPUISD::TC_RETURN_GFX;
4761 break;
4762 case CallingConv::AMDGPU_CS_Chain:
4763 case CallingConv::AMDGPU_CS_ChainPreserve:
4764 OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
4765 : AMDGPUISD::TC_RETURN_CHAIN;
4766 break;
4767 }
4768
4769 // If the caller is a whole wave function, we need to use a special opcode
4770 // so we can patch up EXEC.
4771 if (Info->isWholeWaveFunction())
4772 OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
4773
4774 SDValue Ret = DAG.getNode(Opcode: OPC, DL, VT: MVT::Other, Ops);
4775 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
4776 return Ret;
4777 }
4778
4779 // Returns a chain and a flag for retval copy to use.
4780 SDValue Call = DAG.getNode(Opcode: AMDGPUISD::CALL, DL, ResultTys: {MVT::Other, MVT::Glue}, Ops);
4781 DAG.addNoMergeSiteInfo(Node: Call.getNode(), NoMerge: CLI.NoMerge);
4782 Chain = Call.getValue(R: 0);
4783 InGlue = Call.getValue(R: 1);
4784
4785 uint64_t CalleePopBytes = NumBytes;
4786 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: CalleePopBytes, Glue: InGlue, DL);
4787 if (!Ins.empty())
4788 InGlue = Chain.getValue(R: 1);
4789
4790 // Handle result values, copying them out of physregs into vregs that we
4791 // return.
4792 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
4793 InVals, /*IsThisReturn=*/false, ThisVal: SDValue());
4794}
4795
4796// This is similar to the default implementation in ExpandDYNAMIC_STACKALLOC,
4797// except for:
4798// 1. Stack growth direction(default: downwards, AMDGPU: upwards), and
4799// 2. Scale size where, scale = wave-reduction(alloca-size) * wave-size
4800SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
4801 SelectionDAG &DAG) const {
4802 const MachineFunction &MF = DAG.getMachineFunction();
4803 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4804
4805 SDLoc dl(Op);
4806 EVT VT = Op.getValueType();
4807 SDValue Chain = Op.getOperand(i: 0);
4808 Register SPReg = Info->getStackPtrOffsetReg();
4809
4810 // Chain the dynamic stack allocation so that it doesn't modify the stack
4811 // pointer when other instructions are using the stack.
4812 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
4813
4814 SDValue Size = Op.getOperand(i: 1);
4815 SDValue BaseAddr = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
4816 Align Alignment = cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getAlignValue();
4817
4818 const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
4819 assert(TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
4820 "Stack grows upwards for AMDGPU");
4821
4822 Chain = BaseAddr.getValue(R: 1);
4823 // When using flat-scratch, the stack offset is unscaled.
4824 const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
4825 const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
4826
4827 Align StackAlign = TFL->getStackAlign();
4828 if (Alignment > StackAlign) {
4829 uint64_t ScaledAlignment = Alignment.value()
4830 << (HasFlatScratch ? 0 : WavefrontSizeLog2);
4831 uint64_t StackAlignMask = ScaledAlignment - 1;
4832 SDValue TmpAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr,
4833 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT));
4834 BaseAddr = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TmpAddr,
4835 N2: DAG.getSignedConstant(Val: -ScaledAlignment, DL: dl, VT));
4836 }
4837
4838 assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
4839 SDValue NewSP;
4840 if (isa<ConstantSDNode>(Val: Size)) {
4841 // Increase the stack pointer by the size of the alloca.
4842 // If not using flat-scratch, we have to scale the size by the wave-size.
4843 SDValue ScaledSize =
4844 HasFlatScratch
4845 ? Size
4846 : DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4847 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4848 NewSP = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value
4849 } else {
4850 // For dynamic sized alloca, perform wave-wide reduction to get max of
4851 // alloca size(divergent), and then scale it (when not using flat-scratch)
4852 // by wave-size.
4853 SDValue WaveReduction =
4854 DAG.getTargetConstant(Val: Intrinsic::amdgcn_wave_reduce_umax, DL: dl, VT: MVT::i32);
4855 Size = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: WaveReduction,
4856 N2: Size, N3: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
4857 SDValue ScaledSize = Size;
4858 if (!HasFlatScratch) {
4859 ScaledSize =
4860 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4861 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4862 }
4863 NewSP =
4864 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value in vgpr.
4865 SDValue ReadFirstLaneID =
4866 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: dl, VT: MVT::i32);
4867 NewSP = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: ReadFirstLaneID,
4868 N2: NewSP);
4869 }
4870
4871 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: NewSP); // Output chain
4872 SDValue CallSeqEnd = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
4873
4874 return DAG.getMergeValues(Ops: {BaseAddr, CallSeqEnd}, dl);
4875}
4876
4877SDValue SITargetLowering::LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const {
4878 if (Op.getValueType() != MVT::i32)
4879 return Op; // Defer to cannot select error.
4880
4881 Register SP = getStackPointerRegisterToSaveRestore();
4882 SDLoc SL(Op);
4883
4884 SDValue CopyFromSP = DAG.getCopyFromReg(Chain: Op->getOperand(Num: 0), dl: SL, Reg: SP, VT: MVT::i32);
4885
4886 // Convert from wave uniform to swizzled vector address. This should protect
4887 // from any edge cases where the stacksave result isn't directly used with
4888 // stackrestore.
4889 SDValue VectorAddress =
4890 DAG.getNode(Opcode: AMDGPUISD::WAVE_ADDRESS, DL: SL, VT: MVT::i32, Operand: CopyFromSP);
4891 return DAG.getMergeValues(Ops: {VectorAddress, CopyFromSP.getValue(R: 1)}, dl: SL);
4892}
4893
4894SDValue SITargetLowering::lowerGET_ROUNDING(SDValue Op,
4895 SelectionDAG &DAG) const {
4896 SDLoc SL(Op);
4897 assert(Op.getValueType() == MVT::i32);
4898
4899 uint32_t BothRoundHwReg =
4900 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4901 SDValue GetRoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4902
4903 SDValue IntrinID =
4904 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
4905 SDValue GetReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList: Op->getVTList(),
4906 N1: Op.getOperand(i: 0), N2: IntrinID, N3: GetRoundBothImm);
4907
4908 // There are two rounding modes, one for f32 and one for f64/f16. We only
4909 // report in the standard value range if both are the same.
4910 //
4911 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
4912 // ties away from zero is not supported, and the other values are rotated by
4913 // 1.
4914 //
4915 // If the two rounding modes are not the same, report a target defined value.
4916
4917 // Mode register rounding mode fields:
4918 //
4919 // [1:0] Single-precision round mode.
4920 // [3:2] Double/Half-precision round mode.
4921 //
4922 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
4923 //
4924 // Hardware Spec
4925 // Toward-0 3 0
4926 // Nearest Even 0 1
4927 // +Inf 1 2
4928 // -Inf 2 3
4929 // NearestAway0 N/A 4
4930 //
4931 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
4932 // table we can index by the raw hardware mode.
4933 //
4934 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
4935
4936 SDValue BitTable =
4937 DAG.getConstant(Val: AMDGPU::FltRoundConversionTable, DL: SL, VT: MVT::i64);
4938
4939 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4940 SDValue RoundModeTimesNumBits =
4941 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: GetReg, N2: Two);
4942
4943 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
4944 // knew only one mode was demanded.
4945 SDValue TableValue =
4946 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4947 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4948
4949 SDValue EntryMask = DAG.getConstant(Val: 0xf, DL: SL, VT: MVT::i32);
4950 SDValue TableEntry =
4951 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TruncTable, N2: EntryMask);
4952
4953 // There's a gap in the 4-bit encoded table and actual enum values, so offset
4954 // if it's an extended value.
4955 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4956 SDValue IsStandardValue =
4957 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: TableEntry, RHS: Four, Cond: ISD::SETULT);
4958 SDValue EnumOffset = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: TableEntry, N2: Four);
4959 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: IsStandardValue,
4960 N2: TableEntry, N3: EnumOffset);
4961
4962 return DAG.getMergeValues(Ops: {Result, GetReg.getValue(R: 1)}, dl: SL);
4963}
4964
4965SDValue SITargetLowering::lowerSET_ROUNDING(SDValue Op,
4966 SelectionDAG &DAG) const {
4967 SDLoc SL(Op);
4968
4969 SDValue NewMode = Op.getOperand(i: 1);
4970 assert(NewMode.getValueType() == MVT::i32);
4971
4972 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
4973 // hardware MODE.fp_round values.
4974 if (auto *ConstMode = dyn_cast<ConstantSDNode>(Val&: NewMode)) {
4975 uint32_t ClampedVal = std::min(
4976 a: static_cast<uint32_t>(ConstMode->getZExtValue()),
4977 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
4978 NewMode = DAG.getConstant(
4979 Val: AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal), DL: SL, VT: MVT::i32);
4980 } else {
4981 // If we know the input can only be one of the supported standard modes in
4982 // the range 0-3, we can use a simplified mapping to hardware values.
4983 KnownBits KB = DAG.computeKnownBits(Op: NewMode);
4984 const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
4985 // The supported standard values are 0-3. The extended values start at 8. We
4986 // need to offset by 4 if the value is in the extended range.
4987
4988 if (UseReducedTable) {
4989 // Truncate to the low 32-bits.
4990 SDValue BitTable = DAG.getConstant(
4991 Val: AMDGPU::FltRoundToHWConversionTable & 0xffff, DL: SL, VT: MVT::i32);
4992
4993 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4994 SDValue RoundModeTimesNumBits =
4995 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewMode, N2: Two);
4996
4997 NewMode =
4998 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: BitTable, N2: RoundModeTimesNumBits);
4999
5000 // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
5001 // the table extracted bits into inline immediates.
5002 } else {
5003 // table_index = umin(value, value - 4)
5004 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
5005 SDValue BitTable =
5006 DAG.getConstant(Val: AMDGPU::FltRoundToHWConversionTable, DL: SL, VT: MVT::i64);
5007
5008 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
5009 SDValue OffsetEnum = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewMode, N2: Four);
5010 SDValue IndexVal =
5011 DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewMode, N2: OffsetEnum);
5012
5013 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
5014 SDValue RoundModeTimesNumBits =
5015 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: IndexVal, N2: Two);
5016
5017 SDValue TableValue =
5018 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
5019 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
5020
5021 // No need to mask out the high bits since the setreg will ignore them
5022 // anyway.
5023 NewMode = TruncTable;
5024 }
5025
5026 // Insert a readfirstlane in case the value is a VGPR. We could do this
5027 // earlier and keep more operations scalar, but that interferes with
5028 // combining the source.
5029 SDValue ReadFirstLaneID =
5030 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
5031 NewMode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5032 N1: ReadFirstLaneID, N2: NewMode);
5033 }
5034
5035 // N.B. The setreg will be later folded into s_round_mode on supported
5036 // targets.
5037 SDValue IntrinID =
5038 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
5039 uint32_t BothRoundHwReg =
5040 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
5041 SDValue RoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
5042
5043 SDValue SetReg =
5044 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VTList: Op->getVTList(), N1: Op.getOperand(i: 0),
5045 N2: IntrinID, N3: RoundBothImm, N4: NewMode);
5046
5047 return SetReg;
5048}
5049
5050SDValue SITargetLowering::lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const {
5051 unsigned AS = cast<MemSDNode>(Val&: Op)->getAddressSpace();
5052 if (AS == AMDGPUAS::BUFFER_RESOURCE)
5053 return Op.getOperand(i: 0);
5054 if (AS != AMDGPUAS::FLAT_ADDRESS && !AMDGPU::isExtendedGlobalAddrSpace(AS))
5055 return SDValue();
5056
5057 // I$ prefetch
5058 if (!Subtarget->hasSafeSmemPrefetch() && !Op.getConstantOperandVal(i: 4))
5059 return SDValue();
5060
5061 // Prefetch the 64-bit address that a 32-bit constant pointer is cast to, as
5062 // a load does.
5063 SDValue Ptr = Op.getOperand(i: 1);
5064 if (Ptr.getValueType() != MVT::i32)
5065 return Op;
5066 SmallVector<SDValue, 5> Ops(Op->ops());
5067 Ops[1] = DAG.getAddrSpaceCast(dl: SDLoc(Op), VT: MVT::i64, Ptr,
5068 SrcAS: AMDGPUAS::CONSTANT_ADDRESS_32BIT,
5069 DestAS: AMDGPUAS::CONSTANT_ADDRESS);
5070 return SDValue(DAG.UpdateNodeOperands(N: Op.getNode(), Ops), 0);
5071}
5072
5073// Work around DAG legality rules only based on the result type.
5074SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
5075 bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
5076 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
5077 EVT SrcVT = Src.getValueType();
5078
5079 if (SrcVT.getScalarType() != MVT::bf16)
5080 return Op;
5081
5082 SDLoc SL(Op);
5083 SDValue BitCast =
5084 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcVT.changeTypeToInteger(), Operand: Src);
5085
5086 EVT DstVT = Op.getValueType();
5087 if (IsStrict)
5088 llvm_unreachable("Need STRICT_BF16_TO_FP");
5089
5090 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SL, VT: DstVT, Operand: BitCast);
5091}
5092
5093SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5094 SDLoc SL(Op);
5095 if (Op.getValueType() != MVT::i64)
5096 return Op;
5097
5098 uint32_t ModeHwReg =
5099 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5100 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5101 uint32_t TrapHwReg =
5102 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5103 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5104
5105 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
5106 SDValue IntrinID =
5107 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
5108 SDValue GetModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5109 N1: Op.getOperand(i: 0), N2: IntrinID, N3: ModeHwRegImm);
5110 SDValue GetTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5111 N1: Op.getOperand(i: 0), N2: IntrinID, N3: TrapHwRegImm);
5112 SDValue TokenReg =
5113 DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: GetModeReg.getValue(R: 1),
5114 N2: GetTrapReg.getValue(R: 1));
5115
5116 SDValue CvtPtr =
5117 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: GetModeReg, N2: GetTrapReg);
5118 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
5119
5120 return DAG.getMergeValues(Ops: {Result, TokenReg}, dl: SL);
5121}
5122
5123SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5124 SDLoc SL(Op);
5125 if (Op.getOperand(i: 1).getValueType() != MVT::i64)
5126 return Op;
5127
5128 SDValue Input = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
5129 SDValue NewModeReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5130 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
5131 SDValue NewTrapReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5132 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
5133
5134 SDValue ReadFirstLaneID =
5135 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
5136 NewModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5137 N1: ReadFirstLaneID, N2: NewModeReg);
5138 NewTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5139 N1: ReadFirstLaneID, N2: NewTrapReg);
5140
5141 unsigned ModeHwReg =
5142 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5143 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5144 unsigned TrapHwReg =
5145 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5146 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5147
5148 SDValue IntrinID =
5149 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
5150 SDValue SetModeReg =
5151 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5152 N2: IntrinID, N3: ModeHwRegImm, N4: NewModeReg);
5153 SDValue SetTrapReg =
5154 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5155 N2: IntrinID, N3: TrapHwRegImm, N4: NewTrapReg);
5156 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: SetTrapReg, N2: SetModeReg);
5157}
5158
5159Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT,
5160 const MachineFunction &MF) const {
5161 Register Reg =
5162 StringSwitch<Register>(RegName)
5163 .Case(S: "m0", Value: AMDGPU::M0)
5164 .Case(S: "exec", Value: AMDGPU::EXEC)
5165 .Case(S: "exec_lo", Value: AMDGPU::EXEC_LO)
5166 .Case(S: "exec_hi", Value: AMDGPU::EXEC_HI)
5167 .Case(S: "flat_scratch", Value: AMDGPU::FLAT_SCR)
5168 .Case(S: "flat_scratch_lo", Value: AMDGPU::FLAT_SCR_LO)
5169 .Case(S: "flat_scratch_hi", Value: AMDGPU::FLAT_SCR_HI)
5170 .Case(S: "src_flat_scratch_base", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE)
5171 .Case(S: "src_flat_scratch_base_lo", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
5172 .Case(S: "src_flat_scratch_base_hi", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
5173 .Default(Value: Register());
5174 if (!Reg)
5175 return Reg;
5176
5177 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
5178 if (!Subtarget->hasFlatScrRegister() &&
5179 TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::FLAT_SCR))
5180 return Register();
5181
5182 if (!Subtarget->hasGloballyAddressableScratch() &&
5183 TRI->regsOverlap(RegA: Reg, RegB: AMDGPU::SRC_FLAT_SCRATCH_BASE))
5184 return Register();
5185
5186 switch (Reg) {
5187 case AMDGPU::M0:
5188 case AMDGPU::EXEC_LO:
5189 case AMDGPU::EXEC_HI:
5190 case AMDGPU::FLAT_SCR_LO:
5191 case AMDGPU::FLAT_SCR_HI:
5192 case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
5193 case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
5194 if (VT.getSizeInBits() == 32)
5195 return Reg;
5196 break;
5197 case AMDGPU::EXEC:
5198 case AMDGPU::FLAT_SCR:
5199 case AMDGPU::SRC_FLAT_SCRATCH_BASE:
5200 if (VT.getSizeInBits() == 64)
5201 return Reg;
5202 break;
5203 default:
5204 llvm_unreachable("missing register type checking");
5205 }
5206
5207 report_fatal_error(
5208 reason: Twine("invalid type for register \"" + StringRef(RegName) + "\"."));
5209}
5210
5211// If kill is not the last instruction, split the block so kill is always a
5212// proper terminator.
5213MachineBasicBlock *
5214SITargetLowering::splitKillBlock(MachineInstr &MI,
5215 MachineBasicBlock *BB) const {
5216 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, /*UpdateLiveIns=*/true);
5217 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5218 MI.setDesc(TII->getKillTerminatorFromPseudo(Opcode: MI.getOpcode()));
5219 return SplitBB;
5220}
5221
5222// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
5223// \p MI will be the only instruction in the loop body block. Otherwise, it will
5224// be the first instruction in the remainder block.
5225//
5226/// \returns { LoopBody, Remainder }
5227static std::pair<MachineBasicBlock *, MachineBasicBlock *>
5228splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
5229 MachineFunction *MF = MBB.getParent();
5230 MachineBasicBlock::iterator I(&MI);
5231
5232 // To insert the loop we need to split the block. Move everything after this
5233 // point to a new block, and insert a new empty block between the two.
5234 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
5235 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
5236 MachineFunction::iterator MBBI(MBB);
5237 ++MBBI;
5238
5239 MF->insert(MBBI, MBB: LoopBB);
5240 MF->insert(MBBI, MBB: RemainderBB);
5241
5242 LoopBB->addSuccessor(Succ: LoopBB);
5243 LoopBB->addSuccessor(Succ: RemainderBB);
5244
5245 // Move the rest of the block into a new block.
5246 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
5247
5248 if (InstInLoop) {
5249 auto Next = std::next(x: I);
5250
5251 // Move instruction to loop body.
5252 LoopBB->splice(Where: LoopBB->begin(), Other: &MBB, From: I, To: Next);
5253
5254 // Move the rest of the block.
5255 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Next, To: MBB.end());
5256 } else {
5257 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: I, To: MBB.end());
5258 }
5259
5260 MBB.addSuccessor(Succ: LoopBB);
5261
5262 return std::pair(LoopBB, RemainderBB);
5263}
5264
5265/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
5266void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
5267 MachineBasicBlock *MBB = MI.getParent();
5268 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5269 auto I = MI.getIterator();
5270 auto E = std::next(x: I);
5271
5272 // clang-format off
5273 BuildMI(BB&: *MBB, I: E, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_WAITCNT))
5274 .addImm(Val: 0);
5275 // clang-format on
5276
5277 MIBundleBuilder Bundler(*MBB, I, E);
5278 finalizeBundle(MBB&: *MBB, FirstMI: Bundler.begin());
5279}
5280
5281MachineBasicBlock *
5282SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
5283 MachineBasicBlock *BB) const {
5284 const DebugLoc &DL = MI.getDebugLoc();
5285
5286 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5287
5288 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5289
5290 // Apparently kill flags are only valid if the def is in the same block?
5291 if (MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0))
5292 Src->setIsKill(false);
5293
5294 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB&: *BB, InstInLoop: true);
5295
5296 MachineBasicBlock::iterator I = LoopBB->end();
5297
5298 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
5299 Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: AMDGPU::Hwreg::OFFSET_MEM_VIOL, Values: 1);
5300
5301 // Clear TRAP_STS.MEM_VIOL
5302 BuildMI(BB&: *LoopBB, I: LoopBB->begin(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
5303 .addImm(Val: 0)
5304 .addImm(Val: EncodedReg);
5305
5306 bundleInstWithWaitcnt(MI);
5307
5308 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5309
5310 // Load and check TRAP_STS.MEM_VIOL
5311 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: Reg)
5312 .addImm(Val: EncodedReg);
5313
5314 // clang-format off
5315 // FIXME: Do we need to use an isel pseudo that may clobber scc?
5316 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
5317 .addReg(RegNo: Reg)
5318 .addImm(Val: 0);
5319 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
5320 .addMBB(MBB: LoopBB);
5321 // clang-format on
5322
5323 return RemainderBB;
5324}
5325
5326// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
5327// wavefront. If the value is uniform and just happens to be in a VGPR, this
5328// will only do one iteration. In the worst case, this will loop 64 times.
5329//
5330// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
5331static MachineBasicBlock::iterator
5332emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
5333 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
5334 const DebugLoc &DL, const MachineOperand &Idx,
5335 unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
5336 unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
5337 Register &SGPRIdxReg) {
5338
5339 MachineFunction *MF = OrigBB.getParent();
5340 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5341 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5342 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5343 MachineBasicBlock::iterator I = LoopBB.begin();
5344
5345 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5346 Register PhiExec = MRI.createVirtualRegister(RegClass: BoolRC);
5347 Register NewExec = MRI.createVirtualRegister(RegClass: BoolRC);
5348 Register CurrentIdxReg =
5349 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5350 Register CondReg = MRI.createVirtualRegister(RegClass: BoolRC);
5351
5352 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiReg)
5353 .addReg(RegNo: InitReg)
5354 .addMBB(MBB: &OrigBB)
5355 .addReg(RegNo: ResultReg)
5356 .addMBB(MBB: &LoopBB);
5357
5358 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
5359 .addReg(RegNo: InitSaveExecReg)
5360 .addMBB(MBB: &OrigBB)
5361 .addReg(RegNo: NewExec)
5362 .addMBB(MBB: &LoopBB);
5363
5364 // Read the next variant <- also loop target.
5365 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurrentIdxReg)
5366 .addReg(RegNo: Idx.getReg(), Flags: getUndefRegState(B: Idx.isUndef()));
5367
5368 // Compare the just read M0 value to all possible Idx values.
5369 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CondReg)
5370 .addReg(RegNo: CurrentIdxReg)
5371 .addReg(RegNo: Idx.getReg(), Flags: {}, SubReg: Idx.getSubReg());
5372
5373 // Update EXEC, save the original EXEC value to VCC.
5374 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.AndSaveExecOpc), DestReg: NewExec)
5375 .addReg(RegNo: CondReg)
5376 .setOperandDead(3); // Dead scc
5377
5378 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
5379
5380 if (UseGPRIdxMode) {
5381 if (Offset == 0) {
5382 SGPRIdxReg = CurrentIdxReg;
5383 } else {
5384 SGPRIdxReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
5385 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: SGPRIdxReg)
5386 .addReg(RegNo: CurrentIdxReg)
5387 .addImm(Val: Offset)
5388 .setOperandDead(3); // Dead scc
5389 }
5390 } else {
5391 // Move index from VCC into M0
5392 if (Offset == 0) {
5393 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5394 .addReg(RegNo: CurrentIdxReg);
5395 } else {
5396 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5397 .addReg(RegNo: CurrentIdxReg)
5398 .addImm(Val: Offset)
5399 .setOperandDead(3); // Dead scc
5400 }
5401 }
5402
5403 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
5404 MachineInstr *InsertPt =
5405 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
5406 .addReg(RegNo: LMC.ExecReg)
5407 .addReg(RegNo: NewExec)
5408 .setOperandDead(3); // Dead scc
5409
5410 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
5411 // s_cbranch_scc0?
5412
5413 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
5414 // clang-format off
5415 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
5416 .addMBB(MBB: &LoopBB);
5417 // clang-format on
5418
5419 return InsertPt->getIterator();
5420}
5421
5422// This has slightly sub-optimal regalloc when the source vector is killed by
5423// the read. The register allocator does not understand that the kill is
5424// per-workitem, so is kept alive for the whole loop so we end up not re-using a
5425// subregister from it, using 1 more VGPR than necessary. This was saved when
5426// this was expanded after register allocation.
5427static MachineBasicBlock::iterator
5428loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
5429 unsigned InitResultReg, unsigned PhiReg, int Offset,
5430 bool UseGPRIdxMode, Register &SGPRIdxReg) {
5431 MachineFunction *MF = MBB.getParent();
5432 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5433 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5434 MachineRegisterInfo &MRI = MF->getRegInfo();
5435 const DebugLoc &DL = MI.getDebugLoc();
5436 MachineBasicBlock::iterator I(&MI);
5437
5438 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
5439 Register DstReg = MI.getOperand(i: 0).getReg();
5440 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5441 Register TmpExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5442 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5443
5444 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: TmpExec);
5445
5446 // Save the EXEC mask
5447 // clang-format off
5448 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExec)
5449 .addReg(RegNo: LMC.ExecReg);
5450 // clang-format on
5451
5452 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, InstInLoop: false);
5453
5454 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5455
5456 auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, OrigBB&: MBB, LoopBB&: *LoopBB, DL, Idx: *Idx,
5457 InitReg: InitResultReg, ResultReg: DstReg, PhiReg, InitSaveExecReg: TmpExec,
5458 Offset, UseGPRIdxMode, SGPRIdxReg);
5459
5460 MachineBasicBlock *LandingPad = MF->CreateMachineBasicBlock();
5461 MachineFunction::iterator MBBI(LoopBB);
5462 ++MBBI;
5463 MF->insert(MBBI, MBB: LandingPad);
5464 LoopBB->removeSuccessor(Succ: RemainderBB);
5465 LandingPad->addSuccessor(Succ: RemainderBB);
5466 LoopBB->addSuccessor(Succ: LandingPad);
5467 MachineBasicBlock::iterator First = LandingPad->begin();
5468 // clang-format off
5469 BuildMI(BB&: *LandingPad, I: First, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
5470 .addReg(RegNo: SaveExec);
5471 // clang-format on
5472
5473 return InsPt;
5474}
5475
5476// Returns subreg index, offset
5477static std::pair<unsigned, int>
5478computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
5479 const TargetRegisterClass *SuperRC, unsigned VecReg,
5480 int Offset) {
5481 int NumElts = TRI.getRegSizeInBits(RC: *SuperRC) / 32;
5482
5483 // Skip out of bounds offsets, or else we would end up using an undefined
5484 // register.
5485 if (Offset >= NumElts || Offset < 0)
5486 return std::pair(AMDGPU::sub0, Offset);
5487
5488 return std::pair(SIRegisterInfo::getSubRegFromChannel(Channel: Offset), 0);
5489}
5490
5491static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
5492 MachineRegisterInfo &MRI, MachineInstr &MI,
5493 int Offset) {
5494 MachineBasicBlock *MBB = MI.getParent();
5495 const DebugLoc &DL = MI.getDebugLoc();
5496 MachineBasicBlock::iterator I(&MI);
5497
5498 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5499
5500 assert(Idx->getReg() != AMDGPU::NoRegister);
5501
5502 if (Offset == 0) {
5503 // clang-format off
5504 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5505 .add(MO: *Idx);
5506 // clang-format on
5507 } else {
5508 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5509 .add(MO: *Idx)
5510 .addImm(Val: Offset)
5511 .setOperandDead(3); // Dead scc
5512 }
5513}
5514
5515static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
5516 MachineRegisterInfo &MRI, MachineInstr &MI,
5517 int Offset) {
5518 MachineBasicBlock *MBB = MI.getParent();
5519 const DebugLoc &DL = MI.getDebugLoc();
5520 MachineBasicBlock::iterator I(&MI);
5521
5522 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5523
5524 if (Offset == 0)
5525 return Idx->getReg();
5526
5527 Register Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5528 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: Tmp)
5529 .add(MO: *Idx)
5530 .addImm(Val: Offset)
5531 .setOperandDead(3); // Dead scc
5532 return Tmp;
5533}
5534
5535static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
5536 MachineBasicBlock &MBB,
5537 const GCNSubtarget &ST) {
5538 const SIInstrInfo *TII = ST.getInstrInfo();
5539 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5540 MachineFunction *MF = MBB.getParent();
5541 MachineRegisterInfo &MRI = MF->getRegInfo();
5542
5543 Register Dst = MI.getOperand(i: 0).getReg();
5544 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5545 Register SrcReg = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src)->getReg();
5546 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5547
5548 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcReg);
5549 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5550
5551 unsigned SubReg;
5552 std::tie(args&: SubReg, args&: Offset) =
5553 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcReg, Offset);
5554
5555 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5556
5557 // Check for a SGPR index.
5558 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5559 MachineBasicBlock::iterator I(&MI);
5560 const DebugLoc &DL = MI.getDebugLoc();
5561
5562 if (UseGPRIdxMode) {
5563 // TODO: Look at the uses to avoid the copy. This may require rescheduling
5564 // to avoid interfering with other uses, so probably requires a new
5565 // optimization pass.
5566 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5567
5568 const MCInstrDesc &GPRIDXDesc =
5569 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5570 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5571 .addReg(RegNo: SrcReg)
5572 .addReg(RegNo: Idx)
5573 .addImm(Val: SubReg);
5574 } else {
5575 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5576
5577 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5578 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5579 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5580 }
5581
5582 MI.eraseFromParent();
5583
5584 return &MBB;
5585 }
5586
5587 // Control flow needs to be inserted if indexing with a VGPR.
5588 const DebugLoc &DL = MI.getDebugLoc();
5589 MachineBasicBlock::iterator I(&MI);
5590
5591 Register PhiReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5592 Register InitReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5593
5594 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: InitReg);
5595
5596 Register SGPRIdxReg;
5597 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: InitReg, PhiReg, Offset,
5598 UseGPRIdxMode, SGPRIdxReg);
5599
5600 MachineBasicBlock *LoopBB = InsPt->getParent();
5601
5602 if (UseGPRIdxMode) {
5603 const MCInstrDesc &GPRIDXDesc =
5604 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5605
5606 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5607 .addReg(RegNo: SrcReg)
5608 .addReg(RegNo: SGPRIdxReg)
5609 .addImm(Val: SubReg);
5610 } else {
5611 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5612 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5613 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5614 }
5615
5616 MI.eraseFromParent();
5617
5618 return LoopBB;
5619}
5620
5621static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
5622 MachineBasicBlock &MBB,
5623 const GCNSubtarget &ST) {
5624 const SIInstrInfo *TII = ST.getInstrInfo();
5625 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5626 MachineFunction *MF = MBB.getParent();
5627 MachineRegisterInfo &MRI = MF->getRegInfo();
5628
5629 Register Dst = MI.getOperand(i: 0).getReg();
5630 const MachineOperand *SrcVec = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src);
5631 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5632 const MachineOperand *Val = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::val);
5633 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5634 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcVec->getReg());
5635 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5636
5637 // This can be an immediate, but will be folded later.
5638 assert(Val->getReg());
5639
5640 unsigned SubReg;
5641 std::tie(args&: SubReg, args&: Offset) =
5642 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcVec->getReg(), Offset);
5643 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5644
5645 if (Idx->getReg() == AMDGPU::NoRegister) {
5646 MachineBasicBlock::iterator I(&MI);
5647 const DebugLoc &DL = MI.getDebugLoc();
5648
5649 assert(Offset == 0);
5650
5651 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dst)
5652 .add(MO: *SrcVec)
5653 .add(MO: *Val)
5654 .addImm(Val: SubReg);
5655
5656 MI.eraseFromParent();
5657 return &MBB;
5658 }
5659
5660 // Check for a SGPR index.
5661 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5662 MachineBasicBlock::iterator I(&MI);
5663 const DebugLoc &DL = MI.getDebugLoc();
5664
5665 if (UseGPRIdxMode) {
5666 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5667
5668 const MCInstrDesc &GPRIDXDesc =
5669 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5670 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5671 .addReg(RegNo: SrcVec->getReg())
5672 .add(MO: *Val)
5673 .addReg(RegNo: Idx)
5674 .addImm(Val: SubReg);
5675 } else {
5676 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5677
5678 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5679 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5680 BuildMI(BB&: MBB, I, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5681 .addReg(RegNo: SrcVec->getReg())
5682 .add(MO: *Val)
5683 .addImm(Val: SubReg);
5684 }
5685 MI.eraseFromParent();
5686 return &MBB;
5687 }
5688
5689 // Control flow needs to be inserted if indexing with a VGPR.
5690 const DebugLoc &DL = MI.getDebugLoc();
5691
5692 Register PhiReg = MRI.createVirtualRegister(RegClass: VecRC);
5693
5694 Register SGPRIdxReg;
5695 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: SrcVec->getReg(), PhiReg, Offset,
5696 UseGPRIdxMode, SGPRIdxReg);
5697 MachineBasicBlock *LoopBB = InsPt->getParent();
5698
5699 if (UseGPRIdxMode) {
5700 const MCInstrDesc &GPRIDXDesc =
5701 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5702
5703 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5704 .addReg(RegNo: PhiReg)
5705 .add(MO: *Val)
5706 .addReg(RegNo: SGPRIdxReg)
5707 .addImm(Val: SubReg);
5708 } else {
5709 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5710 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5711 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5712 .addReg(RegNo: PhiReg)
5713 .add(MO: *Val)
5714 .addImm(Val: SubReg);
5715 }
5716
5717 MI.eraseFromParent();
5718 return LoopBB;
5719}
5720
5721static MachineBasicBlock *expand64BitScalarArithmetic(MachineInstr &MI,
5722 MachineBasicBlock *BB) {
5723 // For targets older than GFX12, we emit a sequence of 32-bit operations.
5724 // For GFX12, we emit s_add_u64 and s_sub_u64.
5725 MachineFunction *MF = BB->getParent();
5726 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
5727 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5728 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5729 const DebugLoc &DL = MI.getDebugLoc();
5730 MachineOperand &Dest = MI.getOperand(i: 0);
5731 MachineOperand &Src0 = MI.getOperand(i: 1);
5732 MachineOperand &Src1 = MI.getOperand(i: 2);
5733 bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5734 if (ST.hasScalarAddSub64()) {
5735 // FIXME: If scc is used, this deletes the def
5736 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5737 // clang-format off
5738 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg())
5739 .add(MO: Src0)
5740 .add(MO: Src1);
5741 // clang-format on
5742 } else {
5743 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5744 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5745
5746 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5747 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5748
5749 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5750 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5751 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5752 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5753
5754 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5755 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5756 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5757 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5758
5759 const MachineOperand &ImpDefSCC = MI.getOperand(i: 3);
5760 assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef());
5761
5762 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5763 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5764 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0).add(MO: Src0Sub0).add(MO: Src1Sub0);
5765 auto Hi = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1)
5766 .add(MO: Src0Sub1)
5767 .add(MO: Src1Sub1);
5768 if (ImpDefSCC.isDead())
5769 Hi.setOperandDead(3);
5770 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
5771 .addReg(RegNo: DestSub0)
5772 .addImm(Val: AMDGPU::sub0)
5773 .addReg(RegNo: DestSub1)
5774 .addImm(Val: AMDGPU::sub1);
5775 }
5776 MI.eraseFromParent();
5777 return BB;
5778}
5779
5780static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
5781 MachineFunction *MF = BB->getParent();
5782 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5783 const SIInstrInfo *TII = ST.getInstrInfo();
5784 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5785 MachineRegisterInfo &MRI = MF->getRegInfo();
5786 const DebugLoc &DL = MI.getDebugLoc();
5787 Register Dst = MI.getOperand(i: 0).getReg();
5788 const MachineOperand &Src0 = MI.getOperand(i: 1);
5789 const MachineOperand &Src1 = MI.getOperand(i: 2);
5790 Register SrcCond = MI.getOperand(i: 3).getReg();
5791
5792 Register DstLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5793 Register DstHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5794 const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
5795 Register SrcCondCopy = MRI.createVirtualRegister(RegClass: CondRC);
5796
5797 int Src0Idx =
5798 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
5799 int Src1Idx =
5800 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
5801 const TargetRegisterClass *Src0RC =
5802 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src0Idx));
5803 const TargetRegisterClass *Src1RC =
5804 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src1Idx));
5805
5806 const TargetRegisterClass *Src0SubRC =
5807 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5808 const TargetRegisterClass *Src1SubRC =
5809 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5810
5811 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5812 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
5813 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5814 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
5815
5816 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5817 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
5818 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5819 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
5820
5821 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SrcCondCopy).addReg(RegNo: SrcCond);
5822 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstLo)
5823 .addImm(Val: 0)
5824 .add(MO: Src0Sub0)
5825 .addImm(Val: 0)
5826 .add(MO: Src1Sub0)
5827 .addReg(RegNo: SrcCondCopy);
5828
5829 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstHi)
5830 .addImm(Val: 0)
5831 .add(MO: Src0Sub1)
5832 .addImm(Val: 0)
5833 .add(MO: Src1Sub1)
5834 .addReg(RegNo: SrcCondCopy);
5835
5836 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
5837 .addReg(RegNo: DstLo)
5838 .addImm(Val: AMDGPU::sub0)
5839 .addReg(RegNo: DstHi)
5840 .addImm(Val: AMDGPU::sub1);
5841 MI.eraseFromParent();
5842}
5843
5844static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
5845 switch (Opc) {
5846 case AMDGPU::S_MIN_U32:
5847 return std::numeric_limits<uint32_t>::max();
5848 case AMDGPU::S_MIN_I32:
5849 return std::numeric_limits<int32_t>::max();
5850 case AMDGPU::S_MAX_U32:
5851 return std::numeric_limits<uint32_t>::min();
5852 case AMDGPU::S_MAX_I32:
5853 return std::numeric_limits<int32_t>::min();
5854 case AMDGPU::V_ADD_F32_e64: // -0.0
5855 return 0x80000000;
5856 case AMDGPU::V_SUB_F32_e64: // +0.0
5857 return 0x0;
5858 case AMDGPU::S_ADD_I32:
5859 case AMDGPU::S_SUB_I32:
5860 case AMDGPU::S_OR_B32:
5861 case AMDGPU::S_XOR_B32:
5862 return std::numeric_limits<uint32_t>::min();
5863 case AMDGPU::S_AND_B32:
5864 return std::numeric_limits<uint32_t>::max();
5865 case AMDGPU::V_MIN_F32_e64:
5866 case AMDGPU::V_MAX_F32_e64:
5867 return 0x7fc00000; // qNAN
5868 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5869 return std::numeric_limits<uint64_t>::max();
5870 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5871 return std::numeric_limits<int64_t>::max();
5872 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5873 return std::numeric_limits<uint64_t>::min();
5874 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5875 return std::numeric_limits<int64_t>::min();
5876 case AMDGPU::V_MIN_F64_e64:
5877 case AMDGPU::V_MAX_F64_e64:
5878 case AMDGPU::V_MIN_NUM_F64_e64:
5879 case AMDGPU::V_MAX_NUM_F64_e64:
5880 return 0x7FF8000000000000; // qNAN
5881 case AMDGPU::S_ADD_U64_PSEUDO:
5882 case AMDGPU::S_SUB_U64_PSEUDO:
5883 case AMDGPU::S_OR_B64:
5884 case AMDGPU::S_XOR_B64:
5885 return std::numeric_limits<uint64_t>::min();
5886 case AMDGPU::S_AND_B64:
5887 return std::numeric_limits<uint64_t>::max();
5888 case AMDGPU::V_ADD_F64_e64:
5889 case AMDGPU::V_ADD_F64_pseudo_e64:
5890 return 0x8000000000000000; // -0.0
5891 default:
5892 llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
5893 }
5894}
5895
5896static bool is32bitWaveReduceOperation(unsigned Opc) {
5897 return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
5898 Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
5899 Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
5900 Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
5901 Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
5902 Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
5903 Opc == AMDGPU::V_SUB_F32_e64;
5904}
5905
5906static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
5907 return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
5908 Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
5909 Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
5910 Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
5911 Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
5912}
5913
5914static std::tuple<unsigned, unsigned>
5915getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
5916 unsigned DPPOpc;
5917 switch (Opc) {
5918 case AMDGPU::S_MIN_U32:
5919 DPPOpc = AMDGPU::V_MIN_U32_dpp;
5920 break;
5921 case AMDGPU::S_MIN_I32:
5922 DPPOpc = AMDGPU::V_MIN_I32_dpp;
5923 break;
5924 case AMDGPU::S_MAX_U32:
5925 DPPOpc = AMDGPU::V_MAX_U32_dpp;
5926 break;
5927 case AMDGPU::S_MAX_I32:
5928 DPPOpc = AMDGPU::V_MAX_I32_dpp;
5929 break;
5930 case AMDGPU::S_ADD_I32:
5931 case AMDGPU::S_SUB_I32:
5932 DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
5933 : AMDGPU::V_ADD_CO_U32_dpp;
5934 break;
5935 case AMDGPU::S_AND_B32:
5936 DPPOpc = AMDGPU::V_AND_B32_dpp;
5937 break;
5938 case AMDGPU::S_OR_B32:
5939 DPPOpc = AMDGPU::V_OR_B32_dpp;
5940 break;
5941 case AMDGPU::S_XOR_B32:
5942 DPPOpc = AMDGPU::V_XOR_B32_dpp;
5943 break;
5944 case AMDGPU::V_ADD_F32_e64:
5945 case AMDGPU::V_SUB_F32_e64:
5946 DPPOpc = AMDGPU::V_ADD_F32_dpp;
5947 break;
5948 case AMDGPU::V_MIN_F32_e64:
5949 DPPOpc = AMDGPU::V_MIN_F32_dpp;
5950 break;
5951 case AMDGPU::V_MAX_F32_e64:
5952 DPPOpc = AMDGPU::V_MAX_F32_dpp;
5953 break;
5954 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5955 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5956 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5957 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5958 case AMDGPU::S_ADD_U64_PSEUDO:
5959 case AMDGPU::S_SUB_U64_PSEUDO:
5960 case AMDGPU::S_AND_B64:
5961 case AMDGPU::S_OR_B64:
5962 case AMDGPU::S_XOR_B64:
5963 case AMDGPU::V_MIN_NUM_F64_e64:
5964 case AMDGPU::V_MIN_F64_e64:
5965 case AMDGPU::V_MAX_NUM_F64_e64:
5966 case AMDGPU::V_MAX_F64_e64:
5967 case AMDGPU::V_ADD_F64_pseudo_e64:
5968 case AMDGPU::V_ADD_F64_e64:
5969 DPPOpc = AMDGPU::V_MOV_B64_DPP_PSEUDO;
5970 break;
5971 default:
5972 llvm_unreachable("unhandled lane op");
5973 }
5974 unsigned ClampOpc = Opc;
5975 if (Opc == AMDGPU::V_SUB_F32_e64)
5976 ClampOpc = AMDGPU::V_ADD_F32_e64;
5977 if (!ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/false)) {
5978 if (Opc == AMDGPU::S_SUB_I32)
5979 ClampOpc = AMDGPU::S_ADD_I32;
5980 if (Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO)
5981 ClampOpc = AMDGPU::V_ADD_CO_U32_e64;
5982 else if (Opc == AMDGPU::S_AND_B64)
5983 ClampOpc = AMDGPU::V_AND_B32_e64;
5984 else if (Opc == AMDGPU::S_OR_B64)
5985 ClampOpc = AMDGPU::V_OR_B32_e64;
5986 else if (Opc == AMDGPU::S_XOR_B64)
5987 ClampOpc = AMDGPU::V_XOR_B32_e64;
5988 else
5989 ClampOpc = ST.getInstrInfo()->getVALUOp(Opc: ClampOpc);
5990 }
5991 return {DPPOpc, ClampOpc};
5992}
5993
5994static std::pair<Register, Register>
5995ExtractSubRegs(MachineInstr &MI, MachineOperand &Op,
5996 const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
5997 MachineRegisterInfo &MRI) {
5998 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5999 const SIInstrInfo *TII = ST.getInstrInfo();
6000 const TargetRegisterClass *SrcSubRC =
6001 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
6002 Register Op1L =
6003 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
6004 Register Op1H =
6005 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
6006 return {Op1L, Op1H};
6007}
6008
6009static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
6010 MachineBasicBlock &BB,
6011 const GCNSubtarget &ST,
6012 unsigned Opc) {
6013 MachineRegisterInfo &MRI = BB.getParent()->getRegInfo();
6014 const SIRegisterInfo *TRI = ST.getRegisterInfo();
6015 const DebugLoc &DL = MI.getDebugLoc();
6016 const SIInstrInfo *TII = ST.getInstrInfo();
6017
6018 // Reduction operations depend on whether the input operand is SGPR or VGPR.
6019 Register SrcReg = MI.getOperand(i: 1).getReg();
6020 bool isSGPR = TRI->isSGPRClass(RC: MRI.getRegClass(Reg: SrcReg));
6021 Register DstReg = MI.getOperand(i: 0).getReg();
6022 unsigned Stratergy = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
6023 enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
6024 MachineBasicBlock *RetBB = nullptr;
6025 unsigned MIOpc = MI.getOpcode();
6026 auto BuildRegSequence = [&](MachineBasicBlock &BB,
6027 MachineBasicBlock::iterator MI, Register Dst,
6028 Register Src0, Register Src1) {
6029 auto RegSequence =
6030 BuildMI(BB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
6031 .addReg(RegNo: Src0)
6032 .addImm(Val: AMDGPU::sub0)
6033 .addReg(RegNo: Src1)
6034 .addImm(Val: AMDGPU::sub1);
6035 return RegSequence;
6036 };
6037 if (isSGPR) {
6038 switch (Opc) {
6039 case AMDGPU::S_MIN_U32:
6040 case AMDGPU::S_MIN_I32:
6041 case AMDGPU::V_MIN_F32_e64:
6042 case AMDGPU::S_MAX_U32:
6043 case AMDGPU::S_MAX_I32:
6044 case AMDGPU::V_MAX_F32_e64:
6045 case AMDGPU::S_AND_B32:
6046 case AMDGPU::S_OR_B32: {
6047 // Idempotent operations.
6048 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstReg).addReg(RegNo: SrcReg);
6049 RetBB = &BB;
6050 break;
6051 }
6052 case AMDGPU::V_CMP_LT_U64_e64: // umin
6053 case AMDGPU::V_CMP_LT_I64_e64: // min
6054 case AMDGPU::V_CMP_GT_U64_e64: // umax
6055 case AMDGPU::V_CMP_GT_I64_e64: // max
6056 case AMDGPU::V_MIN_F64_e64:
6057 case AMDGPU::V_MIN_NUM_F64_e64:
6058 case AMDGPU::V_MAX_F64_e64:
6059 case AMDGPU::V_MAX_NUM_F64_e64:
6060 case AMDGPU::S_AND_B64:
6061 case AMDGPU::S_OR_B64: {
6062 // Idempotent operations.
6063 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg).addReg(RegNo: SrcReg);
6064 RetBB = &BB;
6065 break;
6066 }
6067 case AMDGPU::S_XOR_B32:
6068 case AMDGPU::S_XOR_B64:
6069 case AMDGPU::S_ADD_I32:
6070 case AMDGPU::S_ADD_U64_PSEUDO:
6071 case AMDGPU::V_ADD_F32_e64:
6072 case AMDGPU::V_ADD_F64_e64:
6073 case AMDGPU::V_ADD_F64_pseudo_e64:
6074 case AMDGPU::S_SUB_I32:
6075 case AMDGPU::S_SUB_U64_PSEUDO:
6076 case AMDGPU::V_SUB_F32_e64: {
6077 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6078 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
6079 Register ExecMask = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6080 Register NumActiveLanes =
6081 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6082
6083 bool IsWave32 = ST.isWave32();
6084 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6085 MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6086 unsigned BitCountOpc =
6087 IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
6088
6089 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MovOpc), DestReg: ExecMask).addReg(RegNo: ExecReg);
6090
6091 auto NewAccumulator =
6092 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BitCountOpc), DestReg: NumActiveLanes)
6093 .addReg(RegNo: ExecMask)
6094 .setOperandDead(2); // Dead scc
6095
6096 switch (Opc) {
6097 case AMDGPU::S_XOR_B32:
6098 case AMDGPU::S_XOR_B64: {
6099 // Performing an XOR operation on a uniform value
6100 // depends on the parity of the number of active lanes.
6101 // For even parity, the result will be 0, for odd
6102 // parity the result will be the same as the input value.
6103 Register ParityRegister =
6104 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6105 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_AND_B32), DestReg: ParityRegister)
6106 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6107 .addImm(Val: 1)
6108 .setOperandDead(3); // Dead scc
6109 // Check if Src is a known identity constant.
6110 if (TII->getImmOrMaterializedImm(MRI, Reg: SrcReg) == 1) {
6111 // 1 * parity(exec) = parity(exec)
6112 if (Opc == AMDGPU::S_XOR_B32) {
6113 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstReg)
6114 .addReg(RegNo: ParityRegister);
6115 } else {
6116 Register DstHi =
6117 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6118 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi).addImm(Val: 0);
6119 BuildRegSequence(BB, MI, DstReg, ParityRegister, DstHi);
6120 }
6121 break;
6122 }
6123 if (Opc == AMDGPU::S_XOR_B32) {
6124 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6125 .addReg(RegNo: SrcReg)
6126 .addReg(RegNo: ParityRegister);
6127 } else {
6128 Register DestSub0 =
6129 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6130 Register DestSub1 =
6131 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6132 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6133 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6134 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6135 .addReg(RegNo: Op1L)
6136 .addReg(RegNo: ParityRegister);
6137 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub1)
6138 .addReg(RegNo: Op1H)
6139 .addReg(RegNo: ParityRegister);
6140 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6141 }
6142 break;
6143 }
6144 case AMDGPU::S_SUB_I32: {
6145 Register NegatedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6146 // Take the negation of the source operand.
6147 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedVal)
6148 .addImm(Val: 0)
6149 .addReg(RegNo: SrcReg)
6150 .setOperandDead(3); // Dead scc
6151 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6152 .addReg(RegNo: NegatedVal)
6153 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6154 break;
6155 }
6156 case AMDGPU::S_ADD_I32: {
6157 // Check if Src is a known identity constant.
6158 if (TII->getImmOrMaterializedImm(MRI, Reg: SrcReg) == 1) {
6159 // 1 * bitcount(exec) = bitcount(exec)
6160 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
6161 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6162 break;
6163 }
6164 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6165 .addReg(RegNo: SrcReg)
6166 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6167 break;
6168 }
6169 case AMDGPU::S_ADD_U64_PSEUDO:
6170 case AMDGPU::S_SUB_U64_PSEUDO: {
6171 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6172 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6173 Register Op1H_Op0L_Reg =
6174 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6175 Register Op1L_Op0H_Reg =
6176 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6177 Register CarryReg =
6178 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6179 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6180 Register NegatedValLo =
6181 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6182 Register NegatedValHi =
6183 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6184 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6185 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6186 // Check if Src is a known identity constant.
6187 if (Opc == AMDGPU::S_ADD_U64_PSEUDO &&
6188 TII->getImmOrMaterializedImm(MRI, Reg: SrcReg) == 1) {
6189 // 1 * bitcount(exec) = bitcount(exec)
6190 Register DstHi = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6191 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstHi).addImm(Val: 0);
6192 BuildRegSequence(BB, MI, DstReg,
6193 NewAccumulator->getOperand(i: 0).getReg(), DstHi);
6194 break;
6195 }
6196 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6197 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedValLo)
6198 .addImm(Val: 0)
6199 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6200 .setOperandDead(3); // Dead scc
6201 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ASHR_I32), DestReg: NegatedValHi)
6202 .addReg(RegNo: NegatedValLo)
6203 .addImm(Val: 31)
6204 .setOperandDead(3); // Dead scc
6205 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1L_Op0H_Reg)
6206 .addReg(RegNo: Op1L)
6207 .addReg(RegNo: NegatedValHi);
6208 }
6209 Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
6210 ? NegatedValLo
6211 : NewAccumulator->getOperand(i: 0).getReg();
6212 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6213 .addReg(RegNo: Op1L)
6214 .addReg(RegNo: LowOpcode);
6215 if (ST.hasScalarMulHiInsts()) {
6216 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_HI_U32), DestReg: CarryReg)
6217 .addReg(RegNo: Op1L)
6218 .addReg(RegNo: LowOpcode);
6219 } else {
6220 Register VCarryReg =
6221 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6222 Register LowOpVGPR =
6223 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6224 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LowOpVGPR)
6225 .addReg(RegNo: LowOpcode);
6226 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: VCarryReg)
6227 .addReg(RegNo: Op1L)
6228 .addReg(RegNo: LowOpVGPR);
6229 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CarryReg)
6230 .addReg(RegNo: VCarryReg);
6231 }
6232 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1H_Op0L_Reg)
6233 .addReg(RegNo: Op1H)
6234 .addReg(RegNo: LowOpcode);
6235
6236 Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
6237 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: HiVal)
6238 .addReg(RegNo: CarryReg)
6239 .addReg(RegNo: Op1H_Op0L_Reg)
6240 .setOperandDead(3); // Dead scc
6241
6242 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6243 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: DestSub1)
6244 .addReg(RegNo: HiVal)
6245 .addReg(RegNo: Op1L_Op0H_Reg)
6246 .setOperandDead(3); // Dead scc
6247 }
6248 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6249 break;
6250 }
6251 case AMDGPU::V_ADD_F32_e64:
6252 case AMDGPU::V_ADD_F64_e64:
6253 case AMDGPU::V_ADD_F64_pseudo_e64:
6254 case AMDGPU::V_SUB_F32_e64: {
6255 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6256 const TargetRegisterClass *VregRC = TII->getRegClass(MCID: TII->get(Opcode: Opc), OpNum: 0);
6257 Register ActiveLanesVreg = MRI.createVirtualRegister(RegClass: VregRC);
6258 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6259 // Get number of active lanes as a float val.
6260 BuildMI(BB, I&: MI, MIMD: DL,
6261 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
6262 : AMDGPU::V_CVT_F64_I32_e64),
6263 DestReg: ActiveLanesVreg)
6264 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6265 .addImm(Val: 0) // clamp
6266 .addImm(Val: 0); // output-modifier
6267
6268 // Take negation of input for SUB reduction
6269 unsigned srcMod = (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6270 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
6271 ? SISrcMods::NEG
6272 : SISrcMods::NONE;
6273 unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
6274 : ST.getGeneration() >= AMDGPUSubtarget::GFX12
6275 ? AMDGPU::V_MUL_F64_pseudo_e64
6276 : AMDGPU::V_MUL_F64_e64;
6277 auto DestVregInst = BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MulOpc),
6278 DestReg: DstVreg)
6279 .addImm(Val: srcMod) // src0 modifier
6280 .addReg(RegNo: SrcReg)
6281 .addImm(Val: SISrcMods::NONE) // src1 modifier
6282 .addReg(RegNo: ActiveLanesVreg)
6283 .addImm(Val: SISrcMods::NONE) // clamp
6284 .addImm(Val: SISrcMods::NONE); // output-mod
6285 if (is32BitOpc) {
6286 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
6287 .addReg(RegNo: DstVreg);
6288 } else {
6289 Register LaneValueLoReg =
6290 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6291 Register LaneValueHiReg =
6292 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6293 auto [Op1L, Op1H] =
6294 ExtractSubRegs(MI, Op&: DestVregInst->getOperand(i: 0), SrcRC: VregRC, ST, MRI);
6295 // lane value input should be in an sgpr
6296 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6297 DestReg: LaneValueLoReg)
6298 .addReg(RegNo: Op1L);
6299 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6300 DestReg: LaneValueHiReg)
6301 .addReg(RegNo: Op1H);
6302 NewAccumulator =
6303 BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
6304 }
6305 }
6306 }
6307 RetBB = &BB;
6308 }
6309 }
6310 } else {
6311 MachineBasicBlock::iterator I = BB.end();
6312 Register SrcReg = MI.getOperand(i: 1).getReg();
6313 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6314 bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
6315 bool NeedsMovDPP = !is32BitOpc;
6316 // Create virtual registers required for lowering.
6317 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6318 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
6319 const TargetRegisterClass *SrcRegClass = MRI.getRegClass(Reg: SrcReg);
6320 bool IsWave32 = ST.isWave32();
6321 unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6322 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6323 if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
6324 !ST.hasDPP()) { // If target doesn't support DPP operations, default to
6325 // iterative stratergy
6326
6327 // To reduce the VGPR using iterative approach, we need to iterate
6328 // over all the active lanes. Lowering consists of ComputeLoop,
6329 // which iterate over only active lanes. We use copy of EXEC register
6330 // as induction variable and every active lane modifies it using bitset0
6331 // so that we will get the next active lane for next iteration.
6332
6333 // Create Control flow for loop
6334 // Split MI's Machine Basic block into For loop
6335 auto [ComputeLoop, ComputeEnd] = splitBlockForLoop(MI, MBB&: BB, InstInLoop: true);
6336
6337 Register LoopIterator = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6338 Register IdentityValReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6339 Register AccumulatorReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6340 Register ActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6341 Register NewActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6342 Register FF1Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6343 Register LaneValueReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6344
6345 // Create initial values of induction variable from Exec, Accumulator and
6346 // insert branch instr to newly created ComputeBlock
6347 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: MovOpcForExec), DestReg: LoopIterator).addReg(RegNo: ExecReg);
6348 uint64_t IdentityValue =
6349 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6350 ? 0x0 // +0.0 for double sub reduction
6351 : getIdentityValueForWaveReduction(Opc);
6352 BuildMI(BB, I, MIMD: DL,
6353 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6354 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6355 DestReg: IdentityValReg)
6356 .addImm(Val: IdentityValue);
6357 // clang-format off
6358 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_BRANCH))
6359 .addMBB(MBB: ComputeLoop);
6360 // clang-format on
6361
6362 // Start constructing ComputeLoop
6363 I = ComputeLoop->begin();
6364 auto Accumulator =
6365 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: AccumulatorReg)
6366 .addReg(RegNo: IdentityValReg)
6367 .addMBB(MBB: &BB);
6368 auto ActiveBits =
6369 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: ActiveBitsReg)
6370 .addReg(RegNo: LoopIterator)
6371 .addMBB(MBB: &BB);
6372
6373 I = ComputeLoop->end();
6374 MachineInstr *NewAccumulator;
6375 // Perform the computations
6376 unsigned SFFOpc =
6377 IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
6378 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: SFFOpc), DestReg: FF1Reg)
6379 .addReg(RegNo: ActiveBitsReg);
6380 if (is32BitOpc) {
6381 Register OpDstReg = DstReg;
6382 bool hasSrc0Modifier = AMDGPU::getNamedOperandIdx(
6383 Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers) != -1;
6384 bool hasSrc1Modifier = AMDGPU::getNamedOperandIdx(
6385 Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers) != -1;
6386 bool hasClamp =
6387 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::clamp) != -1;
6388 bool hasOpSel =
6389 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel) != -1;
6390 bool hasOMod =
6391 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::omod) != -1;
6392 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6393 DestReg: LaneValueReg)
6394 .addReg(RegNo: SrcReg)
6395 .addReg(RegNo: FF1Reg);
6396 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/false)) {
6397 // Get the Lane Value in VGPR to avoid the Constant Bus Restriction
6398 Register LaneValVgpr = MRI.createVirtualRegister(RegClass: SrcRegClass);
6399 Register VgprResultReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6400 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LaneValVgpr)
6401 .addReg(RegNo: LaneValueReg);
6402 OpDstReg = VgprResultReg;
6403 LaneValueReg = LaneValVgpr;
6404 }
6405 auto OpInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: OpDstReg);
6406 if (hasSrc0Modifier)
6407 OpInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6408 OpInstr.addReg(RegNo: AccumulatorReg); // src0
6409 if (hasSrc1Modifier)
6410 OpInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6411 OpInstr.addReg(RegNo: LaneValueReg); // src1
6412 if (hasClamp)
6413 OpInstr.addImm(Val: 0); // clamp
6414 if (hasOpSel)
6415 OpInstr.addImm(Val: 0); // opsel
6416 if (hasOMod)
6417 OpInstr.addImm(Val: 0); // omod
6418 if (TII->isSALU(Opcode: Opc))
6419 OpInstr.setOperandDead(3); // Dead scc
6420 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/false)) {
6421 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6422 DestReg: DstReg)
6423 .addReg(RegNo: OpDstReg);
6424 }
6425 } else {
6426 Register LaneValueLoReg =
6427 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6428 Register LaneValueHiReg =
6429 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6430 Register LaneValReg =
6431 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6432 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6433 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6434 // lane value input should be in an sgpr
6435 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6436 DestReg: LaneValueLoReg)
6437 .addReg(RegNo: Op1L)
6438 .addReg(RegNo: FF1Reg);
6439 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6440 DestReg: LaneValueHiReg)
6441 .addReg(RegNo: Op1H)
6442 .addReg(RegNo: FF1Reg);
6443 auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
6444 LaneValueLoReg, LaneValueHiReg);
6445 switch (Opc) {
6446 case AMDGPU::S_OR_B64:
6447 case AMDGPU::S_AND_B64:
6448 case AMDGPU::S_XOR_B64: {
6449 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6450 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6451 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6452 .setOperandDead(3); // Dead scc
6453 break;
6454 }
6455 case AMDGPU::V_CMP_GT_I64_e64:
6456 case AMDGPU::V_CMP_GT_U64_e64:
6457 case AMDGPU::V_CMP_LT_I64_e64:
6458 case AMDGPU::V_CMP_LT_U64_e64: {
6459 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6460 Register ComparisonResultReg =
6461 MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6462 int SrcIdx =
6463 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6464 const TargetRegisterClass *VregClass =
6465 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6466 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregClass);
6467 auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
6468 MI, Op&: Accumulator->getOperand(i: 0), SrcRC: VregClass, ST, MRI);
6469 BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
6470 SrcReg0Sub1);
6471 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: LaneMaskReg)
6472 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6473 .addReg(RegNo: AccumulatorVReg);
6474
6475 unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6476 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AndOpc), DestReg: ComparisonResultReg)
6477 .addReg(RegNo: LaneMaskReg)
6478 .addReg(RegNo: ActiveBitsReg);
6479
6480 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6481 MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
6482 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6483 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6484 break;
6485 }
6486 case AMDGPU::V_MIN_F64_e64:
6487 case AMDGPU::V_MIN_NUM_F64_e64:
6488 case AMDGPU::V_MAX_F64_e64:
6489 case AMDGPU::V_MAX_NUM_F64_e64:
6490 case AMDGPU::V_ADD_F64_e64:
6491 case AMDGPU::V_ADD_F64_pseudo_e64: {
6492 int SrcIdx =
6493 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6494 const TargetRegisterClass *VregRC =
6495 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6496 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregRC);
6497 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6498 Register LaneValLo =
6499 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6500 Register LaneValHi =
6501 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6502 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AccumulatorVReg)
6503 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6504 unsigned Modifier =
6505 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6506 ? SISrcMods::NEG
6507 : SISrcMods::NONE;
6508 auto DstVregInst =
6509 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstVreg)
6510 .addImm(Val: Modifier) // src0 modifiers
6511 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6512 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6513 .addReg(RegNo: AccumulatorVReg)
6514 .addImm(Val: SISrcMods::NONE) // clamp
6515 .addImm(Val: SISrcMods::NONE); // omod
6516 auto ReadLaneLo =
6517 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6518 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValLo);
6519 auto ReadLaneHi =
6520 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6521 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValHi);
6522 MachineBasicBlock::iterator Iters = *ReadLaneLo;
6523 auto [Op1L, Op1H] = ExtractSubRegs(MI&: *Iters, Op&: DstVregInst->getOperand(i: 0),
6524 SrcRC: VregRC, ST, MRI);
6525 ReadLaneLo.addReg(RegNo: Op1L);
6526 ReadLaneHi.addReg(RegNo: Op1H);
6527 NewAccumulator =
6528 BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
6529 break;
6530 }
6531 case AMDGPU::S_ADD_U64_PSEUDO:
6532 case AMDGPU::S_SUB_U64_PSEUDO: {
6533 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6534 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6535 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6536 .setOperandDead(3); // Dead scc
6537 ComputeLoop =
6538 expand64BitScalarArithmetic(MI&: *NewAccumulator, BB: ComputeLoop);
6539 break;
6540 }
6541 }
6542 }
6543 // Manipulate the iterator to get the next active lane
6544 unsigned BITSETOpc =
6545 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
6546 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: BITSETOpc), DestReg: NewActiveBitsReg)
6547 .addReg(RegNo: FF1Reg)
6548 .addReg(RegNo: ActiveBitsReg);
6549
6550 // Add phi nodes
6551 Accumulator.addReg(RegNo: DstReg).addMBB(MBB: ComputeLoop);
6552 ActiveBits.addReg(RegNo: NewActiveBitsReg).addMBB(MBB: ComputeLoop);
6553
6554 // Creating branching
6555 MachineInstrBuilder SetSCCInstr;
6556 if (!ST.hasScalarCompareEq64()) {
6557 // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
6558 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6559 unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
6560 SetSCCInstr =
6561 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc), DestReg: LaneMaskReg);
6562 } else {
6563 unsigned CMPOpc =
6564 IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
6565 SetSCCInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc));
6566 }
6567 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6568 if (ST.hasScalarCompareEq64())
6569 SetSCCInstr.addImm(Val: 0);
6570 else
6571 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6572 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
6573 .addMBB(MBB: ComputeLoop);
6574
6575 RetBB = ComputeEnd;
6576 } else {
6577 assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
6578 MachineBasicBlock *CurrBB = &BB;
6579 Register SrcWithIdentity = MRI.createVirtualRegister(RegClass: SrcRegClass);
6580 Register IdentityVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6581 Register IdentitySGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6582 Register DPPRowShr1 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6583 Register DPPRowShr2 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6584 Register DPPRowShr4 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6585 Register DPPRowShr8 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6586 Register RowBcast15 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6587 Register ReducedValSGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6588 Register NegatedReducedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6589 Register RowBcast31 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6590 Register UndefExec = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6591 Register FinalDPPResult;
6592 MachineInstr *SrcWithIdentityInstr;
6593 MachineInstr *LastBcastInstr;
6594 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExec);
6595
6596 uint64_t IdentityValue = getIdentityValueForWaveReduction(Opc);
6597 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL,
6598 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6599 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6600 DestReg: IdentitySGPR)
6601 .addImm(Val: IdentityValue);
6602 auto IdentityCopyInstr =
6603 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: IdentityVGPR)
6604 .addReg(RegNo: IdentitySGPR);
6605 unsigned DPPOpc, ClampOpc;
6606 std::tie(args&: DPPOpc, args&: ClampOpc) = getDPPOpcForWaveReduction(Opc, ST);
6607 auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
6608 Register Src1) {
6609 return BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
6610 DestReg: Dst)
6611 .addImm(Val: 0) // src0 modifiers
6612 .addReg(RegNo: Src0) // src0
6613 .addImm(Val: 0) // src1 modifiers
6614 .addReg(RegNo: Src1) // identity value for inactive lanes
6615 .addReg(RegNo: UndefExec); // bool i1
6616 };
6617 auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
6618 unsigned DPPCtrl) {
6619 auto DPPInstr =
6620 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: DPPOpc), DestReg: Dst).addReg(RegNo: Src); // old
6621 if (isFPOp && !NeedsMovDPP)
6622 DPPInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6623 DPPInstr.addReg(RegNo: Src); // src0
6624 if (isFPOp && !NeedsMovDPP)
6625 DPPInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6626 if (!NeedsMovDPP)
6627 DPPInstr.addReg(RegNo: Src); // src1
6628 if (AMDGPU::getNamedOperandIdx(Opcode: DPPOpc, Name: AMDGPU::OpName::clamp) >= 0)
6629 DPPInstr.addImm(Val: 0); // clamp
6630 DPPInstr
6631 .addImm(Val: DPPCtrl) // dpp-ctrl
6632 .addImm(Val: 0xf) // row-mask
6633 .addImm(Val: 0xf) // bank-mask
6634 .addImm(Val: 0); // bound-control
6635 };
6636 auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1,
6637 bool isAddSub = false,
6638 bool needsCarryIn = false,
6639 Register CarryIn = Register()) {
6640 unsigned InstrOpc = ClampOpc;
6641 Register CarryOutReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6642 if (needsCarryIn)
6643 InstrOpc = AMDGPU::V_ADDC_U32_e64;
6644 auto ClampInstr = BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InstrOpc), DestReg: Dst);
6645 if (isFPOp)
6646 ClampInstr.addImm(Val: SISrcMods::NONE); // src0 mod
6647 if (isAddSub) {
6648 if (needsCarryIn)
6649 ClampInstr.addReg(RegNo: CarryOutReg,
6650 Flags: RegState::Define |
6651 RegState::Dead); // killed carry-out reg
6652 else
6653 ClampInstr.addReg(RegNo: CarryOutReg, Flags: RegState::Define); // carry-out reg
6654 }
6655 ClampInstr.addReg(RegNo: Src0); // src0
6656 if (isFPOp)
6657 ClampInstr.addImm(Val: SISrcMods::NONE); // src1 mod
6658 ClampInstr.addReg(RegNo: Src1); // src1
6659 if (needsCarryIn)
6660 ClampInstr.addReg(RegNo: CarryIn); // carry-in reg
6661 if (AMDGPU::getNamedOperandIdx(Opcode: InstrOpc, Name: AMDGPU::OpName::clamp) >= 0)
6662 ClampInstr.addImm(Val: 0); // clamp
6663 if (isFPOp)
6664 ClampInstr.addImm(Val: 0); // omod
6665 LastBcastInstr = ClampInstr;
6666 return CarryOutReg;
6667 };
6668 auto BuildPostDPPInstr = [&](Register Src0, Register Src1) {
6669 bool isAddSubOpc =
6670 Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO;
6671 bool isBitWiseOpc = Opc == AMDGPU::S_AND_B64 ||
6672 Opc == AMDGPU::S_OR_B64 || Opc == AMDGPU::S_XOR_B64;
6673 Register ReturnReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6674 if (isAddSubOpc || isBitWiseOpc) {
6675 Register ResLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6676 Register ResHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6677 MachineOperand Src0Operand =
6678 MachineOperand::CreateReg(Reg: Src0, /*isDef=*/false);
6679 MachineOperand Src1Operand =
6680 MachineOperand::CreateReg(Reg: Src1, /*isDef=*/false);
6681 auto [Src0Lo, Src0Hi] =
6682 ExtractSubRegs(MI, Op&: Src0Operand, SrcRC: SrcRegClass, ST, MRI);
6683 auto [Src1Lo, Src1Hi] =
6684 ExtractSubRegs(MI, Op&: Src1Operand, SrcRC: SrcRegClass, ST, MRI);
6685 Register CarryReg = BuildClampInstr(
6686 ResLo, Src0Lo, Src1Lo, isAddSubOpc, /*needsCarryIn*/ false);
6687 BuildClampInstr(ResHi, Src0Hi, Src1Hi, isAddSubOpc,
6688 /*needsCarryIn*/ isAddSubOpc, CarryReg);
6689 BuildRegSequence(*CurrBB, MI, ReturnReg, ResLo, ResHi);
6690 } else {
6691 if (isFPOp) {
6692 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: ReturnReg)
6693 .addImm(Val: SISrcMods::NONE) // src0 modifiers
6694 .addReg(RegNo: Src0)
6695 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6696 .addReg(RegNo: Src1)
6697 .addImm(Val: SISrcMods::NONE) // clamp
6698 .addImm(Val: SISrcMods::NONE); // omod
6699 } else {
6700 Register CmpMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6701 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: CmpMaskReg)
6702 .addReg(RegNo: Src0) // src0
6703 .addReg(RegNo: Src1); // src1
6704 LastBcastInstr =
6705 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO),
6706 DestReg: ReturnReg)
6707 .addReg(RegNo: Src1) // src0
6708 .addReg(RegNo: Src0) // src1
6709 .addReg(RegNo: CmpMaskReg); // src2
6710 expand64BitV_CNDMASK(MI&: *LastBcastInstr, BB: CurrBB);
6711 }
6712 }
6713 return ReturnReg;
6714 };
6715
6716 // Set inactive lanes to the identity value.
6717 if (is32BitOpc) {
6718 SrcWithIdentityInstr =
6719 BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
6720 } else {
6721 Register SrcWithIdentitylo =
6722 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6723 Register SrcWithIdentityhi =
6724 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6725 auto [Reg0Sub0, Reg0Sub1] = ExtractSubRegs(
6726 MI, Op&: IdentityCopyInstr->getOperand(i: 0), SrcRC: SrcRegClass, ST, MRI);
6727 auto [SrcReg0Sub0, SrcReg0Sub1] =
6728 ExtractSubRegs(MI, Op&: MI.getOperand(i: 1), SrcRC: SrcRegClass, ST, MRI);
6729 MachineInstr *SetInactiveLoInstr =
6730 BuildSetInactiveInstr(SrcWithIdentitylo, SrcReg0Sub0, Reg0Sub0);
6731 MachineInstr *SetInactiveHiInstr =
6732 BuildSetInactiveInstr(SrcWithIdentityhi, SrcReg0Sub1, Reg0Sub1);
6733 SrcWithIdentityInstr =
6734 BuildRegSequence(*CurrBB, MI, SrcWithIdentity,
6735 SetInactiveLoInstr->getOperand(i: 0).getReg(),
6736 SetInactiveHiInstr->getOperand(i: 0).getReg());
6737 }
6738 // DPP reduction
6739 Register SrcWithIdentityReg =
6740 SrcWithIdentityInstr->getOperand(i: 0).getReg();
6741 BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentityReg,
6742 AMDGPU::DPP::ROW_SHR_FIRST);
6743 if (NeedsMovDPP)
6744 DPPRowShr1 = BuildPostDPPInstr(SrcWithIdentityReg, DPPRowShr1);
6745
6746 BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
6747 (AMDGPU::DPP::ROW_SHR_FIRST + 1));
6748 if (NeedsMovDPP)
6749 DPPRowShr2 = BuildPostDPPInstr(DPPRowShr1, DPPRowShr2);
6750
6751 BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
6752 (AMDGPU::DPP::ROW_SHR_FIRST + 3));
6753 if (NeedsMovDPP)
6754 DPPRowShr4 = BuildPostDPPInstr(DPPRowShr2, DPPRowShr4);
6755
6756 BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
6757 (AMDGPU::DPP::ROW_SHR_FIRST + 7));
6758 if (NeedsMovDPP)
6759 DPPRowShr8 = BuildPostDPPInstr(DPPRowShr4, DPPRowShr8);
6760
6761 if (ST.hasDPPBroadcasts()) {
6762 BuildDPPMachineInstr(RowBcast15, DPPRowShr8, AMDGPU::DPP::BCAST15);
6763 if (NeedsMovDPP)
6764 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, RowBcast15);
6765 } else {
6766 // magic constant: 0x1E0
6767 // To Set BIT_MODE : bit 15 = 0
6768 // XOR mask : bit [14:10] = 0
6769 // OR mask : bit [9:5] = 15
6770 // AND mask : bit [4:0] = 0
6771 if (is32BitOpc) {
6772 Register SwizzledValue =
6773 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6774 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6775 DestReg: SwizzledValue)
6776 .addReg(RegNo: DPPRowShr8) // addr
6777 .addImm(Val: 0x1E0) // swizzle offset (i16)
6778 .addImm(Val: 0x0); // gds (i1)
6779 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
6780 } else {
6781 Register SwizzledValuelo =
6782 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6783 Register SwizzledValuehi =
6784 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6785 Register SwizzledValue64 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6786 MachineOperand DPPRowShr8Op =
6787 MachineOperand::CreateReg(Reg: DPPRowShr8, /*isDef=*/false);
6788 auto [Op1L, Op1H] =
6789 ExtractSubRegs(MI, Op&: DPPRowShr8Op, SrcRC: SrcRegClass, ST, MRI);
6790 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6791 DestReg: SwizzledValuelo)
6792 .addReg(RegNo: Op1L) // addr
6793 .addImm(Val: 0x1E0) // swizzle offset (i16)
6794 .addImm(Val: 0x0); // gds (i1)
6795 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6796 DestReg: SwizzledValuehi)
6797 .addReg(RegNo: Op1H) // addr
6798 .addImm(Val: 0x1E0) // swizzle offset (i16)
6799 .addImm(Val: 0x0); // gds (i1)
6800 BuildRegSequence(*CurrBB, MI, SwizzledValue64, SwizzledValuelo,
6801 SwizzledValuehi);
6802 if (NeedsMovDPP)
6803 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, SwizzledValue64);
6804 else
6805 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue64);
6806 }
6807 }
6808 FinalDPPResult = RowBcast15;
6809 if (!IsWave32) {
6810 if (ST.hasDPPBroadcasts()) {
6811 BuildDPPMachineInstr(RowBcast31, RowBcast15, AMDGPU::DPP::BCAST31);
6812 if (NeedsMovDPP)
6813 RowBcast31 = BuildPostDPPInstr(RowBcast15, RowBcast31);
6814 } else {
6815 Register ShiftedThreadID =
6816 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6817 Register PermuteByteOffset =
6818 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6819 Register PermutedValue = MRI.createVirtualRegister(RegClass: SrcRegClass);
6820 Register Lane32Offset =
6821 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6822 Register WordSizeConst =
6823 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6824 Register ThreadIDRegLo =
6825 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6826 Register ThreadIDReg =
6827 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6828 // Get the thread ID.
6829 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64),
6830 DestReg: ThreadIDRegLo)
6831 .addImm(Val: -1)
6832 .addImm(Val: 0);
6833 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_HI_U32_B32_e64),
6834 DestReg: ThreadIDReg)
6835 .addImm(Val: -1)
6836 .addReg(RegNo: ThreadIDRegLo);
6837 // shift each lane over by 32 positions, so value in 31st lane is
6838 // present in 63rd lane.
6839 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: Lane32Offset)
6840 .addImm(Val: 0x20);
6841 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U32_e64),
6842 DestReg: ShiftedThreadID)
6843 .addReg(RegNo: ThreadIDReg)
6844 .addReg(RegNo: Lane32Offset)
6845 .addImm(Val: 0); // clamp
6846 // multiply by reg size.
6847 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: WordSizeConst)
6848 .addImm(Val: 0x4);
6849 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_LO_U32_e64),
6850 DestReg: PermuteByteOffset)
6851 .addReg(RegNo: WordSizeConst)
6852 .addReg(RegNo: ShiftedThreadID);
6853 // Permute the lanes
6854 if (is32BitOpc) {
6855 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6856 DestReg: PermutedValue)
6857 .addReg(RegNo: PermuteByteOffset) // addr
6858 .addReg(RegNo: RowBcast15) // data
6859 .addImm(Val: 0); // offset
6860 } else {
6861 Register PermutedValuelo =
6862 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6863 Register PermutedValuehi =
6864 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6865 MachineOperand RowBcast15Op =
6866 MachineOperand::CreateReg(Reg: RowBcast15, /*isDef=*/false);
6867 auto [RowBcast15Lo, RowBcast15Hi] =
6868 ExtractSubRegs(MI, Op&: RowBcast15Op, SrcRC: SrcRegClass, ST, MRI);
6869 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6870 DestReg: PermutedValuelo)
6871 .addReg(RegNo: PermuteByteOffset) // addr
6872 .addReg(RegNo: RowBcast15Lo) // swizzle offset (i16)
6873 .addImm(Val: 0x0); // gds (i1)
6874 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6875 DestReg: PermutedValuehi)
6876 .addReg(RegNo: PermuteByteOffset) // addr
6877 .addReg(RegNo: RowBcast15Hi) // swizzle offset (i16)
6878 .addImm(Val: 0x0); // gds (i1)
6879 BuildRegSequence(*CurrBB, MI, PermutedValue, PermutedValuelo,
6880 PermutedValuehi);
6881 }
6882 if (NeedsMovDPP)
6883 RowBcast31 = BuildPostDPPInstr(RowBcast15, PermutedValue);
6884 else
6885 BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
6886 }
6887 FinalDPPResult = RowBcast31;
6888 }
6889 if (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6890 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64) {
6891 Register NegatedValVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6892 // Opc for f32 reduction is V_SUB_F32.
6893 // For f64, there is no equivalent V_SUB_F64 opcode, so use
6894 // V_ADD_F64/V_ADD_F64_pseudo, and negate the second operand.
6895 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc),
6896 DestReg: NegatedValVGPR)
6897 .addImm(Val: SISrcMods::NONE) // src0 mods
6898 .addReg(RegNo: IdentityVGPR) // src0
6899 .addImm(Val: is32BitOpc ? SISrcMods::NONE : SISrcMods::NEG) // src1 mods
6900 .addReg(RegNo: IsWave32 ? RowBcast15 : RowBcast31) // src1
6901 .addImm(Val: SISrcMods::NONE) // clamp
6902 .addImm(Val: SISrcMods::NONE); // omod
6903 FinalDPPResult = NegatedValVGPR;
6904 }
6905 // The final reduced value is in the last lane.
6906 if (is32BitOpc) {
6907 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6908 DestReg: ReducedValSGPR)
6909 .addReg(RegNo: FinalDPPResult)
6910 .addImm(Val: ST.getWavefrontSize() - 1);
6911 } else {
6912 Register LaneValueLoReg =
6913 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6914 Register LaneValueHiReg =
6915 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6916 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
6917 MachineOperand FinalDPPResultOperand =
6918 MachineOperand::CreateReg(Reg: FinalDPPResult, /*isDef=*/false);
6919 auto [Op1L, Op1H] =
6920 ExtractSubRegs(MI, Op&: FinalDPPResultOperand, SrcRC, ST, MRI);
6921 // lane value input should be in an sgpr
6922 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6923 DestReg: LaneValueLoReg)
6924 .addReg(RegNo: Op1L)
6925 .addImm(Val: ST.getWavefrontSize() - 1);
6926 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6927 DestReg: LaneValueHiReg)
6928 .addReg(RegNo: Op1H)
6929 .addImm(Val: ST.getWavefrontSize() - 1);
6930 BuildRegSequence(*CurrBB, MI, ReducedValSGPR, LaneValueLoReg,
6931 LaneValueHiReg);
6932 }
6933 if (Opc == AMDGPU::S_SUB_I32) {
6934 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedReducedVal)
6935 .addImm(Val: 0)
6936 .addReg(RegNo: ReducedValSGPR)
6937 .setOperandDead(3); // Dead scc
6938 } else if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6939 auto NegatedValInstr =
6940 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: NegatedReducedVal)
6941 .addImm(Val: 0)
6942 .addReg(RegNo: ReducedValSGPR)
6943 .setOperandDead(3); // Dead scc
6944 CurrBB = expand64BitScalarArithmetic(MI&: *NegatedValInstr, BB: CurrBB);
6945 }
6946 // Mark the final result as a whole-wave-mode calculation.
6947 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::STRICT_WWM), DestReg: DstReg)
6948 .addReg(RegNo: Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U64_PSEUDO
6949 ? NegatedReducedVal
6950 : ReducedValSGPR);
6951 RetBB = CurrBB;
6952 }
6953 }
6954 MI.eraseFromParent();
6955 return RetBB;
6956}
6957
6958MachineBasicBlock *
6959SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
6960 MachineBasicBlock *BB) const {
6961 MachineFunction *MF = BB->getParent();
6962 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
6963 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6964 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
6965 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
6966 MachineRegisterInfo &MRI = MF->getRegInfo();
6967 const DebugLoc &DL = MI.getDebugLoc();
6968
6969 switch (MI.getOpcode()) {
6970 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
6971 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_U32);
6972 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
6973 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_U64_e64);
6974 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
6975 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_I32);
6976 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
6977 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_I64_e64);
6978 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
6979 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MIN_F32_e64);
6980 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
6981 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6982 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6983 ? AMDGPU::V_MIN_NUM_F64_e64
6984 : AMDGPU::V_MIN_F64_e64);
6985 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
6986 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_U32);
6987 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
6988 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_U64_e64);
6989 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
6990 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_I32);
6991 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
6992 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_I64_e64);
6993 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
6994 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MAX_F32_e64);
6995 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
6996 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6997 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6998 ? AMDGPU::V_MAX_NUM_F64_e64
6999 : AMDGPU::V_MAX_F64_e64);
7000 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
7001 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_I32);
7002 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
7003 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_U64_PSEUDO);
7004 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
7005 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_ADD_F32_e64);
7006 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
7007 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
7008 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
7009 ? AMDGPU::V_ADD_F64_pseudo_e64
7010 : AMDGPU::V_ADD_F64_e64);
7011 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
7012 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_I32);
7013 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
7014 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_U64_PSEUDO);
7015 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
7016 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_SUB_F32_e64);
7017 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
7018 // There is no S/V_SUB_F64 opcode. Double type subtraction is expanded as
7019 // fadd + neg, by setting the NEG bit in the instruction.
7020 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
7021 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
7022 ? AMDGPU::V_ADD_F64_pseudo_e64
7023 : AMDGPU::V_ADD_F64_e64);
7024 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
7025 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B32);
7026 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
7027 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B64);
7028 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
7029 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B32);
7030 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
7031 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B64);
7032 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
7033 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B32);
7034 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
7035 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B64);
7036 case AMDGPU::S_UADDO_PSEUDO:
7037 case AMDGPU::S_USUBO_PSEUDO: {
7038 MachineOperand &Dest0 = MI.getOperand(i: 0);
7039 MachineOperand &Dest1 = MI.getOperand(i: 1);
7040 MachineOperand &Src0 = MI.getOperand(i: 2);
7041 MachineOperand &Src1 = MI.getOperand(i: 3);
7042
7043 unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
7044 ? AMDGPU::S_ADD_U32
7045 : AMDGPU::S_SUB_U32;
7046 // clang-format off
7047 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest0.getReg())
7048 .add(MO: Src0)
7049 .add(MO: Src1);
7050 // clang-format on
7051
7052 unsigned SelOpc =
7053 Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7054 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: Dest1.getReg()).addImm(Val: -1).addImm(Val: 0);
7055
7056 MI.eraseFromParent();
7057 return BB;
7058 }
7059 case AMDGPU::S_ADD_U64_PSEUDO:
7060 case AMDGPU::S_SUB_U64_PSEUDO: {
7061 return expand64BitScalarArithmetic(MI, BB);
7062 }
7063 case AMDGPU::V_ADD_U64_PSEUDO:
7064 case AMDGPU::V_SUB_U64_PSEUDO: {
7065 bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
7066
7067 MachineOperand &Dest = MI.getOperand(i: 0);
7068 MachineOperand &Src0 = MI.getOperand(i: 1);
7069 MachineOperand &Src1 = MI.getOperand(i: 2);
7070
7071 if (ST.hasAddSubU64Insts()) {
7072 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL,
7073 MCID: TII->get(Opcode: IsAdd ? AMDGPU::V_ADD_U64_e64
7074 : AMDGPU::V_SUB_U64_e64),
7075 DestReg: Dest.getReg())
7076 .add(MO: Src0)
7077 .add(MO: Src1)
7078 .addImm(Val: 0); // clamp
7079 TII->legalizeOperands(MI&: *I);
7080 MI.eraseFromParent();
7081 return BB;
7082 }
7083
7084 if (IsAdd && ST.hasLshlAddU64Inst()) {
7085 auto Add = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_LSHL_ADD_U64_e64),
7086 DestReg: Dest.getReg())
7087 .add(MO: Src0)
7088 .addImm(Val: 0)
7089 .add(MO: Src1);
7090 TII->legalizeOperands(MI&: *Add);
7091 MI.eraseFromParent();
7092 return BB;
7093 }
7094
7095 const auto *CarryRC = TRI->getWaveMaskRegClass();
7096
7097 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7098 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
7099
7100 Register CarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
7101 Register DeadCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
7102
7103 const TargetRegisterClass *Src0RC = Src0.isReg()
7104 ? MRI.getRegClass(Reg: Src0.getReg())
7105 : &AMDGPU::VReg_64RegClass;
7106 const TargetRegisterClass *Src1RC = Src1.isReg()
7107 ? MRI.getRegClass(Reg: Src1.getReg())
7108 : &AMDGPU::VReg_64RegClass;
7109
7110 const TargetRegisterClass *Src0SubRC =
7111 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
7112 const TargetRegisterClass *Src1SubRC =
7113 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
7114
7115 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
7116 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
7117 MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
7118 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
7119
7120 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
7121 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
7122 MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
7123 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
7124
7125 unsigned LoOpc =
7126 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
7127 MachineInstr *LoHalf = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0)
7128 .addReg(RegNo: CarryReg, Flags: RegState::Define)
7129 .add(MO: SrcReg0Sub0)
7130 .add(MO: SrcReg1Sub0)
7131 .addImm(Val: 0); // clamp bit
7132
7133 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
7134 MachineInstr *HiHalf =
7135 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1)
7136 .addReg(RegNo: DeadCarryReg, Flags: RegState::Define | RegState::Dead)
7137 .add(MO: SrcReg0Sub1)
7138 .add(MO: SrcReg1Sub1)
7139 .addReg(RegNo: CarryReg)
7140 .addImm(Val: 0); // clamp bit
7141
7142 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
7143 .addReg(RegNo: DestSub0)
7144 .addImm(Val: AMDGPU::sub0)
7145 .addReg(RegNo: DestSub1)
7146 .addImm(Val: AMDGPU::sub1);
7147 TII->legalizeOperands(MI&: *LoHalf);
7148 TII->legalizeOperands(MI&: *HiHalf);
7149 MI.eraseFromParent();
7150 return BB;
7151 }
7152 case AMDGPU::S_ADD_CO_PSEUDO:
7153 case AMDGPU::S_SUB_CO_PSEUDO: {
7154 // This pseudo has a chance to be selected
7155 // only from uniform add/subcarry node. All the VGPR operands
7156 // therefore assumed to be splat vectors.
7157 MachineBasicBlock::iterator MII = MI;
7158 MachineOperand &Dest = MI.getOperand(i: 0);
7159 MachineOperand &CarryDest = MI.getOperand(i: 1);
7160 MachineOperand &Src0 = MI.getOperand(i: 2);
7161 MachineOperand &Src1 = MI.getOperand(i: 3);
7162 MachineOperand &Src2 = MI.getOperand(i: 4);
7163 if (Src0.isReg() && TRI->isVectorRegister(MRI, Reg: Src0.getReg())) {
7164 Register RegOp0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7165 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp0)
7166 .addReg(RegNo: Src0.getReg());
7167 Src0.setReg(RegOp0);
7168 }
7169 if (Src1.isReg() && TRI->isVectorRegister(MRI, Reg: Src1.getReg())) {
7170 Register RegOp1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7171 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp1)
7172 .addReg(RegNo: Src1.getReg());
7173 Src1.setReg(RegOp1);
7174 }
7175 Register RegOp2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7176 if (TRI->isVectorRegister(MRI, Reg: Src2.getReg())) {
7177 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp2)
7178 .addReg(RegNo: Src2.getReg());
7179 Src2.setReg(RegOp2);
7180 }
7181
7182 if (ST.isWave64()) {
7183 if (ST.hasScalarCompareEq64()) {
7184 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U64))
7185 .addReg(RegNo: Src2.getReg())
7186 .addImm(Val: 0);
7187 } else {
7188 const TargetRegisterClass *Src2RC = MRI.getRegClass(Reg: Src2.getReg());
7189 const TargetRegisterClass *SubRC =
7190 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
7191 MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
7192 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub0, SubRC);
7193 MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
7194 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub1, SubRC);
7195 Register Src2_32 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7196
7197 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_OR_B32), DestReg: Src2_32)
7198 .add(MO: Src2Sub0)
7199 .add(MO: Src2Sub1);
7200
7201 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7202 .addReg(RegNo: Src2_32)
7203 .addImm(Val: 0);
7204 }
7205 } else {
7206 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7207 .addReg(RegNo: Src2.getReg())
7208 .addImm(Val: 0);
7209 }
7210
7211 unsigned Opc = MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
7212 ? AMDGPU::S_ADDC_U32
7213 : AMDGPU::S_SUBB_U32;
7214
7215 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg()).add(MO: Src0).add(MO: Src1);
7216
7217 unsigned SelOpc =
7218 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7219
7220 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: CarryDest.getReg())
7221 .addImm(Val: -1)
7222 .addImm(Val: 0);
7223
7224 MI.eraseFromParent();
7225 return BB;
7226 }
7227 case AMDGPU::SI_INIT_M0: {
7228 MachineOperand &M0Init = MI.getOperand(i: 0);
7229 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7230 MCID: TII->get(Opcode: M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
7231 DestReg: AMDGPU::M0)
7232 .add(MO: M0Init);
7233 MI.eraseFromParent();
7234 return BB;
7235 }
7236 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
7237 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7238 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7239 MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7240 .addImm(Val: 0)
7241 .addImm(Val: 0);
7242 return BB;
7243 }
7244 case AMDGPU::GET_GROUPSTATICSIZE: {
7245 assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
7246 getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
7247 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32))
7248 .add(MO: MI.getOperand(i: 0))
7249 .addImm(Val: MFI->getLDSSize());
7250 MI.eraseFromParent();
7251 return BB;
7252 }
7253 case AMDGPU::GET_SHADERCYCLESHILO: {
7254 assert(MF->getSubtarget<GCNSubtarget>().hasShaderCyclesHiLoRegisters());
7255 // The algorithm is:
7256 //
7257 // hi1 = getreg(SHADER_CYCLES_HI)
7258 // lo1 = getreg(SHADER_CYCLES_LO)
7259 // hi2 = getreg(SHADER_CYCLES_HI)
7260 //
7261 // If hi1 == hi2 then there was no overflow and the result is hi2:lo1.
7262 // Otherwise there was overflow and the result is hi2:0. In both cases the
7263 // result should represent the actual time at some point during the sequence
7264 // of three getregs.
7265 using namespace AMDGPU::Hwreg;
7266 Register RegHi1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7267 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi1)
7268 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7269 Register RegLo1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7270 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegLo1)
7271 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES, Values: 0, Values: 32));
7272 Register RegHi2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7273 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi2)
7274 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7275 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7276 .addReg(RegNo: RegHi1)
7277 .addReg(RegNo: RegHi2);
7278 Register RegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7279 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: RegLo)
7280 .addReg(RegNo: RegLo1)
7281 .addImm(Val: 0);
7282 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE))
7283 .add(MO: MI.getOperand(i: 0))
7284 .addReg(RegNo: RegLo)
7285 .addImm(Val: AMDGPU::sub0)
7286 .addReg(RegNo: RegHi2)
7287 .addImm(Val: AMDGPU::sub1);
7288 MI.eraseFromParent();
7289 return BB;
7290 }
7291 case AMDGPU::SI_INDIRECT_SRC_V1:
7292 case AMDGPU::SI_INDIRECT_SRC_V2:
7293 case AMDGPU::SI_INDIRECT_SRC_V3:
7294 case AMDGPU::SI_INDIRECT_SRC_V4:
7295 case AMDGPU::SI_INDIRECT_SRC_V5:
7296 case AMDGPU::SI_INDIRECT_SRC_V6:
7297 case AMDGPU::SI_INDIRECT_SRC_V7:
7298 case AMDGPU::SI_INDIRECT_SRC_V8:
7299 case AMDGPU::SI_INDIRECT_SRC_V9:
7300 case AMDGPU::SI_INDIRECT_SRC_V10:
7301 case AMDGPU::SI_INDIRECT_SRC_V11:
7302 case AMDGPU::SI_INDIRECT_SRC_V12:
7303 case AMDGPU::SI_INDIRECT_SRC_V16:
7304 case AMDGPU::SI_INDIRECT_SRC_V32:
7305 return emitIndirectSrc(MI, MBB&: *BB, ST: *getSubtarget());
7306 case AMDGPU::SI_INDIRECT_DST_V1:
7307 case AMDGPU::SI_INDIRECT_DST_V2:
7308 case AMDGPU::SI_INDIRECT_DST_V3:
7309 case AMDGPU::SI_INDIRECT_DST_V4:
7310 case AMDGPU::SI_INDIRECT_DST_V5:
7311 case AMDGPU::SI_INDIRECT_DST_V6:
7312 case AMDGPU::SI_INDIRECT_DST_V7:
7313 case AMDGPU::SI_INDIRECT_DST_V8:
7314 case AMDGPU::SI_INDIRECT_DST_V9:
7315 case AMDGPU::SI_INDIRECT_DST_V10:
7316 case AMDGPU::SI_INDIRECT_DST_V11:
7317 case AMDGPU::SI_INDIRECT_DST_V12:
7318 case AMDGPU::SI_INDIRECT_DST_V16:
7319 case AMDGPU::SI_INDIRECT_DST_V32:
7320 return emitIndirectDst(MI, MBB&: *BB, ST: *getSubtarget());
7321 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
7322 case AMDGPU::SI_KILL_I1_PSEUDO:
7323 return splitKillBlock(MI, BB);
7324 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
7325 expand64BitV_CNDMASK(MI, BB);
7326 return BB;
7327 }
7328 case AMDGPU::SI_BR_UNDEF: {
7329 MachineInstr *Br = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
7330 .add(MO: MI.getOperand(i: 0));
7331 Br->getOperand(i: 1).setIsUndef(); // read undef SCC
7332 MI.eraseFromParent();
7333 return BB;
7334 }
7335 case AMDGPU::ADJCALLSTACKUP:
7336 case AMDGPU::ADJCALLSTACKDOWN: {
7337 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
7338 MachineInstrBuilder MIB(*MF, &MI);
7339 MIB.addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::ImplicitDefine)
7340 .addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::Implicit);
7341 return BB;
7342 }
7343 case AMDGPU::SI_CALL_ISEL: {
7344 unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(MF: *MF);
7345
7346 MachineInstrBuilder MIB;
7347 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_CALL))
7348 .addDef(RegNo: ReturnAddrReg, Flags: RegState::Dead);
7349
7350 for (const MachineOperand &MO : MI.operands())
7351 MIB.add(MO);
7352
7353 MIB.cloneMemRefs(OtherMI: MI);
7354 MIB.setMIFlags(MI.getFlags());
7355 MI.eraseFromParent();
7356 return BB;
7357 }
7358 case AMDGPU::V_ADDC_U32_e32:
7359 case AMDGPU::V_SUBB_U32_e32:
7360 case AMDGPU::V_SUBBREV_U32_e32:
7361 // These instructions have an implicit use of vcc which counts towards the
7362 // constant bus limit.
7363 TII->legalizeOperands(MI);
7364 return BB;
7365 case AMDGPU::DS_GWS_INIT:
7366 case AMDGPU::DS_GWS_SEMA_BR:
7367 case AMDGPU::DS_GWS_BARRIER:
7368 case AMDGPU::DS_GWS_SEMA_V:
7369 case AMDGPU::DS_GWS_SEMA_P:
7370 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
7371 // A s_waitcnt 0 is required to be the instruction immediately following.
7372 if (getSubtarget()->hasGWSAutoReplay()) {
7373 bundleInstWithWaitcnt(MI);
7374 return BB;
7375 }
7376
7377 return emitGWSMemViolTestLoop(MI, BB);
7378 case AMDGPU::S_SETREG_B32: {
7379 // Try to optimize cases that only set the denormal mode or rounding mode.
7380 //
7381 // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
7382 // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
7383 // instead.
7384 //
7385 // FIXME: This could be predicates on the immediate, but tablegen doesn't
7386 // allow you to have a no side effect instruction in the output of a
7387 // sideeffecting pattern.
7388 auto [ID, Offset, Width] =
7389 AMDGPU::Hwreg::HwregEncoding::decode(Encoded: MI.getOperand(i: 1).getImm());
7390 if (ID != AMDGPU::Hwreg::ID_MODE)
7391 return BB;
7392
7393 const unsigned WidthMask = maskTrailingOnes<unsigned>(N: Width);
7394 const unsigned SetMask = WidthMask << Offset;
7395
7396 if (getSubtarget()->hasDenormModeInst()) {
7397 unsigned SetDenormOp = 0;
7398 unsigned SetRoundOp = 0;
7399
7400 // The dedicated instructions can only set the whole denorm or round mode
7401 // at once, not a subset of bits in either.
7402 if (SetMask ==
7403 (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) {
7404 // If this fully sets both the round and denorm mode, emit the two
7405 // dedicated instructions for these.
7406 SetRoundOp = AMDGPU::S_ROUND_MODE;
7407 SetDenormOp = AMDGPU::S_DENORM_MODE;
7408 } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
7409 SetRoundOp = AMDGPU::S_ROUND_MODE;
7410 } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
7411 SetDenormOp = AMDGPU::S_DENORM_MODE;
7412 }
7413
7414 if (SetRoundOp || SetDenormOp) {
7415 MachineInstr *Def = MRI.getVRegDef(Reg: MI.getOperand(i: 0).getReg());
7416 if (Def && Def->isMoveImmediate() && Def->getOperand(i: 1).isImm()) {
7417 unsigned ImmVal = Def->getOperand(i: 1).getImm();
7418 if (SetRoundOp) {
7419 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetRoundOp))
7420 .addImm(Val: ImmVal & 0xf);
7421
7422 // If we also have the denorm mode, get just the denorm mode bits.
7423 ImmVal >>= 4;
7424 }
7425
7426 if (SetDenormOp) {
7427 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetDenormOp))
7428 .addImm(Val: ImmVal & 0xf);
7429 }
7430
7431 MI.eraseFromParent();
7432 return BB;
7433 }
7434 }
7435 }
7436
7437 // If only FP bits are touched, used the no side effects pseudo.
7438 if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
7439 AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
7440 MI.setDesc(TII->get(Opcode: AMDGPU::S_SETREG_B32_mode));
7441
7442 return BB;
7443 }
7444 case AMDGPU::S_INVERSE_BALLOT_U32:
7445 case AMDGPU::S_INVERSE_BALLOT_U64:
7446 // These opcodes only exist to let SIFixSGPRCopies insert a readfirstlane if
7447 // necessary. After that they are equivalent to a COPY.
7448 MI.setDesc(TII->get(Opcode: AMDGPU::COPY));
7449 return BB;
7450 case AMDGPU::ENDPGM_TRAP: {
7451 if (BB->succ_empty() && std::next(x: MI.getIterator()) == BB->end()) {
7452 MI.setDesc(TII->get(Opcode: AMDGPU::S_ENDPGM));
7453 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
7454 return BB;
7455 }
7456
7457 // We need a block split to make the real endpgm a terminator. We also don't
7458 // want to break phis in successor blocks, so we can't just delete to the
7459 // end of the block.
7460
7461 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false /*UpdateLiveIns*/);
7462 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
7463 MF->push_back(MBB: TrapBB);
7464 // clang-format off
7465 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ENDPGM))
7466 .addImm(Val: 0);
7467 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
7468 .addMBB(MBB: TrapBB);
7469 // clang-format on
7470
7471 BB->addSuccessor(Succ: TrapBB);
7472 MI.eraseFromParent();
7473 return SplitBB;
7474 }
7475 case AMDGPU::SIMULATED_TRAP: {
7476 assert(Subtarget->hasPrivEnabledTrap2NopBug());
7477 MachineBasicBlock *SplitBB =
7478 TII->insertSimulatedTrap(MRI, MBB&: *BB, MI, DL: MI.getDebugLoc());
7479 MI.eraseFromParent();
7480 return SplitBB;
7481 }
7482 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
7483 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
7484 assert(MFI->isWholeWaveFunction());
7485
7486 // During ISel, it's difficult to propagate the original EXEC mask to use as
7487 // an input to SI_WHOLE_WAVE_FUNC_RETURN. Set it up here instead.
7488 MachineInstr *Setup = TII->getWholeWaveFunctionSetup(MF&: *BB->getParent());
7489 assert(Setup && "Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
7490 Register OriginalExec = Setup->getOperand(i: 0).getReg();
7491 MI.getOperand(i: 0).setReg(OriginalExec);
7492 return BB;
7493 }
7494 case AMDGPU::V_DOT2_F32_F16:
7495 case AMDGPU::V_DOT2_F32_BF16: {
7496 // Hint RA to assign dst and src2 the same physical register.
7497 // For targets without VOP2, but with VOPD, variant of the instruction this
7498 // is one of the conditions to attempt converting VOP3P to VOPD.
7499 MRI.setSimpleHint(VReg: MI.getOperand(i: 0).getReg(), PrefReg: MI.getOperand(i: 6).getReg());
7500 return BB;
7501 }
7502 case AMDGPU::SCHED_BARRIER:
7503 case AMDGPU::SCHED_GROUP_BARRIER:
7504 MI.getOperand(i: 0).setImm(MI.getOperand(i: 0).getImm() &
7505 static_cast<unsigned>(AMDGPU::SchedGroupMask::ALL));
7506 return BB;
7507 default:
7508 if (TII->isImage(MI) || TII->isMUBUF(MI)) {
7509 if (!MI.mayStore())
7510 AddMemOpInit(MI);
7511 return BB;
7512 }
7513 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, MBB: BB);
7514 }
7515}
7516
7517bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
7518 // This currently forces unfolding various combinations of fsub into fma with
7519 // free fneg'd operands. As long as we have fast FMA (controlled by
7520 // isFMAFasterThanFMulAndFAdd), we should perform these.
7521
7522 // When fma is quarter rate, for f64 where add / sub are at best half rate,
7523 // most of these combines appear to be cycle neutral but save on instruction
7524 // count / code size.
7525 return true;
7526}
7527
7528bool SITargetLowering::enableAggressiveFMAFusion(LLT Ty) const { return true; }
7529
7530EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
7531 EVT VT) const {
7532 if (!VT.isVector()) {
7533 return MVT::i1;
7534 }
7535 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i1, NumElements: VT.getVectorNumElements());
7536}
7537
7538MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
7539 // TODO: Should i16 be used always if legal? For now it would force VALU
7540 // shifts.
7541 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
7542}
7543
7544LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
7545 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
7546 ? Ty.changeElementSize(NewEltSize: 16)
7547 : Ty.changeElementSize(NewEltSize: 32);
7548}
7549
7550// Answering this is somewhat tricky and depends on the specific device which
7551// have different rates for fma or all f64 operations.
7552//
7553// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
7554// regardless of which device (although the number of cycles differs between
7555// devices), so it is always profitable for f64.
7556//
7557// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
7558// only on full rate devices. Normally, we should prefer selecting v_mad_f32
7559// which we can always do even without fused FP ops since it returns the same
7560// result as the separate operations and since it is always full
7561// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
7562// however does not support denormals, so we do report fma as faster if we have
7563// a fast fma device and require denormals.
7564//
7565bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT,
7566 DenormalFPEnv FPEnv) const {
7567 VT = VT.getScalarType();
7568 if (!VT.isSimple())
7569 return false;
7570
7571 switch (VT.getSimpleVT().SimpleTy) {
7572 case MVT::f32: {
7573 // If mad is not available this depends only on if f32 fma is full rate.
7574 if (!Subtarget->hasMadMacF32Insts())
7575 return Subtarget->hasFastFMAF32();
7576
7577 // Otherwise f32 mad is always full rate and returns the same result as
7578 // the separate operations so should be preferred over fma.
7579 // However does not support denormals.
7580 if (FPEnv.F32Mode != DenormalMode::getPreserveSign())
7581 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
7582
7583 // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
7584 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
7585 }
7586 case MVT::f64:
7587 return true;
7588 case MVT::f16:
7589 case MVT::bf16:
7590 return Subtarget->has16BitInsts() &&
7591 FPEnv.DefaultMode != DenormalMode::getPreserveSign();
7592 default:
7593 break;
7594 }
7595
7596 return false;
7597}
7598
7599bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7600 EVT VT) const {
7601 return isFMAFasterThanFMulAndFAdd(VT, FPEnv: getDenormalFPEnv(MF));
7602}
7603
7604bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
7605 Type *Ty) const {
7606 return isFMAFasterThanFMulAndFAdd(
7607 VT: getValueType(DL: F.getDataLayout(), Ty, /*AllowUnknown=*/true),
7608 FPEnv: F.getDenormalFPEnv());
7609}
7610
7611bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7612 LLT Ty) const {
7613 switch (Ty.getScalarSizeInBits()) {
7614 case 16:
7615 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f16);
7616 case 32:
7617 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f32);
7618 case 64:
7619 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f64);
7620 default:
7621 break;
7622 }
7623
7624 return false;
7625}
7626
7627bool SITargetLowering::isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const {
7628 // TODO: Check future ftz flag
7629 // v_mad_f32/v_mac_f32 do not support denormals.
7630 if (VT == MVT::f32)
7631 return Subtarget->hasMadMacF32Insts() &&
7632 FPEnv.F32Mode == DenormalMode::getPreserveSign();
7633 if (VT == MVT::f16)
7634 return Subtarget->hasMadF16() &&
7635 FPEnv.DefaultMode == DenormalMode::getPreserveSign();
7636
7637 return false;
7638}
7639
7640bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
7641 if (!Ty.isScalar())
7642 return false;
7643
7644 DenormalFPEnv FPEnv = getDenormalFPEnv(MF: *MI.getMF());
7645 if (Ty.getScalarSizeInBits() == 16)
7646 return isFMADLegal(VT: MVT::f16, FPEnv);
7647 if (Ty.getScalarSizeInBits() == 32)
7648 return isFMADLegal(VT: MVT::f32, FPEnv);
7649
7650 return false;
7651}
7652
7653bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
7654 const SDNode *N) const {
7655 return isFMADLegal(VT: N->getValueType(ResNo: 0),
7656 FPEnv: getDenormalFPEnv(MF: DAG.getMachineFunction()));
7657}
7658
7659bool SITargetLowering::isFMADLegal(const Function &F, Type *Ty) const {
7660 return isFMADLegal(VT: getValueType(DL: F.getDataLayout(), Ty: Ty->getScalarType(),
7661 /*AllowUnknown=*/true),
7662 FPEnv: F.getDenormalFPEnv());
7663}
7664
7665//===----------------------------------------------------------------------===//
7666// Custom DAG Lowering Operations
7667//===----------------------------------------------------------------------===//
7668
7669// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7670// wider vector type is legal.
7671SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
7672 SelectionDAG &DAG) const {
7673 unsigned Opc = Op.getOpcode();
7674 EVT VT = Op.getValueType();
7675 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7676
7677 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7678 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
7679
7680 SDLoc SL(Op);
7681
7682 // Forward any trailing scalar operands unchanged to both halves.
7683 SmallVector<SDValue, 2> LoOps = {Lo};
7684 SmallVector<SDValue, 2> HiOps = {Hi};
7685 auto TrailingOps = drop_begin(RangeOrContainer: Op->ops());
7686 LoOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7687 HiOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7688
7689 SDValue OpLo = DAG.getNode(Opcode: Opc, DL: SL, VT: LoVT, Ops: LoOps, Flags: Op->getFlags());
7690 SDValue OpHi = DAG.getNode(Opcode: Opc, DL: SL, VT: HiVT, Ops: HiOps, Flags: Op->getFlags());
7691
7692 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7693}
7694
7695// Enable lowering of ROTR for vxi32 types. This is a workaround for a
7696// regression whereby extra unnecessary instructions were added to codegen
7697// for rotr operations, casued by legalising v2i32 or. This resulted in extra
7698// instructions to extract the result from the vector.
7699SDValue SITargetLowering::lowerROTR(SDValue Op, SelectionDAG &DAG) const {
7700 [[maybe_unused]] EVT VT = Op.getValueType();
7701
7702 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
7703 VT == MVT::v16i32) &&
7704 "Unexpected ValueType.");
7705
7706 return DAG.UnrollVectorOp(N: Op.getNode());
7707}
7708
7709// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7710// wider vector type is legal.
7711SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
7712 SelectionDAG &DAG) const {
7713 unsigned Opc = Op.getOpcode();
7714 EVT VT = Op.getValueType();
7715 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7716
7717 auto [Lo0, Hi0] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7718 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7719
7720 SDLoc SL(Op);
7721
7722 SDValue OpLo =
7723 DAG.getNode(Opcode: Opc, DL: SL, VT: Lo0.getValueType(), N1: Lo0, N2: Lo1, Flags: Op->getFlags());
7724 SDValue OpHi =
7725 DAG.getNode(Opcode: Opc, DL: SL, VT: Hi0.getValueType(), N1: Hi0, N2: Hi1, Flags: Op->getFlags());
7726
7727 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7728}
7729
7730SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
7731 SelectionDAG &DAG) const {
7732 unsigned Opc = Op.getOpcode();
7733 EVT VT = Op.getValueType();
7734 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7735
7736 SDValue Op0 = Op.getOperand(i: 0);
7737 SDValue Lo0, Hi0;
7738 if (Op0.getValueType().isVector())
7739 std::tie(args&: Lo0, args&: Hi0) = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7740 else
7741 Lo0 = Hi0 = DAG.getFreeze(V: Op0);
7742
7743 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7744 auto [Lo2, Hi2] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 2);
7745
7746 SDLoc SL(Op);
7747 auto ResVT = DAG.GetSplitDestVTs(VT);
7748
7749 SDValue OpLo =
7750 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.first, N1: Lo0, N2: Lo1, N3: Lo2, Flags: Op->getFlags());
7751 SDValue OpHi =
7752 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.second, N1: Hi0, N2: Hi1, N3: Hi2, Flags: Op->getFlags());
7753
7754 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7755}
7756
7757SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
7758 switch (Op.getOpcode()) {
7759 default:
7760 return AMDGPUTargetLowering::LowerOperation(Op, DAG);
7761 case ISD::BRCOND:
7762 return LowerBRCOND(Op, DAG);
7763 case ISD::RETURNADDR:
7764 return LowerRETURNADDR(Op, DAG);
7765 case ISD::FrameIndex:
7766 return lowerFrameIndex(Op, DAG);
7767 case ISD::SPONENTRY:
7768 return LowerSPONENTRY(Op, DAG);
7769 case ISD::LOAD: {
7770 SDValue Result = LowerLOAD(Op, DAG);
7771 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
7772 "Load should return a value and a chain");
7773 return Result;
7774 }
7775 case ISD::FSQRT: {
7776 EVT VT = Op.getValueType();
7777 if (VT == MVT::f32)
7778 return lowerFSQRTF32(Op, DAG);
7779 if (VT == MVT::f64)
7780 return lowerFSQRTF64(Op, DAG);
7781 return SDValue();
7782 }
7783 case ISD::FSIN:
7784 case ISD::FCOS:
7785 return LowerTrig(Op, DAG);
7786 case ISD::SELECT:
7787 return LowerSELECT(Op, DAG);
7788 case ISD::FDIV:
7789 return LowerFDIV(Op, DAG);
7790 case ISD::FFREXP:
7791 return LowerFFREXP(Op, DAG);
7792 case ISD::ATOMIC_CMP_SWAP:
7793 return LowerATOMIC_CMP_SWAP(Op, DAG);
7794 case ISD::STORE:
7795 return LowerSTORE(Op, DAG);
7796 case ISD::GlobalAddress: {
7797 MachineFunction &MF = DAG.getMachineFunction();
7798 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
7799 return LowerGlobalAddress(MFI, Op, DAG);
7800 }
7801 case ISD::BlockAddress:
7802 return LowerBlockAddress(Op, DAG);
7803 case ISD::ExternalSymbol:
7804 return LowerExternalSymbol(Op, DAG);
7805 case ISD::INTRINSIC_WO_CHAIN:
7806 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7807 case ISD::CONVERT_FROM_ARBITRARY_FP:
7808 return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
7809 case ISD::CONVERT_TO_ARBITRARY_FP:
7810 return LowerCONVERT_TO_ARBITRARY_FP(Op, DAG);
7811 case ISD::INTRINSIC_W_CHAIN:
7812 return LowerINTRINSIC_W_CHAIN(Op, DAG);
7813 case ISD::INTRINSIC_VOID:
7814 return LowerINTRINSIC_VOID(Op, DAG);
7815 case ISD::ADDRSPACECAST:
7816 return lowerADDRSPACECAST(Op, DAG);
7817 case ISD::INSERT_SUBVECTOR:
7818 return lowerINSERT_SUBVECTOR(Op, DAG);
7819 case ISD::INSERT_VECTOR_ELT:
7820 return lowerINSERT_VECTOR_ELT(Op, DAG);
7821 case ISD::EXTRACT_VECTOR_ELT:
7822 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7823 case ISD::VECTOR_SHUFFLE:
7824 return lowerVECTOR_SHUFFLE(Op, DAG);
7825 case ISD::SCALAR_TO_VECTOR:
7826 return lowerSCALAR_TO_VECTOR(Op, DAG);
7827 case ISD::BUILD_VECTOR:
7828 return lowerBUILD_VECTOR(Op, DAG);
7829 case ISD::FP_ROUND:
7830 case ISD::STRICT_FP_ROUND:
7831 return lowerFP_ROUND(Op, DAG);
7832 case ISD::TRAP:
7833 return lowerTRAP(Op, DAG);
7834 case ISD::DEBUGTRAP:
7835 return lowerDEBUGTRAP(Op, DAG);
7836 case ISD::ABS:
7837 case ISD::FABS:
7838 case ISD::FNEG:
7839 case ISD::FCANONICALIZE:
7840 case ISD::BSWAP:
7841 return splitUnaryVectorOp(Op, DAG);
7842 case ISD::FP_TO_SINT_SAT:
7843 case ISD::FP_TO_UINT_SAT:
7844 if (Op.getValueType().isVector() && Op.getValueType() != MVT::v2i16 &&
7845 Op.getOperand(i: 0).getValueType().getScalarType() == MVT::f32)
7846 return splitUnaryVectorOp(Op, DAG);
7847 return LowerFP_TO_INT_SAT(Op, DAG);
7848 case ISD::FSUB:
7849 if (Op.getValueType() == MVT::bf16) {
7850 // Custom expansion:
7851 // fsub bf16 %a, %b -> fadd v2bf16(widen %a), fneg v2bf16(widen %b)
7852 // Then extract back to bf16.
7853 //
7854 // We create fneg on v2bf16 (not bf16) so the instruction selector can
7855 // fold the negation into the packed add's neg_lo/neg_hi modifiers,
7856 // generating a single v_pk_add_bf16 instruction. If we negate bf16 first,
7857 // it becomes a separate v_xor instruction before widening.
7858 SDLoc DL(Op);
7859 SDValue Op0 = Op.getOperand(i: 0);
7860 SDValue Op1 = Op.getOperand(i: 1);
7861
7862 // Widen both operands to v2bf16
7863 SDValue Vec0 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Op0);
7864 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Op1);
7865
7866 // Create FNEG v2bf16 for the second operand
7867 SDValue NegVec1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::v2bf16, Operand: Vec1);
7868
7869 // Perform FADD v2bf16
7870 SDValue Result = DAG.getNode(Opcode: ISD::FADD, DL, VT: MVT::v2bf16, N1: Vec0, N2: NegVec1);
7871
7872 // Extract element 0 back to bf16
7873 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::bf16, N1: Result,
7874 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
7875 }
7876 return SDValue();
7877 case ISD::FMINNUM:
7878 case ISD::FMAXNUM:
7879 return lowerFMINNUM_FMAXNUM(Op, DAG);
7880 case ISD::FMINIMUMNUM:
7881 case ISD::FMAXIMUMNUM:
7882 return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
7883 case ISD::FLDEXP:
7884 case ISD::STRICT_FLDEXP:
7885 return lowerFLDEXP(Op, DAG);
7886 case ISD::FMA:
7887 return splitTernaryVectorOp(Op, DAG);
7888 case ISD::FP_TO_SINT:
7889 case ISD::FP_TO_UINT:
7890 if (Subtarget->hasVCvtPkIU16F32() && Op.getValueType() == MVT::i16 &&
7891 Op.getOperand(i: 0).getValueType() == MVT::f32) {
7892 // Make f32->i16 legal so we can select V_CVT_PK_[IU]16_F32.
7893 return Op;
7894 }
7895 return LowerFP_TO_INT(Op, DAG);
7896 case ISD::SHL:
7897 case ISD::SRA:
7898 case ISD::SRL:
7899 case ISD::ADD:
7900 case ISD::SUB:
7901 case ISD::SMIN:
7902 case ISD::SMAX:
7903 case ISD::UMIN:
7904 case ISD::UMAX:
7905 case ISD::FMINNUM_IEEE:
7906 case ISD::FMAXNUM_IEEE:
7907 case ISD::FMINIMUM:
7908 case ISD::FMAXIMUM:
7909 case ISD::UADDSAT:
7910 case ISD::USUBSAT:
7911 case ISD::SADDSAT:
7912 case ISD::SSUBSAT:
7913 case ISD::FADD:
7914 case ISD::FMUL:
7915 return splitBinaryVectorOp(Op, DAG);
7916 case ISD::FCOPYSIGN:
7917 return lowerFCOPYSIGN(Op, DAG);
7918 case ISD::MUL:
7919 return lowerMUL(Op, DAG);
7920 case ISD::SMULO:
7921 case ISD::UMULO:
7922 return lowerXMULO(Op, DAG);
7923 case ISD::SMUL_LOHI:
7924 case ISD::UMUL_LOHI:
7925 return lowerXMUL_LOHI(Op, DAG);
7926 case ISD::DYNAMIC_STACKALLOC:
7927 return LowerDYNAMIC_STACKALLOC(Op, DAG);
7928 case ISD::STACKSAVE:
7929 return LowerSTACKSAVE(Op, DAG);
7930 case ISD::GET_ROUNDING:
7931 return lowerGET_ROUNDING(Op, DAG);
7932 case ISD::SET_ROUNDING:
7933 return lowerSET_ROUNDING(Op, DAG);
7934 case ISD::PREFETCH:
7935 return lowerPREFETCH(Op, DAG);
7936 case ISD::FP_EXTEND:
7937 case ISD::STRICT_FP_EXTEND:
7938 return lowerFP_EXTEND(Op, DAG);
7939 case ISD::GET_FPENV:
7940 return lowerGET_FPENV(Op, DAG);
7941 case ISD::SET_FPENV:
7942 return lowerSET_FPENV(Op, DAG);
7943 case ISD::ROTR:
7944 return lowerROTR(Op, DAG);
7945 case ISD::INLINEASM:
7946 return LowerINLINEASM(Op, DAG);
7947 }
7948 return SDValue();
7949}
7950
7951// TFE results are dword granular: value dwords followed by one status dword.
7952static std::pair<SDValue, SDValue>
7953splitTFEValueAndStatus(SDValue Op, EVT VT, const SDLoc &DL, SelectionDAG &DAG) {
7954 LLVMContext &C = *DAG.getContext();
7955 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
7956 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
7957 N2: DAG.getVectorIdxConstant(Val: NumValueDWords, DL));
7958 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
7959 SDValue ValueDWords =
7960 NumValueDWords == 1
7961 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op, N2: ZeroIdx)
7962 : DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL,
7963 VT: EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumValueDWords), N1: Op,
7964 N2: ZeroIdx);
7965 if (!VT.isVector() && VT.getSizeInBits() < 32)
7966 ValueDWords =
7967 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VT.changeTypeToInteger(), Operand: ValueDWords);
7968 return {DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ValueDWords), Status};
7969}
7970
7971// Used for D16: Casts the result of an instruction into the right vector,
7972// packs values if loads return unpacked values.
7973static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
7974 const SDLoc &DL, SelectionDAG &DAG,
7975 bool Unpacked) {
7976 if (!LoadVT.isVector())
7977 return Result;
7978
7979 // Cast back to the original packed type or to a larger type that is a
7980 // multiple of 32 bit for D16. Widening the return type is a required for
7981 // legalization.
7982 EVT FittingLoadVT = LoadVT;
7983 if ((LoadVT.getVectorNumElements() % 2) == 1) {
7984 FittingLoadVT =
7985 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7986 NumElements: LoadVT.getVectorNumElements() + 1);
7987 }
7988
7989 if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
7990 // Truncate to v2i16/v4i16.
7991 EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
7992
7993 // Workaround legalizer not scalarizing truncate after vector op
7994 // legalization but not creating intermediate vector trunc.
7995 SmallVector<SDValue, 4> Elts;
7996 DAG.ExtractVectorElements(Op: Result, Args&: Elts);
7997 for (SDValue &Elt : Elts)
7998 Elt = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Elt);
7999
8000 // Pad illegal v1i16/v3fi6 to v4i16
8001 if ((LoadVT.getVectorNumElements() % 2) == 1)
8002 Elts.push_back(Elt: DAG.getPOISON(VT: MVT::i16));
8003
8004 Result = DAG.getBuildVector(VT: IntLoadVT, DL, Ops: Elts);
8005
8006 // Bitcast to original type (v2f16/v4f16).
8007 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
8008 }
8009
8010 // Cast back to the original packed type.
8011 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
8012}
8013
8014SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
8015 SelectionDAG &DAG,
8016 ArrayRef<SDValue> Ops,
8017 bool IsIntrinsic) const {
8018 SDLoc DL(M);
8019
8020 bool IsTFE = M->getNumValues() == 3;
8021 bool Unpacked = Subtarget->hasUnpackedD16VMem();
8022 EVT LoadVT = M->getValueType(ResNo: 0);
8023
8024 EVT EquivLoadVT = LoadVT;
8025 if (LoadVT.isVector()) {
8026 if (Unpacked) {
8027 EquivLoadVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
8028 NumElements: LoadVT.getVectorNumElements());
8029 } else if ((LoadVT.getVectorNumElements() % 2) == 1) {
8030 // Widen v3f16 to legal type
8031 EquivLoadVT =
8032 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
8033 NumElements: LoadVT.getVectorNumElements() + 1);
8034 }
8035 }
8036
8037 if (IsTFE) {
8038 unsigned NumValueDWords = divideCeil(Numerator: EquivLoadVT.getSizeInBits(), Denominator: 32);
8039 EVT LoadDWordsVT =
8040 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumValueDWords + 1);
8041 SDVTList VTList = DAG.getVTList(VT1: LoadDWordsVT, VT2: MVT::Other);
8042 SDValue Load = DAG.getMemIntrinsicNode(
8043 Opcode, dl: DL, VTList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
8044 auto [Value, Status] = splitTFEValueAndStatus(Op: Load, VT: EquivLoadVT, DL, DAG);
8045 SDValue Adjusted =
8046 adjustLoadValueTypeImpl(Result: Value, LoadVT, DL, DAG, Unpacked);
8047 return DAG.getMergeValues(Ops: {Adjusted, Status, Load.getValue(R: 1)}, dl: DL);
8048 }
8049
8050 // Change from v4f16/v2f16 to EquivLoadVT.
8051 SDVTList VTList = DAG.getVTList(VT1: EquivLoadVT, VT2: MVT::Other);
8052
8053 SDValue Load = DAG.getMemIntrinsicNode(
8054 Opcode: IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, dl: DL, VTList, Ops,
8055 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
8056
8057 SDValue Adjusted = adjustLoadValueTypeImpl(Result: Load, LoadVT, DL, DAG, Unpacked);
8058
8059 return DAG.getMergeValues(Ops: {Adjusted, Load.getValue(R: 1)}, dl: DL);
8060}
8061
8062SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
8063 SelectionDAG &DAG,
8064 ArrayRef<SDValue> Ops) const {
8065 SDLoc DL(M);
8066 EVT LoadVT = M->getValueType(ResNo: 0);
8067 EVT EltType = LoadVT.getScalarType();
8068 EVT IntVT = LoadVT.changeTypeToInteger();
8069
8070 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
8071
8072 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
8073 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8074 DAG.getMachineFunction().getFunction(),
8075 "unsupported sub-dword format buffer load", DL.getDebugLoc()));
8076 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: LoadVT), M->getOperand(Num: 0)}, dl: DL);
8077 }
8078
8079 assert(M->getNumValues() == 2 || M->getNumValues() == 3);
8080 bool IsTFE = M->getNumValues() == 3;
8081
8082 if (IsD16 && IsTFE && !Subtarget->hasBufferTFEFormatD16()) {
8083 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8084 DAG.getMachineFunction().getFunction(),
8085 "TFE D16 format buffer load is not supported on this GPU",
8086 DL.getDebugLoc()));
8087 return DAG.getErrorMergeValues(ResultTypes: {M->value_begin(), M->value_end()},
8088 Chain: M->getOperand(Num: 0), dl: DL);
8089 }
8090
8091 unsigned Opc = IsD16 ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_D16_TFE
8092 : AMDGPUISD::BUFFER_LOAD_FORMAT_D16)
8093 : IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
8094 : AMDGPUISD::BUFFER_LOAD_FORMAT)
8095 : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
8096 : AMDGPUISD::BUFFER_LOAD;
8097
8098 if (IsD16)
8099 return adjustLoadValueType(Opcode: Opc, M, DAG, Ops);
8100
8101 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
8102 if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
8103 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, MMO: M->getMemOperand(),
8104 IsTFE);
8105
8106 if (isTypeLegal(VT: LoadVT)) {
8107 return getMemIntrinsicNode(Opcode: Opc, DL, VTList: M->getVTList(), Ops, MemVT: IntVT,
8108 MMO: M->getMemOperand(), DAG);
8109 }
8110
8111 EVT CastVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: LoadVT);
8112 SDVTList VTList = IsTFE ? DAG.getVTList(VT1: CastVT, VT2: MVT::i32, VT3: MVT::Other)
8113 : DAG.getVTList(VT1: CastVT, VT2: MVT::Other);
8114 SDValue MemNode = getMemIntrinsicNode(Opcode: Opc, DL, VTList, Ops, MemVT: CastVT,
8115 MMO: M->getMemOperand(), DAG);
8116 SDValue Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: MemNode);
8117 if (IsTFE)
8118 return DAG.getMergeValues(Ops: {Data, MemNode.getValue(R: 1), MemNode.getValue(R: 2)},
8119 dl: DL);
8120 return DAG.getMergeValues(Ops: {Data, MemNode.getValue(R: 1)}, dl: DL);
8121}
8122
8123static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
8124 SelectionDAG &DAG) {
8125 EVT VT = N->getValueType(ResNo: 0);
8126 SDValue Src = N->getOperand(Num: 1);
8127 SDLoc SL(N);
8128
8129 if (Src.getOpcode() == ISD::SETCC) {
8130 SDValue Op0 = Src.getOperand(i: 0);
8131 SDValue Op1 = Src.getOperand(i: 1);
8132 // Need to expand bfloat to float for comparison (setcc).
8133 if (Op0.getValueType() == MVT::bf16) {
8134 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op0);
8135 Op1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op1);
8136 }
8137 // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
8138 return DAG.getNode(Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: Op0, N2: Op1, N3: Src.getOperand(i: 2));
8139 }
8140 if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Val&: Src)) {
8141 // (ballot 0) -> 0
8142 if (Arg->isZero())
8143 return DAG.getConstant(Val: 0, DL: SL, VT);
8144
8145 // (ballot 1) -> EXEC/EXEC_LO
8146 if (Arg->isOne()) {
8147 Register Exec;
8148 if (VT.getScalarSizeInBits() == 32)
8149 Exec = AMDGPU::EXEC_LO;
8150 else if (VT.getScalarSizeInBits() == 64)
8151 Exec = AMDGPU::EXEC;
8152 else
8153 return SDValue();
8154
8155 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: Exec, VT);
8156 }
8157 }
8158
8159 // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
8160 // ISD::SETNE)
8161 return DAG.getNode(
8162 Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: DAG.getZExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32),
8163 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), N3: DAG.getCondCode(Cond: ISD::SETNE));
8164}
8165
8166static SDValue lowerBFEIntrinsic(SDValue Op, SelectionDAG &DAG,
8167 Intrinsic::ID IntrinsicID) {
8168 bool Signed = IntrinsicID == Intrinsic::amdgcn_sbfe;
8169 SDLoc DL(Op);
8170 EVT VT = Op.getValueType();
8171 SDValue Src = Op.getOperand(i: 1);
8172 SDValue Offset = Op.getOperand(i: 2);
8173 SDValue Width = Op.getOperand(i: 3);
8174
8175 if (VT != MVT::i32) {
8176 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
8177 DAG.getMachineFunction().getFunction(),
8178 Twine(Intrinsic::getBaseName(id: IntrinsicID)) + " only supports i32",
8179 DL.getDebugLoc()));
8180 return DAG.getPOISON(VT);
8181 }
8182
8183 return DAG.getNode(Opcode: Signed ? AMDGPUISD::BFE_I32 : AMDGPUISD::BFE_U32, DL, VT,
8184 N1: Src, N2: Offset, N3: Width);
8185}
8186
8187static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
8188 EVT VT);
8189
8190static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
8191 SelectionDAG &DAG) {
8192 EVT VT = N->getValueType(ResNo: 0);
8193 unsigned ValSize = VT.getSizeInBits();
8194 unsigned IID = N->getConstantOperandVal(Num: 0);
8195 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
8196 IID == Intrinsic::amdgcn_permlanex16;
8197 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
8198 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
8199 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
8200 IID == Intrinsic::amdgcn_permlane_up ||
8201 IID == Intrinsic::amdgcn_permlane_down ||
8202 IID == Intrinsic::amdgcn_permlane_xor;
8203 SDLoc SL(N);
8204 MVT IntVT = MVT::getIntegerVT(BitWidth: ValSize);
8205 const GCNSubtarget *ST = TLI.getSubtarget();
8206
8207 unsigned SplitSize = 32;
8208 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
8209 ST->hasDPALU_DPP() &&
8210 AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: N->getConstantOperandVal(Num: 3)))
8211 SplitSize = 64;
8212
8213 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
8214 SDValue Src2, MVT ValT) -> SDValue {
8215 SmallVector<SDValue, 8> Operands;
8216 switch (IID) {
8217 case Intrinsic::amdgcn_permlane16:
8218 case Intrinsic::amdgcn_permlanex16:
8219 case Intrinsic::amdgcn_update_dpp:
8220 Operands.push_back(Elt: N->getOperand(Num: 6));
8221 Operands.push_back(Elt: N->getOperand(Num: 5));
8222 Operands.push_back(Elt: N->getOperand(Num: 4));
8223 [[fallthrough]];
8224 case Intrinsic::amdgcn_writelane:
8225 case Intrinsic::amdgcn_permlane_bcast:
8226 case Intrinsic::amdgcn_permlane_up:
8227 case Intrinsic::amdgcn_permlane_down:
8228 case Intrinsic::amdgcn_permlane_xor:
8229 Operands.push_back(Elt: Src2);
8230 [[fallthrough]];
8231 case Intrinsic::amdgcn_readlane:
8232 case Intrinsic::amdgcn_set_inactive:
8233 case Intrinsic::amdgcn_set_inactive_chain_arg:
8234 case Intrinsic::amdgcn_mov_dpp8:
8235 Operands.push_back(Elt: Src1);
8236 [[fallthrough]];
8237 case Intrinsic::amdgcn_readfirstlane:
8238 case Intrinsic::amdgcn_permlane64:
8239 Operands.push_back(Elt: Src0);
8240 break;
8241 default:
8242 llvm_unreachable("unhandled lane op");
8243 }
8244
8245 Operands.push_back(Elt: DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32));
8246 std::reverse(first: Operands.begin(), last: Operands.end());
8247
8248 if (SDNode *GL = N->getGluedNode()) {
8249 assert(GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8250 GL = GL->getOperand(Num: 0).getNode();
8251 Operands.push_back(Elt: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8252 Operand: SDValue(GL, 0)));
8253 }
8254
8255 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: ValT, Ops: Operands);
8256 };
8257
8258 SDValue Src0 = N->getOperand(Num: 1);
8259 SDValue Src1, Src2;
8260 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
8261 IID == Intrinsic::amdgcn_mov_dpp8 ||
8262 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
8263 IsPermlaneShuffle) {
8264 Src1 = N->getOperand(Num: 2);
8265 if (IID == Intrinsic::amdgcn_writelane ||
8266 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
8267 IsPermlaneShuffle)
8268 Src2 = N->getOperand(Num: 3);
8269 }
8270
8271 if (ValSize == SplitSize) {
8272 // Already legal
8273 return SDValue();
8274 }
8275
8276 if (ValSize < 32) {
8277 bool IsFloat = VT.isFloatingPoint();
8278 Src0 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src0) : Src0,
8279 DL: SL, VT: MVT::i32);
8280
8281 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
8282 Src1 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src1) : Src1,
8283 DL: SL, VT: MVT::i32);
8284 }
8285
8286 if (IID == Intrinsic::amdgcn_writelane) {
8287 Src2 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src2) : Src2,
8288 DL: SL, VT: MVT::i32);
8289 }
8290
8291 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
8292 SDValue Trunc = DAG.getAnyExtOrTrunc(Op: LaneOp, DL: SL, VT: IntVT);
8293 return IsFloat ? DAG.getBitcast(VT, V: Trunc) : Trunc;
8294 }
8295
8296 if (ValSize % SplitSize != 0)
8297 return SDValue();
8298
8299 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
8300 EVT VT = N->getValueType(ResNo: 0);
8301 unsigned NE = VT.getVectorNumElements();
8302 EVT EltVT = VT.getVectorElementType();
8303 SmallVector<SDValue, 8> Scalars;
8304 unsigned NumOperands = N->getNumOperands();
8305 SmallVector<SDValue, 4> Operands(NumOperands);
8306 SDNode *GL = N->getGluedNode();
8307
8308 // only handle convergencectrl_glue
8309 assert(!GL || GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8310
8311 for (unsigned i = 0; i != NE; ++i) {
8312 for (unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
8313 ++j) {
8314 SDValue Operand = N->getOperand(Num: j);
8315 EVT OperandVT = Operand.getValueType();
8316 if (OperandVT.isVector()) {
8317 // A vector operand; extract a single element.
8318 EVT OperandEltVT = OperandVT.getVectorElementType();
8319 Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: OperandEltVT,
8320 N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: SL));
8321 } else {
8322 // A scalar operand; just use it as is.
8323 Operands[j] = Operand;
8324 }
8325 }
8326
8327 if (GL)
8328 Operands[NumOperands - 1] =
8329 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8330 Operand: SDValue(GL->getOperand(Num: 0).getNode(), 0));
8331
8332 Scalars.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: EltVT, Ops: Operands));
8333 }
8334
8335 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NE);
8336 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops: Scalars);
8337 };
8338
8339 if (VT.isVector()) {
8340 switch (MVT::SimpleValueType EltTy =
8341 VT.getVectorElementType().getSimpleVT().SimpleTy) {
8342 case MVT::i32:
8343 case MVT::f32:
8344 if (SplitSize == 32) {
8345 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT());
8346 return unrollLaneOp(LaneOp.getNode());
8347 }
8348 [[fallthrough]];
8349 case MVT::i16:
8350 case MVT::f16:
8351 case MVT::bf16: {
8352 unsigned SubVecNumElt =
8353 SplitSize / VT.getVectorElementType().getSizeInBits();
8354 MVT SubVecVT = MVT::getVectorVT(VT: EltTy, NumElements: SubVecNumElt);
8355 SmallVector<SDValue, 4> Pieces;
8356 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
8357 for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
8358 Src0SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src0,
8359 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8360
8361 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
8362 IsPermLane16) {
8363 Src1SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src1,
8364 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8365
8366 Pieces.push_back(
8367 Elt: createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
8368 } else if (IID == Intrinsic::amdgcn_writelane) {
8369 Src2SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src2,
8370 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8371 Pieces.push_back(
8372 Elt: createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
8373 } else {
8374 Pieces.push_back(Elt: createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
8375 }
8376
8377 EltIdx += SubVecNumElt;
8378 }
8379 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, Ops: Pieces);
8380 }
8381 default:
8382 // Handle all other cases by bitcasting to i32 vectors
8383 break;
8384 }
8385 }
8386
8387 MVT VecVT =
8388 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplitSize), NumElements: ValSize / SplitSize);
8389 Src0 = DAG.getBitcast(VT: VecVT, V: Src0);
8390
8391 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
8392 Src1 = DAG.getBitcast(VT: VecVT, V: Src1);
8393
8394 if (IID == Intrinsic::amdgcn_writelane)
8395 Src2 = DAG.getBitcast(VT: VecVT, V: Src2);
8396
8397 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
8398 SDValue UnrolledLaneOp = unrollLaneOp(LaneOp.getNode());
8399 return DAG.getBitcast(VT, V: UnrolledLaneOp);
8400}
8401
8402static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N,
8403 SelectionDAG &DAG) {
8404 EVT VT = N->getValueType(ResNo: 0);
8405
8406 if (VT.getSizeInBits() != 32)
8407 return SDValue();
8408
8409 SDLoc SL(N);
8410
8411 SDValue Value = N->getOperand(Num: 1);
8412 SDValue Index = N->getOperand(Num: 2);
8413
8414 // ds_bpermute requires index to be multiplied by 4
8415 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: 2, VT: MVT::i32, DL: SL);
8416 SDValue ShiftedIndex =
8417 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: Index.getValueType(), N1: Index, N2: ShiftAmount);
8418
8419 // Intrinsics will require i32 to operate on
8420 SDValue ValueI32 = DAG.getBitcast(VT: MVT::i32, V: Value);
8421
8422 auto MakeIntrinsic = [&DAG, &SL](unsigned IID, MVT RetVT,
8423 SmallVector<SDValue> IntrinArgs) -> SDValue {
8424 SmallVector<SDValue> Operands(1);
8425 Operands[0] = DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32);
8426 Operands.append(RHS: IntrinArgs);
8427 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: RetVT, Ops: Operands);
8428 };
8429
8430 // If we can bpermute across the whole wave, then just do that
8431 if (TLI.getSubtarget()->supportsWaveWideBPermute()) {
8432 SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8433 {ShiftedIndex, ValueI32});
8434 return DAG.getBitcast(VT, V: BPermute);
8435 }
8436
8437 assert(TLI.getSubtarget()->isWave64());
8438
8439 // Otherwise, we need to make use of whole wave mode
8440 SDValue PoisonVal = DAG.getPOISON(VT: ValueI32->getValueType(ResNo: 0));
8441
8442 // Set inactive lanes to poison
8443 SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8444 {ValueI32, PoisonVal});
8445 SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8446 {ShiftedIndex, PoisonVal});
8447
8448 SDValue Swapped =
8449 MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
8450
8451 // Get permutation of each half, then we'll select which one to use
8452 SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8453 {WWMIndex, WWMValue});
8454 SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
8455 MVT::i32, {WWMIndex, Swapped});
8456 SDValue BPermOtherHalfWWM =
8457 MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
8458
8459 // Select which side to take the permute from
8460 SDValue ThreadIDMask = DAG.getAllOnesConstant(DL: SL, VT: MVT::i32);
8461 // We can get away with only using mbcnt_lo here since we're only
8462 // trying to detect which side of 32 each lane is on, and mbcnt_lo
8463 // returns 32 for lanes 32-63.
8464 SDValue ThreadID =
8465 MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
8466 {ThreadIDMask, DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32)});
8467
8468 SDValue SameOrOtherHalf =
8469 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32,
8470 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: ThreadID, N2: Index),
8471 N2: DAG.getTargetConstant(Val: 32, DL: SL, VT: MVT::i32));
8472 SDValue UseSameHalf =
8473 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SameOrOtherHalf,
8474 RHS: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond: ISD::SETEQ);
8475 SDValue Result = DAG.getSelect(DL: SL, VT: MVT::i32, Cond: UseSameHalf, LHS: BPermSameHalf,
8476 RHS: BPermOtherHalfWWM);
8477 return DAG.getBitcast(VT, V: Result);
8478}
8479
8480void SITargetLowering::ReplaceNodeResults(SDNode *N,
8481 SmallVectorImpl<SDValue> &Results,
8482 SelectionDAG &DAG) const {
8483 switch (N->getOpcode()) {
8484 case ISD::INSERT_VECTOR_ELT: {
8485 if (SDValue Res = lowerINSERT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8486 Results.push_back(Elt: Res);
8487 return;
8488 }
8489 case ISD::EXTRACT_VECTOR_ELT: {
8490 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8491 Results.push_back(Elt: Res);
8492 return;
8493 }
8494 case ISD::CONVERT_TO_ARBITRARY_FP: {
8495 if (SDValue Res = LowerCONVERT_TO_ARBITRARY_FP(Op: SDValue(N, 0), DAG))
8496 Results.push_back(Elt: Res);
8497 return;
8498 }
8499 case ISD::INTRINSIC_WO_CHAIN: {
8500 unsigned IID = N->getConstantOperandVal(Num: 0);
8501 switch (IID) {
8502 case Intrinsic::amdgcn_wave_reduce_min:
8503 case Intrinsic::amdgcn_wave_reduce_umin:
8504 case Intrinsic::amdgcn_wave_reduce_max:
8505 case Intrinsic::amdgcn_wave_reduce_umax:
8506 case Intrinsic::amdgcn_wave_reduce_add:
8507 case Intrinsic::amdgcn_wave_reduce_sub:
8508 case Intrinsic::amdgcn_wave_reduce_and:
8509 case Intrinsic::amdgcn_wave_reduce_or:
8510 case Intrinsic::amdgcn_wave_reduce_xor: {
8511 EVT VT = N->getValueType(ResNo: 0);
8512 if (isTypeLegal(VT))
8513 return;
8514 SDLoc SL(N);
8515 bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
8516 IID == Intrinsic::amdgcn_wave_reduce_max ||
8517 IID == Intrinsic::amdgcn_wave_reduce_add ||
8518 IID == Intrinsic::amdgcn_wave_reduce_sub;
8519 unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8520 SDValue ExtSrc = DAG.getNode(Opcode: ExtOpc, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 1));
8521 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
8522 N1: N->getOperand(Num: 0), N2: ExtSrc, N3: N->getOperand(Num: 2));
8523 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Result));
8524 return;
8525 }
8526 case Intrinsic::amdgcn_make_buffer_rsrc:
8527 Results.push_back(Elt: lowerPointerAsRsrcIntrin(Op: N, DAG));
8528 return;
8529 case Intrinsic::amdgcn_cvt_pkrtz: {
8530 SDValue Src0 = N->getOperand(Num: 1);
8531 SDValue Src1 = N->getOperand(Num: 2);
8532 SDLoc SL(N);
8533 SDValue Cvt =
8534 DAG.getNode(Opcode: AMDGPUISD::CVT_PKRTZ_F16_F32, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8535 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Cvt));
8536 return;
8537 }
8538 case Intrinsic::amdgcn_cvt_pknorm_i16:
8539 case Intrinsic::amdgcn_cvt_pknorm_u16:
8540 case Intrinsic::amdgcn_cvt_pk_i16:
8541 case Intrinsic::amdgcn_cvt_pk_u16: {
8542 SDValue Src0 = N->getOperand(Num: 1);
8543 SDValue Src1 = N->getOperand(Num: 2);
8544 SDLoc SL(N);
8545 unsigned Opcode;
8546
8547 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
8548 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
8549 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
8550 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
8551 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
8552 Opcode = AMDGPUISD::CVT_PK_I16_I32;
8553 else
8554 Opcode = AMDGPUISD::CVT_PK_U16_U32;
8555
8556 EVT VT = N->getValueType(ResNo: 0);
8557 if (isTypeLegal(VT))
8558 Results.push_back(Elt: DAG.getNode(Opcode, DL: SL, VT, N1: Src0, N2: Src1));
8559 else {
8560 SDValue Cvt = DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8561 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: Cvt));
8562 }
8563 return;
8564 }
8565 case Intrinsic::amdgcn_s_buffer_load: {
8566 SDValue Op = SDValue(N, 0);
8567 EVT VT = Op.getValueType();
8568 Results.push_back(Elt: lowerSBuffer(VT, MemVT: VT, DL: SDLoc(Op), Chain: DAG.getEntryNode(),
8569 Rsrc: Op.getOperand(i: 1), Offset: Op.getOperand(i: 2),
8570 CachePolicy: Op.getOperand(i: 3), DAG));
8571 return;
8572 }
8573 case Intrinsic::amdgcn_dead: {
8574 for (unsigned I = 0, E = N->getNumValues(); I < E; ++I)
8575 Results.push_back(Elt: DAG.getPOISON(VT: N->getValueType(ResNo: I)));
8576 return;
8577 }
8578 }
8579 break;
8580 }
8581 case ISD::INTRINSIC_W_CHAIN: {
8582 if (N->getConstantOperandVal(Num: 1) != Intrinsic::amdgcn_ptr_s_buffer_load &&
8583 N->getValueType(ResNo: 0).isSimple() &&
8584 SBufferLoadDiagnosticVTs[N->getSimpleValueType(ResNo: 0).SimpleTy])
8585 break;
8586 if (SDValue Res = LowerINTRINSIC_W_CHAIN(Op: SDValue(N, 0), DAG)) {
8587 if (Res.getOpcode() == ISD::MERGE_VALUES) {
8588 // FIXME: Hacky
8589 for (unsigned I = 0; I < Res.getNumOperands(); I++) {
8590 Results.push_back(Elt: Res.getOperand(i: I));
8591 }
8592 } else {
8593 for (unsigned I = 0; I < N->getNumValues(); ++I)
8594 Results.push_back(Elt: Res.getValue(R: I));
8595 }
8596 return;
8597 }
8598
8599 break;
8600 }
8601 case ISD::SELECT: {
8602 SDLoc SL(N);
8603 EVT VT = N->getValueType(ResNo: 0);
8604 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT);
8605 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 1));
8606 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 2));
8607
8608 EVT SelectVT = NewVT;
8609 if (NewVT.bitsLT(VT: MVT::i32)) {
8610 LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: LHS);
8611 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: RHS);
8612 SelectVT = MVT::i32;
8613 }
8614
8615 SDValue NewSelect =
8616 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: SelectVT, N1: N->getOperand(Num: 0), N2: LHS, N3: RHS);
8617
8618 if (NewVT != SelectVT)
8619 NewSelect = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: NewVT, Operand: NewSelect);
8620 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewSelect));
8621 return;
8622 }
8623 case ISD::FNEG: {
8624 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8625 break;
8626
8627 SDLoc SL(N);
8628 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8629
8630 SDValue Op = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: BC,
8631 N2: DAG.getConstant(Val: 0x80008000, DL: SL, VT: MVT::i32));
8632 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8633 return;
8634 }
8635 case ISD::FABS: {
8636 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8637 break;
8638
8639 SDLoc SL(N);
8640 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8641
8642 SDValue Op = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: BC,
8643 N2: DAG.getConstant(Val: 0x7fff7fff, DL: SL, VT: MVT::i32));
8644 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8645 return;
8646 }
8647 case ISD::FSQRT: {
8648 if (N->getValueType(ResNo: 0) != MVT::f16)
8649 break;
8650 Results.push_back(Elt: lowerFSQRTF16(Op: SDValue(N, 0), DAG));
8651 break;
8652 }
8653 default:
8654 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
8655 break;
8656 }
8657}
8658
8659/// Helper function for LowerBRCOND
8660static SDNode *findUser(SDValue Value, unsigned Opcode) {
8661
8662 for (SDUse &U : Value->uses()) {
8663 if (U.get() != Value)
8664 continue;
8665
8666 if (U.getUser()->getOpcode() == Opcode)
8667 return U.getUser();
8668 }
8669 return nullptr;
8670}
8671
8672unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
8673 if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
8674 switch (Intr->getConstantOperandVal(Num: 1)) {
8675 case Intrinsic::amdgcn_if:
8676 return AMDGPUISD::IF;
8677 case Intrinsic::amdgcn_else:
8678 return AMDGPUISD::ELSE;
8679 case Intrinsic::amdgcn_loop:
8680 return AMDGPUISD::LOOP;
8681 case Intrinsic::amdgcn_end_cf:
8682 llvm_unreachable("should not occur");
8683 default:
8684 return 0;
8685 }
8686 }
8687
8688 // break, if_break, else_break are all only used as inputs to loop, not
8689 // directly as branch conditions.
8690 return 0;
8691}
8692
8693bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
8694 const Triple &TT = GV->getParent()->getTargetTriple();
8695 return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
8696 GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
8697 AMDGPU::shouldEmitConstantsToTextSection(TT);
8698}
8699
8700bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
8701 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
8702 return false;
8703
8704 // FIXME: Either avoid relying on address space here or change the default
8705 // address space for functions to avoid the explicit check.
8706 return (GV->getValueType()->isFunctionTy() ||
8707 !isNonGlobalAddrSpace(AS: GV->getAddressSpace())) &&
8708 !shouldEmitFixup(GV) && !getTargetMachine().shouldAssumeDSOLocal(GV);
8709}
8710
8711bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
8712 return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
8713}
8714
8715bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const {
8716 if (!GV->hasExternalLinkage())
8717 return true;
8718
8719 // With object linking, external LDS declarations need relocations so the
8720 // linker can assign their offsets.
8721 if (AMDGPUTargetMachine::EnableObjectLinking) {
8722 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: GV)) {
8723 if (GVar->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
8724 GVar->getAddressSpace() == AMDGPUAS::BARRIER) {
8725 assert(GVar->isDeclaration() &&
8726 "AS 3 & 13 GVs should be declaration here "
8727 "when object linking is enabled");
8728 return false;
8729 }
8730 }
8731 }
8732
8733 const auto OS = getTargetMachine().getTargetTriple().getOS();
8734 return OS == Triple::AMDHSA || OS == Triple::AMDPAL;
8735}
8736
8737/// This transforms the control flow intrinsics to get the branch destination as
8738/// last parameter, also switches branch target with BR if the need arise
8739SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, SelectionDAG &DAG) const {
8740 SDLoc DL(BRCOND);
8741
8742 SDNode *Intr = BRCOND.getOperand(i: 1).getNode();
8743 SDValue Target = BRCOND.getOperand(i: 2);
8744 SDNode *BR = nullptr;
8745 SDNode *SetCC = nullptr;
8746
8747 switch (Intr->getOpcode()) {
8748 case ISD::SETCC: {
8749 // As long as we negate the condition everything is fine
8750 SetCC = Intr;
8751 Intr = SetCC->getOperand(Num: 0).getNode();
8752 break;
8753 }
8754 case ISD::XOR: {
8755 // Similar to SETCC, if we have (xor c, -1), we will be fine.
8756 SDValue LHS = Intr->getOperand(Num: 0);
8757 SDValue RHS = Intr->getOperand(Num: 1);
8758 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS); C && C->getZExtValue()) {
8759 Intr = LHS.getNode();
8760 break;
8761 }
8762 [[fallthrough]];
8763 }
8764 default: {
8765 // Get the target from BR if we don't negate the condition
8766 BR = findUser(Value: BRCOND, Opcode: ISD::BR);
8767 assert(BR && "brcond missing unconditional branch user");
8768 Target = BR->getOperand(Num: 1);
8769 }
8770 }
8771
8772 unsigned CFNode = isCFIntrinsic(Intr);
8773 if (CFNode == 0) {
8774 // This is a uniform branch so we don't need to legalize.
8775 return BRCOND;
8776 }
8777
8778 bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
8779 Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
8780
8781 assert(!SetCC || (SetCC->getConstantOperandVal(1) == 1 &&
8782 SetCC->getOperandCondCode(2) == ISD::SETNE));
8783
8784 // operands of the new intrinsic call
8785 SmallVector<SDValue, 4> Ops;
8786 if (HaveChain)
8787 Ops.push_back(Elt: BRCOND.getOperand(i: 0));
8788
8789 Ops.append(in_start: Intr->op_begin() + (HaveChain ? 2 : 1), in_end: Intr->op_end());
8790 Ops.push_back(Elt: Target);
8791
8792 ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
8793
8794 // build the new intrinsic call
8795 SDNode *Result = DAG.getNode(Opcode: CFNode, DL, VTList: DAG.getVTList(VTs: Res), Ops).getNode();
8796
8797 if (!HaveChain) {
8798 SDValue Ops[] = {SDValue(Result, 0), BRCOND.getOperand(i: 0)};
8799
8800 Result = DAG.getMergeValues(Ops, dl: DL).getNode();
8801 }
8802
8803 if (BR) {
8804 // Give the branch instruction our target
8805 SDValue Ops[] = {BR->getOperand(Num: 0), BRCOND.getOperand(i: 2)};
8806 SDValue NewBR = DAG.getNode(Opcode: ISD::BR, DL, VTList: BR->getVTList(), Ops);
8807 DAG.ReplaceAllUsesWith(From: BR, To: NewBR.getNode());
8808 }
8809
8810 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
8811
8812 // Copy the intrinsic results to registers
8813 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
8814 SDNode *CopyToReg = findUser(Value: SDValue(Intr, i), Opcode: ISD::CopyToReg);
8815 if (!CopyToReg)
8816 continue;
8817
8818 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CopyToReg->getOperand(Num: 1),
8819 N: SDValue(Result, i - 1), Glue: SDValue());
8820
8821 DAG.ReplaceAllUsesWith(From: SDValue(CopyToReg, 0), To: CopyToReg->getOperand(Num: 0));
8822 }
8823
8824 // Remove the old intrinsic from the chain
8825 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Intr, Intr->getNumValues() - 1),
8826 To: Intr->getOperand(Num: 0));
8827
8828 return Chain;
8829}
8830
8831SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
8832 MVT VT = Op.getSimpleValueType();
8833 SDLoc DL(Op);
8834 // Checking the depth
8835 if (Op.getConstantOperandVal(i: 0) != 0)
8836 return DAG.getConstant(Val: 0, DL, VT);
8837
8838 MachineFunction &MF = DAG.getMachineFunction();
8839 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8840 // Check for kernel and shader functions
8841 if (Info->isEntryFunction())
8842 return DAG.getConstant(Val: 0, DL, VT);
8843
8844 MachineFrameInfo &MFI = MF.getFrameInfo();
8845 // There is a call to @llvm.returnaddress in this function
8846 MFI.setReturnAddressIsTaken(true);
8847
8848 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
8849 // Get the return address reg and mark it as an implicit live-in
8850 Register Reg = MF.addLiveIn(PReg: TRI->getReturnAddressReg(MF),
8851 RC: getRegClassFor(VT, isDivergent: Op.getNode()->isDivergent()));
8852
8853 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT);
8854}
8855
8856SDValue SITargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
8857 MachineFunction &MF = DAG.getMachineFunction();
8858 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8859
8860 // For functions that set up their own stack, select the GET_STACK_BASE
8861 // pseudo.
8862 if (MFI->isBottomOfStack())
8863 return Op;
8864
8865 // For everything else, create a dummy stack object.
8866 int FI = MF.getFrameInfo().CreateFixedObject(Size: 1, SPOffset: 0, /*IsImmutable=*/false);
8867 return DAG.getFrameIndex(FI, VT: Op.getValueType());
8868}
8869
8870SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op,
8871 const SDLoc &DL, EVT VT) const {
8872 return Op.getValueType().bitsLE(VT)
8873 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
8874 : DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
8875 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8876}
8877
8878SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
8879 SelectionDAG &DAG) const {
8880 EVT DstVT = Op.getValueType();
8881 unsigned NumElts = DstVT.getVectorNumElements();
8882 assert(NumElts > 2 && isPowerOf2_32(NumElts));
8883
8884 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
8885
8886 SDLoc DL(Op);
8887 unsigned Opc = Op.getOpcode();
8888 SDValue Flags = Op.getOperand(i: 1);
8889 EVT HalfDstVT =
8890 EVT::getVectorVT(Context&: *DAG.getContext(), VT: DstVT.getScalarType(), NumElements: NumElts / 2);
8891 SDValue OpLo = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Lo, N2: Flags);
8892 SDValue OpHi = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Hi, N2: Flags);
8893
8894 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, N1: OpLo, N2: OpHi);
8895}
8896
8897SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
8898 bool IsStrict = Op->isStrictFPOpcode();
8899 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8900 EVT SrcVT = Src.getValueType();
8901 EVT DstVT = Op.getValueType();
8902
8903 if (DstVT.isVectorOf(EltVT: MVT::f16)) {
8904 assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32");
8905 if (SrcVT.getScalarType() != MVT::f32)
8906 return SDValue();
8907 return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG);
8908 }
8909
8910 if (SrcVT.getScalarType() != MVT::f64)
8911 return Op;
8912
8913 SDLoc DL(Op);
8914 if (DstVT == MVT::f16) {
8915 // TODO: Handle strictfp
8916 if (Op.getOpcode() != ISD::FP_ROUND)
8917 return Op;
8918
8919 if (!Subtarget->has16BitInsts()) {
8920 SDValue FpToFp16 = DAG.getNode(Opcode: ISD::FP_TO_FP16, DL, VT: MVT::i32, Operand: Src);
8921 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8922 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8923 }
8924 if (Op->getFlags().hasApproximateFuncs()) {
8925 SDValue Flags = Op.getOperand(i: 1);
8926 SDValue Src32 = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f32, N1: Src, N2: Flags);
8927 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Src32, N2: Flags);
8928 }
8929 SDValue FpToFp16 = LowerF64ToF16Safe(Src, DL, DAG);
8930 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8931 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8932 }
8933
8934 assert(DstVT.getScalarType() == MVT::bf16 &&
8935 "custom lower FP_ROUND for f16 or bf16");
8936 assert(Subtarget->hasBF16ConversionInsts() && "f32 -> bf16 is legal");
8937
8938 // Round-inexact-to-odd f64 to f32, then do the final rounding using the
8939 // hardware f32 -> bf16 instruction.
8940 EVT F32VT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
8941 SDValue Rod = expandRoundInexactToOdd(ResultVT: F32VT, Op: Src, DL, DAG);
8942 if (IsStrict) {
8943 return DAG.getNode(
8944 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
8945 Ops: {Op.getOperand(i: 0), Rod, DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)});
8946 }
8947 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: Rod,
8948 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8949}
8950
8951SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
8952 SelectionDAG &DAG) const {
8953 EVT VT = Op.getValueType();
8954 const MachineFunction &MF = DAG.getMachineFunction();
8955 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8956 bool IsIEEEMode = Info->getMode().IEEE;
8957
8958 // FIXME: Assert during selection that this is only selected for
8959 // ieee_mode. Currently a combine can produce the ieee version for non-ieee
8960 // mode functions, but this happens to be OK since it's only done in cases
8961 // where there is known no sNaN.
8962 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8963 return expandFMINNUM_FMAXNUM(N: Op.getNode(), DAG);
8964
8965 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8966 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8967 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8968 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8969 return splitBinaryVectorOp(Op, DAG);
8970 return Op;
8971}
8972
8973SDValue
8974SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
8975 SelectionDAG &DAG) const {
8976 EVT VT = Op.getValueType();
8977 const MachineFunction &MF = DAG.getMachineFunction();
8978 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8979 bool IsIEEEMode = Info->getMode().IEEE;
8980
8981 if (IsIEEEMode && !Subtarget->hasIEEEMinimumMaximumInsts())
8982 return expandFMINIMUMNUM_FMAXIMUMNUM(N: Op.getNode(), DAG);
8983
8984 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8985 VT == MVT::v32f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16 ||
8986 VT == MVT::v16bf16 || VT == MVT::v32bf16 || VT == MVT::v4f64 ||
8987 VT == MVT::v8f64 || VT == MVT::v16f64 || VT == MVT::v32f64)
8988 return splitBinaryVectorOp(Op, DAG);
8989 return Op;
8990}
8991
8992SDValue SITargetLowering::lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const {
8993 bool IsStrict = Op.getOpcode() == ISD::STRICT_FLDEXP;
8994 EVT VT = Op.getValueType();
8995 assert(VT == MVT::f16);
8996
8997 SDValue Exp = Op.getOperand(i: IsStrict ? 2 : 1);
8998 EVT ExpVT = Exp.getValueType();
8999 if (ExpVT == MVT::i16)
9000 return Op;
9001
9002 SDLoc DL(Op);
9003
9004 // Correct the exponent type for f16 to i16.
9005 // Clamp the range of the exponent to the instruction's range.
9006
9007 // TODO: This should be a generic narrowing legalization, and can easily be
9008 // for GlobalISel.
9009
9010 SDValue MinExp = DAG.getSignedConstant(Val: minIntN(N: 16), DL, VT: ExpVT);
9011 SDValue ClampMin = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ExpVT, N1: Exp, N2: MinExp);
9012
9013 SDValue MaxExp = DAG.getSignedConstant(Val: maxIntN(N: 16), DL, VT: ExpVT);
9014 SDValue Clamp = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ExpVT, N1: ClampMin, N2: MaxExp);
9015
9016 SDValue TruncExp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Clamp);
9017
9018 if (IsStrict) {
9019 return DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL, ResultTys: {VT, MVT::Other},
9020 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), TruncExp});
9021 }
9022
9023 return DAG.getNode(Opcode: ISD::FLDEXP, DL, VT, N1: Op.getOperand(i: 0), N2: TruncExp);
9024}
9025
9026static unsigned getExtOpcodeForPromotedOp(SDValue Op) {
9027 switch (Op->getOpcode()) {
9028 case ISD::ABS:
9029 case ISD::SRA:
9030 case ISD::SMIN:
9031 case ISD::SMAX:
9032 return ISD::SIGN_EXTEND;
9033 case ISD::SRL:
9034 case ISD::UMIN:
9035 case ISD::UMAX:
9036 case ISD::USUBSAT:
9037 case ISD::UADDSAT:
9038 return ISD::ZERO_EXTEND;
9039 case ISD::ADD:
9040 case ISD::SUB:
9041 case ISD::AND:
9042 case ISD::OR:
9043 case ISD::XOR:
9044 case ISD::SHL:
9045 case ISD::SELECT:
9046 case ISD::MUL:
9047 // operation result won't be influenced by garbage high bits.
9048 // TODO: are all of those cases correct, and are there more?
9049 return ISD::ANY_EXTEND;
9050 case ISD::SETCC: {
9051 ISD::CondCode CC = Op.getOperandCondCode(i: 2);
9052 return ISD::isSignedIntSetCC(Code: CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
9053 }
9054 default:
9055 llvm_unreachable("unexpected opcode!");
9056 }
9057}
9058
9059SDValue
9060SITargetLowering::promoteUniformUnaryOpToI32(SDValue Op,
9061 DAGCombinerInfo &DCI) const {
9062 EVT OpTy = Op.getValueType();
9063 SelectionDAG &DAG = DCI.DAG;
9064 EVT ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
9065
9066 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
9067 return SDValue();
9068
9069 SDLoc DL(Op);
9070 SDValue Input = Op.getOperand(i: 0);
9071 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
9072 Input = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: Input);
9073
9074 SDValue NewVal = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: ExtTy, Operand: Input);
9075
9076 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OpTy, Operand: NewVal);
9077}
9078
9079SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
9080 DAGCombinerInfo &DCI) const {
9081 const unsigned Opc = Op.getOpcode();
9082 assert(Opc == ISD::ADD || Opc == ISD::SUB || Opc == ISD::SHL ||
9083 Opc == ISD::SRL || Opc == ISD::SRA || Opc == ISD::AND ||
9084 Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL ||
9085 Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN ||
9086 Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX ||
9087 Opc == ISD::USUBSAT || Opc == ISD::UADDSAT);
9088
9089 EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType()
9090 : Op->getOperand(Num: 0).getValueType();
9091 auto &DAG = DCI.DAG;
9092 auto ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
9093
9094 if (DCI.isBeforeLegalizeOps() ||
9095 isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
9096 return SDValue();
9097
9098 SDLoc DL(Op);
9099 SDValue LHS;
9100 SDValue RHS;
9101 if (Opc == ISD::SELECT) {
9102 LHS = Op->getOperand(Num: 1);
9103 RHS = Op->getOperand(Num: 2);
9104 } else {
9105 LHS = Op->getOperand(Num: 0);
9106 RHS = Op->getOperand(Num: 1);
9107 }
9108
9109 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
9110 LHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {LHS});
9111
9112 // Special case: for shifts, the RHS always needs a zext.
9113 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
9114 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtTy, Operand: {RHS});
9115 else
9116 RHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {RHS});
9117
9118 // setcc always return i1/i1 vec so no need to truncate after.
9119 if (Opc == ISD::SETCC) {
9120 ISD::CondCode CC = Op.getOperandCondCode(i: 2);
9121 return DAG.getSetCC(DL, VT: Op.getValueType(), LHS, RHS, Cond: CC);
9122 }
9123
9124 // For other ops, we extend the operation's return type as well so we need to
9125 // truncate back to the original type.
9126 SDValue NewVal;
9127 if (Opc == ISD::SELECT)
9128 NewVal = DAG.getNode(Opcode: ISD::SELECT, DL, VT: ExtTy, Ops: {Op->getOperand(Num: 0), LHS, RHS});
9129 else if (Opc == ISD::UADDSAT) {
9130 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ExtTy, N1: LHS, N2: RHS);
9131 SDValue MaxVal = DAG.getConstant(
9132 Val: APInt::getMaxValue(numBits: OpTy.getScalarSizeInBits()).zext(width: 32), DL, VT: ExtTy);
9133 NewVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ExtTy, N1: Sum, N2: MaxVal);
9134 } else
9135 NewVal = DAG.getNode(Opcode: Opc, DL, VT: ExtTy, Ops: {LHS, RHS});
9136
9137 return DAG.getZExtOrTrunc(Op: NewVal, DL, VT: OpTy);
9138}
9139
9140SDValue SITargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
9141 SDValue Mag = Op.getOperand(i: 0);
9142 EVT MagVT = Mag.getValueType();
9143
9144 if (MagVT.getVectorNumElements() > 2)
9145 return splitBinaryVectorOp(Op, DAG);
9146
9147 SDValue Sign = Op.getOperand(i: 1);
9148 EVT SignVT = Sign.getValueType();
9149
9150 if (MagVT == SignVT)
9151 return Op;
9152
9153 // fcopysign v2f16:mag, v2f32:sign ->
9154 // fcopysign v2f16:mag,
9155 // bitcast (trunc (srl (bitcast sign to v2i32), 16) to v2i16)
9156
9157 SDLoc SL(Op);
9158 SDValue SignAsInt32 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Sign);
9159 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: 16, VT: MVT::v2i32, DL: SL);
9160 SDValue SignShifted =
9161 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::v2i32, N1: SignAsInt32, N2: ShiftAmt);
9162 SDValue SignAsInt16 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::v2i16, Operand: SignShifted);
9163
9164 SDValue SignAsHalf16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MagVT, Operand: SignAsInt16);
9165
9166 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MagVT, N1: Mag, N2: SignAsHalf16);
9167}
9168
9169// Custom lowering for vector multiplications and s_mul_u64.
9170SDValue SITargetLowering::lowerMUL(SDValue Op, SelectionDAG &DAG) const {
9171 EVT VT = Op.getValueType();
9172
9173 // Split vector operands.
9174 if (VT.isVector())
9175 return splitBinaryVectorOp(Op, DAG);
9176
9177 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64");
9178
9179 // There are four ways to lower s_mul_u64:
9180 //
9181 // 1. If all the operands are uniform, then we lower it as it is.
9182 //
9183 // 2. If the operands are divergent, then we have to split s_mul_u64 in 32-bit
9184 // multiplications because there is not a vector equivalent of s_mul_u64.
9185 //
9186 // 3. If the cost model decides that it is more efficient to use vector
9187 // registers, then we have to split s_mul_u64 in 32-bit multiplications.
9188 // This happens in splitScalarSMULU64() in SIInstrInfo.cpp .
9189 //
9190 // 4. If the cost model decides to use vector registers and both of the
9191 // operands are zero-extended/sign-extended from 32-bits, then we split the
9192 // s_mul_u64 in two 32-bit multiplications. The problem is that it is not
9193 // possible to check if the operands are zero-extended or sign-extended in
9194 // SIInstrInfo.cpp. For this reason, here, we replace s_mul_u64 with
9195 // s_mul_u64_u32_pseudo if both operands are zero-extended and we replace
9196 // s_mul_u64 with s_mul_i64_i32_pseudo if both operands are sign-extended.
9197 // If the cost model decides that we have to use vector registers, then
9198 // splitScalarSMulPseudo() (in SIInstrInfo.cpp) split s_mul_u64_u32/
9199 // s_mul_i64_i32_pseudo in two vector multiplications. If the cost model
9200 // decides that we should use scalar registers, then s_mul_u64_u32_pseudo/
9201 // s_mul_i64_i32_pseudo is lowered as s_mul_u64 in expandPostRAPseudo() in
9202 // SIInstrInfo.cpp .
9203
9204 if (Op->isDivergent())
9205 return SDValue();
9206
9207 SDValue Op0 = Op.getOperand(i: 0);
9208 SDValue Op1 = Op.getOperand(i: 1);
9209 // If all the operands are zero-enteted to 32-bits, then we replace s_mul_u64
9210 // with s_mul_u64_u32_pseudo. If all the operands are sign-extended to
9211 // 32-bits, then we replace s_mul_u64 with s_mul_i64_i32_pseudo.
9212 KnownBits Op0KnownBits = DAG.computeKnownBits(Op: Op0);
9213 unsigned Op0LeadingZeros = Op0KnownBits.countMinLeadingZeros();
9214 KnownBits Op1KnownBits = DAG.computeKnownBits(Op: Op1);
9215 unsigned Op1LeadingZeros = Op1KnownBits.countMinLeadingZeros();
9216 SDLoc SL(Op);
9217 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
9218 return SDValue(
9219 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_U64_U32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9220 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op0);
9221 unsigned Op1SignBits = DAG.ComputeNumSignBits(Op: Op1);
9222 if (Op0SignBits >= 33 && Op1SignBits >= 33)
9223 return SDValue(
9224 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_I64_I32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9225 // If all the operands are uniform, then we lower s_mul_u64 as it is.
9226 return Op;
9227}
9228
9229SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
9230 EVT VT = Op.getValueType();
9231 SDLoc SL(Op);
9232 SDValue LHS = Op.getOperand(i: 0);
9233 SDValue RHS = Op.getOperand(i: 1);
9234 bool isSigned = Op.getOpcode() == ISD::SMULO;
9235
9236 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
9237 const APInt &C = RHSC->getAPIntValue();
9238 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
9239 if (C.isPowerOf2()) {
9240 // smulo(x, signed_min) is same as umulo(x, signed_min).
9241 bool UseArithShift = isSigned && !C.isMinSignedValue();
9242 SDValue ShiftAmt = DAG.getConstant(Val: C.logBase2(), DL: SL, VT: MVT::i32);
9243 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: LHS, N2: ShiftAmt);
9244 SDValue Overflow =
9245 DAG.getSetCC(DL: SL, VT: MVT::i1,
9246 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL, DL: SL, VT,
9247 N1: Result, N2: ShiftAmt),
9248 RHS: LHS, Cond: ISD::SETNE);
9249 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9250 }
9251 }
9252
9253 SDValue Result = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: LHS, N2: RHS);
9254 SDValue Top =
9255 DAG.getNode(Opcode: isSigned ? ISD::MULHS : ISD::MULHU, DL: SL, VT, N1: LHS, N2: RHS);
9256
9257 SDValue Sign = isSigned
9258 ? DAG.getNode(Opcode: ISD::SRA, DL: SL, VT, N1: Result,
9259 N2: DAG.getConstant(Val: VT.getScalarSizeInBits() - 1,
9260 DL: SL, VT: MVT::i32))
9261 : DAG.getConstant(Val: 0, DL: SL, VT);
9262 SDValue Overflow = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Top, RHS: Sign, Cond: ISD::SETNE);
9263
9264 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9265}
9266
9267SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
9268 if (Op->isDivergent()) {
9269 // Select to V_MAD_[IU]64_[IU]32.
9270 return Op;
9271 }
9272 if (Subtarget->hasSMulHi()) {
9273 // Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
9274 return SDValue();
9275 }
9276 // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
9277 // calculate the high part, so we might as well do the whole thing with
9278 // V_MAD_[IU]64_[IU]32.
9279 return Op;
9280}
9281
9282SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
9283 if (!Subtarget->hasTrapHandler() ||
9284 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
9285 return lowerTrapEndpgm(Op, DAG);
9286
9287 return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(Op, DAG)
9288 : lowerTrapHsaQueuePtr(Op, DAG);
9289}
9290
9291SDValue SITargetLowering::lowerTrapEndpgm(SDValue Op, SelectionDAG &DAG) const {
9292 SDLoc SL(Op);
9293 SDValue Chain = Op.getOperand(i: 0);
9294 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9295}
9296
9297SDValue
9298SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT,
9299 const SDLoc &DL, Align Alignment,
9300 ImplicitParameter Param) const {
9301 MachineFunction &MF = DAG.getMachineFunction();
9302 uint64_t Offset = getImplicitParameterOffset(MF, Param);
9303 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain: DAG.getEntryNode(), Offset);
9304 MachinePointerInfo PtrInfo =
9305 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
9306 return DAG.getLoad(
9307 VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
9308 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
9309}
9310
9311SDValue SITargetLowering::lowerTrapHsaQueuePtr(SDValue Op,
9312 SelectionDAG &DAG) const {
9313 SDLoc SL(Op);
9314 SDValue Chain = Op.getOperand(i: 0);
9315
9316 SDValue QueuePtr;
9317 // For code object version 5, QueuePtr is passed through implicit kernarg.
9318 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9319 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9320 QueuePtr =
9321 loadImplicitKernelArgument(DAG, VT: MVT::i64, DL: SL, Alignment: Align(8), Param: QUEUE_PTR);
9322 } else {
9323 MachineFunction &MF = DAG.getMachineFunction();
9324 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9325 Register UserSGPR = Info->getQueuePtrUserSGPR();
9326
9327 if (UserSGPR == AMDGPU::NoRegister) {
9328 // We probably are in a function incorrectly marked with
9329 // amdgpu-no-queue-ptr. This is undefined. We don't want to delete the
9330 // trap, so just use a null pointer.
9331 QueuePtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9332 } else {
9333 QueuePtr = CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR,
9334 VT: MVT::i64);
9335 }
9336 }
9337
9338 SDValue SGPR01 = DAG.getRegister(Reg: AMDGPU::SGPR0_SGPR1, VT: MVT::i64);
9339 SDValue ToReg = DAG.getCopyToReg(Chain, dl: SL, Reg: SGPR01, N: QueuePtr, Glue: SDValue());
9340
9341 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9342 SDValue Ops[] = {ToReg, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16), SGPR01,
9343 ToReg.getValue(R: 1)};
9344 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9345}
9346
9347SDValue SITargetLowering::lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const {
9348 SDLoc SL(Op);
9349 SDValue Chain = Op.getOperand(i: 0);
9350
9351 // We need to simulate the 's_trap 2' instruction on targets that run in
9352 // PRIV=1 (where it is treated as a nop).
9353 if (Subtarget->hasPrivEnabledTrap2NopBug())
9354 return DAG.getNode(Opcode: AMDGPUISD::SIMULATED_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9355
9356 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9357 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9358 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9359}
9360
9361SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
9362 SDLoc SL(Op);
9363 SDValue Chain = Op.getOperand(i: 0);
9364 MachineFunction &MF = DAG.getMachineFunction();
9365
9366 if (!Subtarget->hasTrapHandler() ||
9367 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
9368 LLVMContext &Ctx = MF.getFunction().getContext();
9369 Ctx.diagnose(DI: DiagnosticInfoUnsupported(MF.getFunction(),
9370 "debugtrap handler not supported",
9371 Op.getDebugLoc(), DS_Warning));
9372 return Chain;
9373 }
9374
9375 uint64_t TrapID =
9376 static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap);
9377 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9378 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9379}
9380
9381/// When a divergent value (in VGPR) is passed to an inline asm with an SGPR
9382/// constraint ('s'), we need to insert v_readfirstlane to move the value from
9383/// VGPR to SGPR. This is done by modifying the CopyToReg nodes in the glue
9384/// chain that feed into the INLINEASM node.
9385SDValue SITargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
9386 unsigned NumOps = Op.getNumOperands();
9387
9388 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
9389 SmallSet<Register, 8> SGPRInputRegs;
9390
9391 unsigned NumVals = 0;
9392 for (unsigned I = InlineAsm::Op_FirstOperand; I < NumOps - 1;
9393 I += 1 + NumVals) {
9394 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i: I));
9395 NumVals = Flags.getNumOperandRegisters();
9396
9397 unsigned RCID;
9398 bool IsSGPRInput = Flags.getKind() == InlineAsm::Kind::RegUse &&
9399 NumVals > 0 && Flags.hasRegClassConstraint(RC&: RCID) &&
9400 TRI->isSGPRClass(RC: TRI->getRegClass(i: RCID));
9401
9402 for (unsigned J = 0; J < NumVals; ++J) {
9403 SDValue Val = Op.getOperand(i: I + 1 + J);
9404 if (const RegisterSDNode *RegNode =
9405 dyn_cast<RegisterSDNode>(Val: Val.getNode())) {
9406 Register Reg = RegNode->getReg();
9407 if (IsSGPRInput || (Reg.isPhysical() && TRI->isSGPRPhysReg(Reg)))
9408 SGPRInputRegs.insert(V: Reg);
9409 }
9410 }
9411 }
9412
9413 if (SGPRInputRegs.empty())
9414 return Op;
9415
9416 // Walk the glue chain and insert readfirstlane for divergent SGPR inputs.
9417 SDLoc DL(Op);
9418 SDNode *N = Op.getOperand(i: NumOps - 1).getNode();
9419
9420 while (N && N->getOpcode() == ISD::CopyToReg) {
9421 Register Reg = cast<RegisterSDNode>(Val: N->getOperand(Num: 1))->getReg();
9422 SDValue SrcVal = N->getOperand(Num: 2);
9423
9424 // Insert readfirstlane if copying a divergent value to an SGPR input.
9425 if (SrcVal->isDivergent() && SGPRInputRegs.count(V: Reg)) {
9426 SDValue ReadFirstLaneID =
9427 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
9428 SDValue ReadFirstLane =
9429 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: SrcVal.getValueType(),
9430 N1: ReadFirstLaneID, N2: SrcVal);
9431
9432 SmallVector<SDValue, 4> Ops = {N->getOperand(Num: 0), N->getOperand(Num: 1),
9433 ReadFirstLane};
9434 if (N->getNumOperands() > 3)
9435 Ops.push_back(Elt: N->getOperand(Num: 3)); // Glue input
9436
9437 DAG.UpdateNodeOperands(N, Ops);
9438 }
9439
9440 // Follow glue chain to next CopyToReg.
9441 SDNode *Next = nullptr;
9442 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I) {
9443 if (N->getOperand(Num: I).getValueType() == MVT::Glue) {
9444 Next = N->getOperand(Num: I).getNode();
9445 break;
9446 }
9447 }
9448 N = Next;
9449 }
9450
9451 return Op;
9452}
9453
9454SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
9455 SelectionDAG &DAG) const {
9456 unsigned BaseAS = AS;
9457 unsigned SANum = AMDGPU::getSyntheticApertureNumber(AS);
9458 if (SANum != AMDGPU::SyntheticAperture::None)
9459 BaseAS = AMDGPUAS::LOCAL_ADDRESS;
9460
9461 SDValue Aperture = getBaseSegmentAperture(AS: BaseAS, DL, DAG);
9462
9463 if (SANum != AMDGPU::SyntheticAperture::None) {
9464 SDValue Tag = DAG.getConstant(Val: SANum, DL, VT: MVT::i32);
9465 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Aperture, N2: Tag);
9466 }
9467
9468 return Aperture;
9469}
9470
9471SDValue SITargetLowering::getBaseSegmentAperture(unsigned AS, const SDLoc &DL,
9472 SelectionDAG &DAG) const {
9473 const bool IsLDS = (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER);
9474
9475 if (Subtarget->hasApertureRegs()) {
9476 const unsigned ApertureRegNo =
9477 IsLDS ? AMDGPU::SRC_SHARED_BASE : AMDGPU::SRC_PRIVATE_BASE;
9478 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
9479 !Subtarget->hasGloballyAddressableScratch()) &&
9480 "Cannot use src_private_base with globally addressable scratch!");
9481 // Note: this feature (register) is broken. When used as a 32-bit operand,
9482 // it returns a wrong value (all zeroes?). The real value is in the upper 32
9483 // bits.
9484 //
9485 // To work around the issue, emit a 64 bit copy from this register
9486 // then extract the high bits. Note that this shouldn't even result in a
9487 // shift being emitted and simply become a pair of registers (e.g.):
9488 // s_mov_b64 s[6:7], src_shared_base
9489 // v_mov_b32_e32 v1, s7
9490 SDValue Copy =
9491 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: ApertureRegNo, VT: MVT::v2i32);
9492 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Copy, Idx: 1);
9493 }
9494
9495 // For code object version 5, private_base and shared_base are passed through
9496 // implicit kernargs.
9497 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9498 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9499 ImplicitParameter Param = IsLDS ? SHARED_BASE : PRIVATE_BASE;
9500 return loadImplicitKernelArgument(DAG, VT: MVT::i32, DL, Alignment: Align(4), Param);
9501 }
9502
9503 MachineFunction &MF = DAG.getMachineFunction();
9504 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9505 Register UserSGPR = Info->getQueuePtrUserSGPR();
9506 if (UserSGPR == AMDGPU::NoRegister) {
9507 // We probably are in a function incorrectly marked with
9508 // amdgpu-no-queue-ptr. This is undefined.
9509 return DAG.getPOISON(VT: MVT::i32);
9510 }
9511
9512 SDValue QueuePtr =
9513 CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR, VT: MVT::i64);
9514
9515 // Offset into amd_queue_t for group_segment_aperture_base_hi /
9516 // private_segment_aperture_base_hi.
9517 uint32_t StructOffset = IsLDS ? 0x40 : 0x44;
9518
9519 SDValue Ptr =
9520 DAG.getObjectPtrOffset(SL: DL, Ptr: QueuePtr, Offset: TypeSize::getFixed(ExactSize: StructOffset));
9521
9522 // TODO: Use custom target PseudoSourceValue.
9523 // TODO: We should use the value from the IR intrinsic call, but it might not
9524 // be available and how do we get it?
9525 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
9526 return DAG.getLoad(VT: MVT::i32, dl: DL, Chain: QueuePtr.getValue(R: 1), Ptr, PtrInfo,
9527 Alignment: commonAlignment(A: Align(64), Offset: StructOffset),
9528 MMOFlags: MachineMemOperand::MODereferenceable |
9529 MachineMemOperand::MOInvariant);
9530}
9531
9532/// Return true if the value is a known valid address, such that a null check is
9533/// not necessary.
9534static bool isKnownNonNull(SDValue Val, SelectionDAG &DAG,
9535 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
9536 if (isa<FrameIndexSDNode, GlobalAddressSDNode, BasicBlockSDNode>(Val))
9537 return true;
9538
9539 if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
9540 return ConstVal->getSExtValue() != AMDGPU::getNullPointerValue(AS: AddrSpace);
9541
9542 // TODO: Search through arithmetic, handle arguments and loads
9543 // marked nonnull.
9544 return false;
9545}
9546
9547SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
9548 SelectionDAG &DAG) const {
9549 SDLoc SL(Op);
9550
9551 const AMDGPUTargetMachine &TM =
9552 static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
9553
9554 const auto *ASC = cast<AddrSpaceCastSDNode>(Val&: Op);
9555 unsigned SrcAS = ASC->getSrcAddressSpace();
9556 SDValue Src = ASC->getOperand(Num: 0);
9557 unsigned DestAS = ASC->getDestAddressSpace();
9558 bool IsNonNull = ASC->getFlags().hasNonNull();
9559
9560 SDValue FlatNullPtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9561
9562 // flat -> local/private/barrier/vgpr
9563 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
9564 if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
9565 DestAS == AMDGPUAS::PRIVATE_ADDRESS || DestAS == AMDGPUAS::BARRIER ||
9566 DestAS == AMDGPUAS::VGPR) {
9567 SDValue Ptr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9568
9569 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
9570 Subtarget->hasGloballyAddressableScratch()) {
9571 // flat -> private with globally addressable scratch: subtract
9572 // src_flat_scratch_base_lo.
9573 SDValue FlatScratchBaseLo(
9574 DAG.getMachineNode(
9575 Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: MVT::i32,
9576 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, VT: MVT::i32)),
9577 0);
9578 Ptr = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: Ptr, N2: FlatScratchBaseLo);
9579 }
9580
9581 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9582 return Ptr;
9583
9584 unsigned NullVal = AMDGPU::getNullPointerValue(AS: DestAS);
9585 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9586 SDValue NonNull = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: FlatNullPtr, Cond: ISD::SETNE);
9587
9588 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: NonNull, N2: Ptr,
9589 N3: SegmentNullPtr);
9590 }
9591 }
9592
9593 // local/private/barrier/vgpr -> flat
9594 if (DestAS == AMDGPUAS::FLAT_ADDRESS) {
9595 if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
9596 SrcAS == AMDGPUAS::PRIVATE_ADDRESS || SrcAS == AMDGPUAS::BARRIER ||
9597 SrcAS == AMDGPUAS::VGPR) {
9598 SDValue CvtPtr;
9599 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
9600 Subtarget->hasGloballyAddressableScratch()) {
9601 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
9602 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
9603 SDValue AllOnes = DAG.getSignedTargetConstant(Val: -1, DL: SL, VT: MVT::i32);
9604 SDValue ThreadID = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
9605 ThreadID = DAG.getNode(
9606 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9607 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_lo, DL: SL, VT: MVT::i32),
9608 N2: AllOnes, N3: ThreadID);
9609 if (Subtarget->isWave64())
9610 ThreadID = DAG.getNode(
9611 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9612 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_hi, DL: SL, VT: MVT::i32),
9613 N2: AllOnes, N3: ThreadID);
9614 SDValue ShAmt = DAG.getShiftAmountConstant(
9615 Val: 57 - 32 - Subtarget->getWavefrontSizeLog2(), VT: MVT::i32, DL: SL);
9616 SDValue SrcHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ThreadID, N2: ShAmt);
9617 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: SrcHi);
9618 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9619 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
9620 // 64-bit hi:lo value.
9621 SDValue FlatScratchBase = {
9622 DAG.getMachineNode(
9623 Opcode: AMDGPU::S_MOV_B64, dl: SL, VT: MVT::i64,
9624 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE, VT: MVT::i64)),
9625 0};
9626 CvtPtr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: CvtPtr, N2: FlatScratchBase);
9627 } else {
9628 SDValue Aperture = getSegmentAperture(AS: SrcAS, DL: SL, DAG);
9629
9630 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Aperture);
9631 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9632 }
9633
9634 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9635 return CvtPtr;
9636
9637 unsigned NullVal = AMDGPU::getNullPointerValue(AS: SrcAS);
9638 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9639
9640 SDValue NonNull =
9641 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: SegmentNullPtr, Cond: ISD::SETNE);
9642
9643 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: NonNull, N2: CvtPtr,
9644 N3: FlatNullPtr);
9645 }
9646 }
9647
9648 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9649 Op.getValueType() == MVT::i64) {
9650 const SIMachineFunctionInfo *Info =
9651 DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
9652 if (Info->get32BitAddressHighBits() == 0)
9653 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: Src);
9654
9655 SDValue Hi = DAG.getConstant(Val: Info->get32BitAddressHighBits(), DL: SL, VT: MVT::i32);
9656 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Hi);
9657 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
9658 }
9659
9660 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9661 Src.getValueType() == MVT::i64)
9662 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9663
9664 // global <-> flat are no-ops and never emitted.
9665
9666 // Invalid casts are poison.
9667 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
9668}
9669
9670// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
9671// the small vector and inserting them into the big vector. That is better than
9672// the default expansion of doing it via a stack slot. Even though the use of
9673// the stack slot would be optimized away afterwards, the stack slot itself
9674// remains.
9675SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
9676 SelectionDAG &DAG) const {
9677 SDValue Vec = Op.getOperand(i: 0);
9678 SDValue Ins = Op.getOperand(i: 1);
9679 SDValue Idx = Op.getOperand(i: 2);
9680 EVT VecVT = Vec.getValueType();
9681 EVT InsVT = Ins.getValueType();
9682 EVT EltVT = VecVT.getVectorElementType();
9683 unsigned InsNumElts = InsVT.getVectorNumElements();
9684 unsigned IdxVal = Idx->getAsZExtVal();
9685 SDLoc SL(Op);
9686
9687 if (EltVT.getScalarSizeInBits() == 16 && IdxVal % 2 == 0) {
9688 // Insert 32-bit registers at a time.
9689 assert(InsNumElts % 2 == 0 && "expect legal vector types");
9690
9691 unsigned VecNumElts = VecVT.getVectorNumElements();
9692 EVT NewVecVT =
9693 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: VecNumElts / 2);
9694 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
9695 : EVT::getVectorVT(Context&: *DAG.getContext(),
9696 VT: MVT::i32, NumElements: InsNumElts / 2);
9697
9698 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVecVT, Operand: Vec);
9699 Ins = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewInsVT, Operand: Ins);
9700
9701 for (unsigned I = 0; I != InsNumElts / 2; ++I) {
9702 SDValue Elt;
9703 if (InsNumElts == 2) {
9704 Elt = Ins;
9705 } else {
9706 Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Ins,
9707 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9708 }
9709 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NewVecVT, N1: Vec, N2: Elt,
9710 N3: DAG.getConstant(Val: IdxVal / 2 + I, DL: SL, VT: MVT::i32));
9711 }
9712
9713 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Vec);
9714 }
9715
9716 for (unsigned I = 0; I != InsNumElts; ++I) {
9717 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Ins,
9718 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9719 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: VecVT, N1: Vec, N2: Elt,
9720 N3: DAG.getConstant(Val: IdxVal + I, DL: SL, VT: MVT::i32));
9721 }
9722 return Vec;
9723}
9724
9725SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
9726 SelectionDAG &DAG) const {
9727 SDValue Vec = Op.getOperand(i: 0);
9728 SDValue InsVal = Op.getOperand(i: 1);
9729 SDValue Idx = Op.getOperand(i: 2);
9730 EVT VecVT = Vec.getValueType();
9731 EVT EltVT = VecVT.getVectorElementType();
9732 unsigned VecSize = VecVT.getSizeInBits();
9733 unsigned EltSize = EltVT.getSizeInBits();
9734 SDLoc SL(Op);
9735
9736 // Specially handle the case of v4i16 with static indexing.
9737 unsigned NumElts = VecVT.getVectorNumElements();
9738 auto *KIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
9739 if (NumElts == 4 && EltSize == 16 && KIdx) {
9740 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Vec);
9741
9742 SDValue LoHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9743 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
9744 SDValue HiHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9745 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
9746
9747 SDValue LoVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: LoHalf);
9748 SDValue HiVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: HiHalf);
9749
9750 unsigned Idx = KIdx->getZExtValue();
9751 bool InsertLo = Idx < 2;
9752 SDValue InsHalf = DAG.getNode(
9753 Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: MVT::v2i16, N1: InsertLo ? LoVec : HiVec,
9754 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: InsVal),
9755 N3: DAG.getConstant(Val: InsertLo ? Idx : (Idx - 2), DL: SL, VT: MVT::i32));
9756
9757 InsHalf = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: InsHalf);
9758
9759 SDValue Concat =
9760 InsertLo ? DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {InsHalf, HiHalf})
9761 : DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoHalf, InsHalf});
9762
9763 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Concat);
9764 }
9765
9766 // Static indexing does not lower to stack access, and hence there is no need
9767 // for special custom lowering to avoid stack access.
9768 if (isa<ConstantSDNode>(Val: Idx))
9769 return SDValue();
9770
9771 // Avoid stack access for dynamic indexing by custom lowering to
9772 // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
9773 // Vectors wider than 64 bits do this on each 64-bit piece.
9774 assert(isPowerOf2_32(EltSize));
9775 unsigned PieceSize = std::min(a: VecSize, b: 64u);
9776 unsigned NumPieces = VecSize / PieceSize;
9777 unsigned EltsPerPiece = PieceSize / EltSize;
9778 MVT IntVT = MVT::getIntegerVT(BitWidth: PieceSize);
9779 EVT PieceVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: EltsPerPiece);
9780
9781 // Convert vector index to bit-index within its piece and get the required
9782 // bit mask. Using the index within the piece keeps the shift below 64.
9783 SDValue PieceIdx = Idx;
9784 if (NumPieces > 1)
9785 PieceIdx = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Idx,
9786 N2: DAG.getConstant(Val: EltsPerPiece - 1, DL: SL, VT: MVT::i32));
9787 const auto EltMask = maskTrailingOnes<uint64_t>(N: EltSize);
9788 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9789 SDValue ScaledIdx =
9790 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: PieceIdx, N2: ScaleFactor);
9791 SDValue BFM = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT,
9792 N1: DAG.getConstant(Val: EltMask, DL: SL, VT: IntVT), N2: ScaledIdx);
9793
9794 // 1. Create a congruent vector with the target value in each element.
9795 SDValue ExtVal =
9796 DAG.getBitcast(VT: IntVT, V: DAG.getSplatBuildVector(VT: PieceVT, DL: SL, Op: InsVal));
9797
9798 auto InsertIntoPiece = [&](SDValue Piece, SDValue Mask) {
9799 // 2. Mask off all other indices except the required index within (1).
9800 SDValue LHS = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: Mask, N2: ExtVal);
9801
9802 // 3. Mask off the required index within the target vector.
9803 SDValue RHS =
9804 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: DAG.getNOT(DL: SL, Val: Mask, VT: IntVT), N2: Piece);
9805
9806 // 4. Get (2) and (3) ORed into the target vector.
9807 return DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: LHS, N2: RHS, Flags: SDNodeFlags::Disjoint);
9808 };
9809
9810 if (NumPieces == 1)
9811 return DAG.getBitcast(VT: VecVT,
9812 V: InsertIntoPiece(DAG.getBitcast(VT: IntVT, V: Vec), BFM));
9813
9814 // 5. Apply (2)-(4) to each 64-bit piece, with the mask cleared in every
9815 // piece except the one that holds the index, and rebuild the vector.
9816 MVT PiecesVT = MVT::getVectorVT(VT: IntVT, NumElements: NumPieces);
9817 SDValue BCVec = DAG.getBitcast(VT: PiecesVT, V: Vec);
9818 SDValue PieceNo =
9819 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Idx,
9820 N2: DAG.getConstant(Val: Log2_32(Value: EltsPerPiece), DL: SL, VT: MVT::i32));
9821 SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: IntVT);
9822 SmallVector<SDValue, 8> Pieces;
9823 Pieces.reserve(N: NumPieces);
9824 for (unsigned I = 0; I != NumPieces; ++I) {
9825 SDValue CurPieceNo = DAG.getConstant(Val: I, DL: SL, VT: MVT::i32);
9826 SDValue Piece =
9827 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: IntVT, N1: BCVec, N2: CurPieceNo);
9828 SDValue Mask =
9829 DAG.getSelectCC(DL: SL, LHS: PieceNo, RHS: CurPieceNo, True: BFM, False: Zero, Cond: ISD::SETEQ);
9830 Pieces.push_back(Elt: InsertIntoPiece(Piece, Mask));
9831 }
9832
9833 return DAG.getBitcast(VT: VecVT, V: DAG.getBuildVector(VT: PiecesVT, DL: SL, Ops: Pieces));
9834}
9835
9836SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
9837 SelectionDAG &DAG) const {
9838 SDLoc SL(Op);
9839
9840 EVT ResultVT = Op.getValueType();
9841 SDValue Vec = Op.getOperand(i: 0);
9842 SDValue Idx = Op.getOperand(i: 1);
9843 EVT VecVT = Vec.getValueType();
9844 unsigned VecSize = VecVT.getSizeInBits();
9845 EVT EltVT = VecVT.getVectorElementType();
9846
9847 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
9848
9849 // Make sure we do any optimizations that will make it easier to fold
9850 // source modifiers before obscuring it with bit operations.
9851
9852 // XXX - Why doesn't this get called when vector_shuffle is expanded?
9853 if (SDValue Combined = performExtractVectorEltCombine(N: Op.getNode(), DCI))
9854 return Combined;
9855
9856 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
9857 SDValue Lo, Hi;
9858 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
9859
9860 if (VecSize == 128) {
9861 SDValue V2 = DAG.getBitcast(VT: MVT::v2i64, V: Vec);
9862 Lo = DAG.getBitcast(VT: LoVT,
9863 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9864 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32)));
9865 Hi = DAG.getBitcast(VT: HiVT,
9866 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9867 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32)));
9868 } else if (VecSize == 256) {
9869 SDValue V2 = DAG.getBitcast(VT: MVT::v4i64, V: Vec);
9870 SDValue Parts[4];
9871 for (unsigned P = 0; P < 4; ++P) {
9872 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9873 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9874 }
9875
9876 Lo = DAG.getBitcast(VT: LoVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9877 N1: Parts[0], N2: Parts[1]));
9878 Hi = DAG.getBitcast(VT: HiVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9879 N1: Parts[2], N2: Parts[3]));
9880 } else {
9881 assert(VecSize == 512);
9882
9883 SDValue V2 = DAG.getBitcast(VT: MVT::v8i64, V: Vec);
9884 SDValue Parts[8];
9885 for (unsigned P = 0; P < 8; ++P) {
9886 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9887 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9888 }
9889
9890 Lo = DAG.getBitcast(VT: LoVT,
9891 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9892 N1: Parts[0], N2: Parts[1], N3: Parts[2], N4: Parts[3]));
9893 Hi = DAG.getBitcast(VT: HiVT,
9894 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9895 N1: Parts[4], N2: Parts[5], N3: Parts[6], N4: Parts[7]));
9896 }
9897
9898 EVT IdxVT = Idx.getValueType();
9899 unsigned NElem = VecVT.getVectorNumElements();
9900 assert(isPowerOf2_32(NElem));
9901 SDValue IdxMask = DAG.getConstant(Val: NElem / 2 - 1, DL: SL, VT: IdxVT);
9902 SDValue NewIdx = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IdxVT, N1: Idx, N2: IdxMask);
9903 SDValue Half = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IdxMask, True: Hi, False: Lo, Cond: ISD::SETUGT);
9904 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Half, N2: NewIdx);
9905 }
9906
9907 assert(VecSize <= 64);
9908
9909 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9910
9911 // If Vec is just a SCALAR_TO_VECTOR, then use the scalar integer directly.
9912 SDValue VecBC = peekThroughBitcasts(V: Vec);
9913 if (VecBC.getOpcode() == ISD::SCALAR_TO_VECTOR) {
9914 SDValue Src = VecBC.getOperand(i: 0);
9915 Src = DAG.getBitcast(VT: Src.getValueType().changeTypeToInteger(), V: Src);
9916 Vec = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: IntVT);
9917 }
9918
9919 unsigned EltSize = EltVT.getSizeInBits();
9920 assert(isPowerOf2_32(EltSize));
9921
9922 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9923
9924 // Convert vector index to bit-index (* EltSize)
9925 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9926
9927 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9928 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: IntVT, N1: BC, N2: ScaledIdx);
9929
9930 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
9931 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Elt);
9932 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResultVT, Operand: Result);
9933 }
9934
9935 return DAG.getAnyExtOrTrunc(Op: Elt, DL: SL, VT: ResultVT);
9936}
9937
9938static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
9939 assert(Elt % 2 == 0);
9940 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
9941}
9942
9943static bool elementPairIsOddToEven(ArrayRef<int> Mask, int Elt) {
9944 assert(Elt % 2 == 0);
9945 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
9946 !(Mask[Elt + 1] & 1);
9947}
9948
9949SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
9950 SelectionDAG &DAG) const {
9951 SDLoc SL(Op);
9952 EVT ResultVT = Op.getValueType();
9953 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
9954 MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
9955 const int NewSrcNumElts = 2;
9956 MVT PackVT = MVT::getVectorVT(VT: EltVT, NumElements: NewSrcNumElts);
9957 int SrcNumElts = Op.getOperand(i: 0).getValueType().getVectorNumElements();
9958
9959 // Break up the shuffle into registers sized pieces.
9960 //
9961 // We're trying to form sub-shuffles that the register allocation pipeline
9962 // won't be able to figure out, like how to use v_pk_mov_b32 to do a register
9963 // blend or 16-bit op_sel. It should be able to figure out how to reassemble a
9964 // pair of copies into a consecutive register copy, so use the ordinary
9965 // extract_vector_elt lowering unless we can use the shuffle.
9966 //
9967 // TODO: This is a bit of hack, and we should probably always use
9968 // extract_subvector for the largest possible subvector we can (or at least
9969 // use it for PackVT aligned pieces). However we have worse support for
9970 // combines on them don't directly treat extract_subvector / insert_subvector
9971 // as legal. The DAG scheduler also ends up doing a worse job with the
9972 // extract_subvectors.
9973 const bool ShouldUseConsecutiveExtract = EltVT.getSizeInBits() == 16;
9974
9975 // vector_shuffle <0,1,6,7> lhs, rhs
9976 // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
9977 //
9978 // vector_shuffle <6,7,2,3> lhs, rhs
9979 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
9980 //
9981 // vector_shuffle <6,7,0,1> lhs, rhs
9982 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
9983
9984 // Avoid scalarizing when both halves are reading from consecutive elements.
9985
9986 // If we're treating 2 element shuffles as legal, also create odd-to-even
9987 // shuffles of neighboring pairs.
9988 //
9989 // vector_shuffle <3,2,7,6> lhs, rhs
9990 // -> concat_vectors vector_shuffle <1, 0> (extract_subvector lhs, 0)
9991 // vector_shuffle <1, 0> (extract_subvector rhs, 2)
9992
9993 SmallVector<SDValue, 16> Pieces;
9994 for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
9995 if (ShouldUseConsecutiveExtract &&
9996 elementPairIsContiguous(Mask: SVN->getMask(), Elt: I)) {
9997 const int Idx = SVN->getMaskElt(Idx: I);
9998 int VecIdx = Idx < SrcNumElts ? 0 : 1;
9999 int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
10000 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT,
10001 N1: SVN->getOperand(Num: VecIdx),
10002 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
10003 Pieces.push_back(Elt: SubVec);
10004 } else if (elementPairIsOddToEven(Mask: SVN->getMask(), Elt: I) &&
10005 isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: PackVT)) {
10006 int Idx0 = SVN->getMaskElt(Idx: I);
10007 int Idx1 = SVN->getMaskElt(Idx: I + 1);
10008
10009 SDValue SrcOp0 = SVN->getOperand(Num: 0);
10010 SDValue SrcOp1 = SrcOp0;
10011 if (Idx0 >= SrcNumElts) {
10012 SrcOp0 = SVN->getOperand(Num: 1);
10013 Idx0 -= SrcNumElts;
10014 }
10015
10016 if (Idx1 >= SrcNumElts) {
10017 SrcOp1 = SVN->getOperand(Num: 1);
10018 Idx1 -= SrcNumElts;
10019 }
10020
10021 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
10022 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
10023
10024 // Extract nearest even aligned piece.
10025 SDValue SubVec0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp0,
10026 N2: DAG.getConstant(Val: AlignedIdx0, DL: SL, VT: MVT::i32));
10027 SDValue SubVec1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp1,
10028 N2: DAG.getConstant(Val: AlignedIdx1, DL: SL, VT: MVT::i32));
10029
10030 int NewMaskIdx0 = Idx0 - AlignedIdx0;
10031 int NewMaskIdx1 = Idx1 - AlignedIdx1;
10032
10033 SDValue Result0 = SubVec0;
10034 SDValue Result1 = SubVec0;
10035
10036 if (SubVec0 != SubVec1) {
10037 NewMaskIdx1 += NewSrcNumElts;
10038 Result1 = SubVec1;
10039 } else {
10040 Result1 = DAG.getPOISON(VT: PackVT);
10041 }
10042
10043 SDValue Shuf = DAG.getVectorShuffle(VT: PackVT, dl: SL, N1: Result0, N2: Result1,
10044 Mask: {NewMaskIdx0, NewMaskIdx1});
10045 Pieces.push_back(Elt: Shuf);
10046 } else {
10047 const int Idx0 = SVN->getMaskElt(Idx: I);
10048 const int Idx1 = SVN->getMaskElt(Idx: I + 1);
10049 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
10050 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
10051 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
10052 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
10053
10054 SDValue Vec0 = SVN->getOperand(Num: VecIdx0);
10055 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec0,
10056 N2: DAG.getSignedConstant(Val: EltIdx0, DL: SL, VT: MVT::i32));
10057
10058 SDValue Vec1 = SVN->getOperand(Num: VecIdx1);
10059 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec1,
10060 N2: DAG.getSignedConstant(Val: EltIdx1, DL: SL, VT: MVT::i32));
10061 Pieces.push_back(Elt: DAG.getBuildVector(VT: PackVT, DL: SL, Ops: {Elt0, Elt1}));
10062 }
10063 }
10064
10065 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: ResultVT, Ops: Pieces);
10066}
10067
10068SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
10069 SelectionDAG &DAG) const {
10070 SDValue SVal = Op.getOperand(i: 0);
10071 EVT ResultVT = Op.getValueType();
10072 EVT SValVT = SVal.getValueType();
10073 SDValue UndefVal = DAG.getPOISON(VT: SValVT);
10074 SDLoc SL(Op);
10075
10076 SmallVector<SDValue, 8> VElts;
10077 VElts.push_back(Elt: SVal);
10078 for (int I = 1, E = ResultVT.getVectorNumElements(); I < E; ++I)
10079 VElts.push_back(Elt: UndefVal);
10080
10081 return DAG.getBuildVector(VT: ResultVT, DL: SL, Ops: VElts);
10082}
10083
10084SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
10085 SelectionDAG &DAG) const {
10086 SDLoc SL(Op);
10087 EVT VT = Op.getValueType();
10088
10089 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
10090 assert(!Subtarget->hasVOP3PInsts() && "this should be legal");
10091
10092 SDValue Lo = Op.getOperand(i: 0);
10093 SDValue Hi = Op.getOperand(i: 1);
10094
10095 // Avoid adding defined bits with the zero_extend.
10096 if (Hi.isUndef()) {
10097 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
10098 SDValue ExtLo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
10099 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ExtLo);
10100 }
10101
10102 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Hi);
10103 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Hi);
10104
10105 SDValue ShlHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Hi,
10106 N2: DAG.getConstant(Val: 16, DL: SL, VT: MVT::i32));
10107 if (Lo.isUndef())
10108 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ShlHi);
10109
10110 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
10111 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
10112
10113 SDValue Or =
10114 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Lo, N2: ShlHi, Flags: SDNodeFlags::Disjoint);
10115 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Or);
10116 }
10117
10118 // Split into 2-element chunks.
10119 const unsigned NumParts = VT.getVectorNumElements() / 2;
10120 EVT PartVT = MVT::getVectorVT(VT: VT.getVectorElementType().getSimpleVT(), NumElements: 2);
10121 MVT PartIntVT = MVT::getIntegerVT(BitWidth: PartVT.getSizeInBits());
10122
10123 SmallVector<SDValue> Casts;
10124 for (unsigned P = 0; P < NumParts; ++P) {
10125 SDValue Vec = DAG.getBuildVector(
10126 VT: PartVT, DL: SL, Ops: {Op.getOperand(i: P * 2), Op.getOperand(i: P * 2 + 1)});
10127 Casts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: PartIntVT, Operand: Vec));
10128 }
10129
10130 SDValue Blend =
10131 DAG.getBuildVector(VT: MVT::getVectorVT(VT: PartIntVT, NumElements: NumParts), DL: SL, Ops: Casts);
10132 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Blend);
10133}
10134
10135bool SITargetLowering::isOffsetFoldingLegal(
10136 const GlobalAddressSDNode *GA) const {
10137 // Named barriers have fixed, non-relocated LDS addresses, so a constant
10138 // offset into an array of them can be folded into the address.
10139 if (GA->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
10140 const auto *GV = dyn_cast<GlobalVariable>(Val: GA->getGlobal());
10141 return GV && AMDGPU::isNamedBarrier(GV: *GV);
10142 }
10143
10144 // OSes that use ELF REL relocations (instead of RELA) can only store a
10145 // 32-bit addend in the instruction, so it is not safe to allow offset folding
10146 // which can create arbitrary 64-bit addends. (This is only a problem for
10147 // R_AMDGPU_*32_HI relocations since other relocation types are unaffected by
10148 // the high 32 bits of the addend.)
10149 //
10150 // This should be kept in sync with how HasRelocationAddend is initialized in
10151 // the constructor of ELFAMDGPUAsmBackend.
10152 if (!Subtarget->isAmdHsaOS())
10153 return false;
10154
10155 // We can fold offsets for anything that doesn't require a GOT relocation.
10156 return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
10157 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
10158 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
10159 !shouldEmitGOTReloc(GV: GA->getGlobal());
10160}
10161
10162static SDValue
10163buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
10164 const SDLoc &DL, int64_t Offset, EVT PtrVT,
10165 unsigned GAFlags = SIInstrInfo::MO_NONE) {
10166 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
10167 // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
10168 // lowered to the following code sequence:
10169 //
10170 // For constant address space:
10171 // s_getpc_b64 s[0:1]
10172 // s_add_u32 s0, s0, $symbol
10173 // s_addc_u32 s1, s1, 0
10174 //
10175 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10176 // a fixup or relocation is emitted to replace $symbol with a literal
10177 // constant, which is a pc-relative offset from the encoding of the $symbol
10178 // operand to the global variable.
10179 //
10180 // For global address space:
10181 // s_getpc_b64 s[0:1]
10182 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
10183 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
10184 //
10185 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10186 // fixups or relocations are emitted to replace $symbol@*@lo and
10187 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
10188 // which is a 64-bit pc-relative offset from the encoding of the $symbol
10189 // operand to the global variable.
10190 if (((const GCNSubtarget &)DAG.getSubtarget()).has64BitLiterals()) {
10191 assert(GAFlags != SIInstrInfo::MO_NONE);
10192
10193 SDValue Ptr =
10194 DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64, offset: Offset, TargetFlags: GAFlags + 2);
10195 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET64, DL, VT: PtrVT, Operand: Ptr);
10196 }
10197
10198 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags);
10199 SDValue PtrHi;
10200 if (GAFlags == SIInstrInfo::MO_NONE)
10201 PtrHi = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
10202 else
10203 PtrHi = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags + 1);
10204 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET, DL, VT: PtrVT, N1: PtrLo, N2: PtrHi);
10205}
10206
10207SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
10208 SDValue Op,
10209 SelectionDAG &DAG) const {
10210 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Val&: Op);
10211 SDLoc DL(GSD);
10212 EVT PtrVT = Op.getValueType();
10213
10214 const GlobalValue *GV = GSD->getGlobal();
10215 const unsigned AS = GSD->getAddressSpace();
10216 if (((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER) &&
10217 shouldUseLDSConstAddress(GV)) ||
10218 AS == AMDGPUAS::REGION_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS) {
10219 if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
10220 const GlobalVariable &GVar = *cast<GlobalVariable>(Val: GV);
10221 // HIP uses an unsized array `extern __shared__ T s[]` or similar
10222 // zero-sized type in other languages to declare the dynamic shared
10223 // memory which size is not known at the compile time. They will be
10224 // allocated by the runtime and placed directly after the static
10225 // allocated ones. They all share the same offset.
10226 if (GVar.getGlobalSize(DL: GVar.getDataLayout()) == 0) {
10227 assert(PtrVT == MVT::i32 && "32-bit pointer is expected.");
10228 // Adjust alignment for that dynamic shared memory array.
10229 Function &F = DAG.getMachineFunction().getFunction();
10230 MFI->setDynLDSAlign(F, GV: GVar);
10231 MFI->setUsesDynamicLDS(true);
10232 return SDValue(
10233 DAG.getMachineNode(Opcode: AMDGPU::GET_GROUPSTATICSIZE, dl: DL, VT: PtrVT), 0);
10234 }
10235 }
10236 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
10237 }
10238
10239 if (AS == AMDGPUAS::BARRIER) {
10240 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
10241 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10242 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0);
10243 }
10244
10245 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
10246 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
10247 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10248 return DAG.getNode(Opcode: AMDGPUISD::LDS, DL, VT: MVT::i32, Operand: GA);
10249 }
10250
10251 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS()) {
10252 if (Subtarget->has64BitLiterals()) {
10253 SDValue Addr = DAG.getTargetGlobalAddress(
10254 GV, DL, VT: MVT::i64, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS64);
10255 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B64, dl: DL, VT: MVT::i64, Op1: Addr),
10256 0);
10257 }
10258
10259 SDValue AddrLo = DAG.getTargetGlobalAddress(
10260 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_LO);
10261 AddrLo = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrLo), 0};
10262
10263 SDValue AddrHi = DAG.getTargetGlobalAddress(
10264 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_HI);
10265 AddrHi = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrHi), 0};
10266
10267 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: AddrLo, N2: AddrHi);
10268 }
10269
10270 if (shouldEmitFixup(GV))
10271 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT);
10272
10273 if (shouldEmitPCReloc(GV))
10274 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT,
10275 GAFlags: SIInstrInfo::MO_REL32);
10276
10277 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, Offset: 0, PtrVT,
10278 GAFlags: SIInstrInfo::MO_GOTPCREL32);
10279 PointerType *PtrTy =
10280 PointerType::get(C&: *DAG.getContext(), AddressSpace: AMDGPUAS::CONSTANT_ADDRESS);
10281 const DataLayout &DataLayout = DAG.getDataLayout();
10282 Align Alignment = DataLayout.getABITypeAlign(Ty: PtrTy);
10283 MachinePointerInfo PtrInfo =
10284 MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction());
10285
10286 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: GOTAddr, PtrInfo, Alignment,
10287 MMOFlags: MachineMemOperand::MODereferenceable |
10288 MachineMemOperand::MOInvariant);
10289}
10290
10291// An alloca in the VGPR ("as memory") address space has no registers to live
10292// in, so its stack offset must not be taken for a register address. Every
10293// other frame index keeps its default lowering.
10294SDValue SITargetLowering::lowerFrameIndex(SDValue Op, SelectionDAG &DAG) const {
10295 MachineFunction &MF = DAG.getMachineFunction();
10296 int FI = cast<FrameIndexSDNode>(Val&: Op)->getIndex();
10297 const AllocaInst *Alloca = MF.getFrameInfo().getObjectAllocation(ObjectIdx: FI);
10298 if (!Alloca || Alloca->getAddressSpace() != AMDGPUAS::VGPR)
10299 return SDValue();
10300
10301 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10302 MF.getFunction(),
10303 "unsupported alloca in VGPR 'as memory' address space (13)",
10304 Alloca->getDebugLoc()));
10305 return DAG.getPOISON(VT: MVT::i32);
10306}
10307
10308SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
10309 SelectionDAG &DAG) const {
10310 // TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
10311 const Function &Fn = DAG.getMachineFunction().getFunction();
10312 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10313 Fn, "unsupported external symbol", Op.getDebugLoc()));
10314 return DAG.getPOISON(VT: Op.getValueType());
10315}
10316
10317SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
10318 const SDLoc &DL, SDValue V) const {
10319 // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
10320 // the destination register.
10321 //
10322 // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
10323 // so we will end up with redundant moves to m0.
10324 //
10325 // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
10326
10327 // A Null SDValue creates a glue result.
10328 SDNode *M0 = DAG.getMachineNode(Opcode: AMDGPU::SI_INIT_M0, dl: DL, VT1: MVT::Other, VT2: MVT::Glue,
10329 Op1: V, Op2: Chain);
10330 return SDValue(M0, 0);
10331}
10332
10333SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
10334 MVT VT,
10335 unsigned Offset) const {
10336 SDLoc SL(Op);
10337 SDValue Param = lowerKernargMemParameter(
10338 DAG, VT: MVT::i32, MemVT: MVT::i32, SL, Chain: DAG.getEntryNode(), Offset, Alignment: Align(4), Signed: false);
10339 // The local size values will have the hi 16-bits as zero.
10340 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Param,
10341 N2: DAG.getValueType(VT));
10342}
10343
10344static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10345 EVT VT) {
10346 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10347 DAG.getMachineFunction().getFunction(),
10348 "non-hsa intrinsic with hsa target", DL.getDebugLoc()));
10349 return DAG.getPOISON(VT);
10350}
10351
10352static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10353 EVT VT) {
10354 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10355 DAG.getMachineFunction().getFunction(),
10356 "intrinsic not supported on subtarget", DL.getDebugLoc()));
10357 return DAG.getPOISON(VT);
10358}
10359
10360static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
10361 ArrayRef<SDValue> Elts) {
10362 assert(!Elts.empty());
10363 MVT Type;
10364 unsigned NumElts = Elts.size();
10365
10366 if (NumElts <= 12) {
10367 Type = MVT::getVectorVT(VT: MVT::f32, NumElements: NumElts);
10368 } else {
10369 assert(Elts.size() <= 16);
10370 Type = MVT::v16f32;
10371 NumElts = 16;
10372 }
10373
10374 SmallVector<SDValue, 16> VecElts(NumElts);
10375 for (unsigned i = 0; i < Elts.size(); ++i) {
10376 SDValue Elt = Elts[i];
10377 if (Elt.getValueType() != MVT::f32)
10378 Elt = DAG.getBitcast(VT: MVT::f32, V: Elt);
10379 VecElts[i] = Elt;
10380 }
10381 for (unsigned i = Elts.size(); i < NumElts; ++i)
10382 VecElts[i] = DAG.getPOISON(VT: MVT::f32);
10383
10384 if (NumElts == 1)
10385 return VecElts[0];
10386 return DAG.getBuildVector(VT: Type, DL, Ops: VecElts);
10387}
10388
10389static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT,
10390 SDValue Src, int ExtraElts) {
10391 EVT SrcVT = Src.getValueType();
10392
10393 SmallVector<SDValue, 8> Elts;
10394
10395 if (SrcVT.isVector())
10396 DAG.ExtractVectorElements(Op: Src, Args&: Elts);
10397 else
10398 Elts.push_back(Elt: Src);
10399
10400 SDValue Undef = DAG.getPOISON(VT: SrcVT.getScalarType());
10401 while (ExtraElts--)
10402 Elts.push_back(Elt: Undef);
10403
10404 return DAG.getBuildVector(VT: CastVT, DL, Ops: Elts);
10405}
10406
10407// Re-construct the required return value for a image load intrinsic.
10408// This is more complicated due to the optional use TexFailCtrl which means the
10409// required return type is an aggregate
10410static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result,
10411 ArrayRef<EVT> ResultTypes, bool IsTexFail,
10412 bool Unpacked, bool IsD16, int DMaskPop,
10413 int NumVDataDwords, bool IsAtomicPacked16Bit,
10414 const SDLoc &DL) {
10415 // Determine the required return type. This is the same regardless of
10416 // IsTexFail flag
10417 EVT ReqRetVT = ResultTypes[0];
10418 int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1;
10419 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
10420 ? (ReqRetNumElts + 1) / 2
10421 : ReqRetNumElts;
10422
10423 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
10424
10425 MVT DataDwordVT =
10426 NumDataDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: NumDataDwords);
10427
10428 MVT MaskPopVT =
10429 MaskPopDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: MaskPopDwords);
10430
10431 SDValue Data(Result, 0);
10432 SDValue TexFail;
10433
10434 if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) {
10435 SDValue ZeroIdx = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
10436 if (MaskPopVT.isVector()) {
10437 Data = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: MaskPopVT,
10438 N1: SDValue(Result, 0), N2: ZeroIdx);
10439 } else {
10440 Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MaskPopVT,
10441 N1: SDValue(Result, 0), N2: ZeroIdx);
10442 }
10443 }
10444
10445 if (DataDwordVT.isVector() && !IsAtomicPacked16Bit)
10446 Data = padEltsToUndef(DAG, DL, CastVT: DataDwordVT, Src: Data,
10447 ExtraElts: NumDataDwords - MaskPopDwords);
10448
10449 if (IsD16)
10450 Data = adjustLoadValueTypeImpl(Result: Data, LoadVT: ReqRetVT, DL, DAG, Unpacked);
10451
10452 EVT LegalReqRetVT = ReqRetVT;
10453 if (!ReqRetVT.isVector()) {
10454 if (!Data.getValueType().isInteger())
10455 Data = DAG.getNode(Opcode: ISD::BITCAST, DL,
10456 VT: Data.getValueType().changeTypeToInteger(), Operand: Data);
10457 Data = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ReqRetVT.changeTypeToInteger(), Operand: Data);
10458 } else {
10459 // We need to widen the return vector to a legal type
10460 if ((ReqRetVT.getVectorNumElements() % 2) == 1 &&
10461 ReqRetVT.getVectorElementType().getSizeInBits() == 16) {
10462 LegalReqRetVT =
10463 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ReqRetVT.getVectorElementType(),
10464 NumElements: ReqRetVT.getVectorNumElements() + 1);
10465 }
10466 }
10467 Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LegalReqRetVT, Operand: Data);
10468
10469 if (IsTexFail) {
10470 TexFail =
10471 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: SDValue(Result, 0),
10472 N2: DAG.getConstant(Val: MaskPopDwords, DL, VT: MVT::i32));
10473
10474 return DAG.getMergeValues(Ops: {Data, TexFail, SDValue(Result, 1)}, dl: DL);
10475 }
10476
10477 if (Result->getNumValues() == 1)
10478 return Data;
10479
10480 return DAG.getMergeValues(Ops: {Data, SDValue(Result, 1)}, dl: DL);
10481}
10482
10483static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
10484 SDValue *LWE, bool &IsTexFail) {
10485 auto *TexFailCtrlConst = cast<ConstantSDNode>(Val: TexFailCtrl.getNode());
10486
10487 uint64_t Value = TexFailCtrlConst->getZExtValue();
10488 if (Value) {
10489 IsTexFail = true;
10490 }
10491
10492 SDLoc DL(TexFailCtrlConst);
10493 *TFE = DAG.getTargetConstant(Val: (Value & 0x1) ? 1 : 0, DL, VT: MVT::i32);
10494 Value &= ~(uint64_t)0x1;
10495 *LWE = DAG.getTargetConstant(Val: (Value & 0x2) ? 1 : 0, DL, VT: MVT::i32);
10496 Value &= ~(uint64_t)0x2;
10497
10498 return Value == 0;
10499}
10500
10501static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op,
10502 MVT PackVectorVT,
10503 SmallVectorImpl<SDValue> &PackedAddrs,
10504 unsigned DimIdx, unsigned EndIdx,
10505 unsigned NumGradients) {
10506 SDLoc DL(Op);
10507 for (unsigned I = DimIdx; I < EndIdx; I++) {
10508 SDValue Addr = Op.getOperand(i: I);
10509
10510 // Gradients are packed with undef for each coordinate.
10511 // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this:
10512 // 1D: undef,dx/dh; undef,dx/dv
10513 // 2D: dy/dh,dx/dh; dy/dv,dx/dv
10514 // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv
10515 if (((I + 1) >= EndIdx) ||
10516 ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 ||
10517 I == DimIdx + NumGradients - 1))) {
10518 if (Addr.getValueType() != MVT::i16)
10519 Addr = DAG.getBitcast(VT: MVT::i16, V: Addr);
10520 Addr = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Addr);
10521 } else {
10522 Addr = DAG.getBuildVector(VT: PackVectorVT, DL, Ops: {Addr, Op.getOperand(i: I + 1)});
10523 I++;
10524 }
10525 Addr = DAG.getBitcast(VT: MVT::f32, V: Addr);
10526 PackedAddrs.push_back(Elt: Addr);
10527 }
10528}
10529
10530/// Emit a DiagnosticInfoUnsupported for an unsupported image intrinsic and
10531/// return poison values of \p ResultTypes, preserving the chain if present.
10532static SDValue diagnoseUnsupportedImage(SelectionDAG &DAG, SDValue Op,
10533 ArrayRef<EVT> ResultTypes,
10534 const SDLoc &DL, const Twine &Msg) {
10535 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10536 DAG.getMachineFunction().getFunction(), Msg, DL.getDebugLoc()));
10537 return DAG.getErrorMergeValues(ResultTypes, Chain: Op.getOperand(i: 0), dl: DL);
10538}
10539
10540SDValue SITargetLowering::lowerImage(SDValue Op,
10541 const AMDGPU::ImageDimIntrinsicInfo *Intr,
10542 SelectionDAG &DAG, bool WithChain) const {
10543 SDLoc DL(Op);
10544 MachineFunction &MF = DAG.getMachineFunction();
10545 const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
10546 unsigned IntrOpcode = Intr->BaseOpcode;
10547 // For image atomic: use no-return opcode if result is unused.
10548 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode &&
10549 !Op.getNode()->hasAnyUseOfValue(Value: 0))
10550 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
10551 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
10552 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
10553 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
10554 bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI: *Subtarget);
10555 bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
10556 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
10557 bool IsGFX13 = AMDGPU::isGFX13(STI: *Subtarget);
10558
10559 SmallVector<EVT, 3> ResultTypes(Op->values());
10560 SmallVector<EVT, 3> OrigResultTypes(Op->values());
10561 if (BaseOpcode->NoReturn && BaseOpcode->Atomic)
10562 ResultTypes.erase(CI: &ResultTypes[0]);
10563
10564 bool IsD16 = false;
10565 bool IsG16 = false;
10566 bool IsA16 = false;
10567 SDValue VData;
10568 int NumVDataDwords = 0;
10569 bool AdjustRetType = false;
10570 bool IsAtomicPacked16Bit = false;
10571
10572 // Offset of intrinsic arguments
10573 const unsigned ArgOffset = WithChain ? 2 : 1;
10574
10575 unsigned DMask;
10576 unsigned DMaskLanes = 0;
10577
10578 if (BaseOpcode->Atomic) {
10579 VData = Op.getOperand(i: 2);
10580
10581 IsAtomicPacked16Bit =
10582 (IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
10583 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16_NORTN ||
10584 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
10585 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
10586
10587 if (!IsAtomicPacked16Bit && VData.getValueSizeInBits() != 32 &&
10588 VData.getValueSizeInBits() != 64) {
10589 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10590 Msg: "unsupported image atomic data type");
10591 }
10592
10593 bool Is64Bit = VData.getValueSizeInBits() == 64;
10594 if (BaseOpcode->AtomicX2) {
10595 SDValue VData2 = Op.getOperand(i: 3);
10596 VData = DAG.getBuildVector(VT: Is64Bit ? MVT::v2i64 : MVT::v2i32, DL,
10597 Ops: {VData, VData2});
10598 if (Is64Bit)
10599 VData = DAG.getBitcast(VT: MVT::v4i32, V: VData);
10600
10601 if (!BaseOpcode->NoReturn)
10602 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
10603
10604 DMask = Is64Bit ? 0xf : 0x3;
10605 NumVDataDwords = Is64Bit ? 4 : 2;
10606 } else {
10607 DMask = Is64Bit ? 0x3 : 0x1;
10608 NumVDataDwords = Is64Bit ? 2 : 1;
10609 }
10610 } else {
10611 DMask = Op->getConstantOperandVal(Num: ArgOffset + Intr->DMaskIndex);
10612 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
10613
10614 if (BaseOpcode->Store) {
10615 VData = Op.getOperand(i: 2);
10616
10617 MVT StoreVT = VData.getSimpleValueType();
10618 MVT StoreScalarVT = StoreVT.getScalarType();
10619 if (StoreScalarVT != MVT::f16 && StoreScalarVT.getSizeInBits() != 32 &&
10620 StoreScalarVT.getSizeInBits() != 64) {
10621 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10622 Msg: "unsupported image store data type");
10623 }
10624 if (StoreScalarVT == MVT::f16) {
10625 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10626 return Op; // D16 is unsupported for this instruction
10627
10628 IsD16 = true;
10629 VData = handleD16VData(VData, DAG, ImageStore: true);
10630 }
10631
10632 NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
10633 } else if (!BaseOpcode->NoReturn) {
10634 // Work out the num dwords based on the dmask popcount and underlying type
10635 // and whether packing is supported.
10636 MVT LoadVT = ResultTypes[0].getSimpleVT();
10637 MVT LoadScalarVT = LoadVT.getScalarType();
10638 if (LoadScalarVT != MVT::f16 && LoadScalarVT.getSizeInBits() != 32 &&
10639 LoadScalarVT.getSizeInBits() != 64) {
10640 return diagnoseUnsupportedImage(DAG, Op, ResultTypes: OrigResultTypes, DL,
10641 Msg: "unsupported image load data type");
10642 }
10643 if (LoadScalarVT == MVT::f16) {
10644 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10645 return Op; // D16 is unsupported for this instruction
10646
10647 IsD16 = true;
10648 }
10649
10650 // Confirm that the return type is large enough for the dmask specified
10651 if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) ||
10652 (!LoadVT.isVector() && DMaskLanes > 1))
10653 return Op;
10654
10655 // The sq block of gfx8 and gfx9 do not estimate register use correctly
10656 // for d16 image_gather4, image_gather4_l, and image_gather4_lz
10657 // instructions.
10658 if (IsD16 && !Subtarget->hasUnpackedD16VMem() &&
10659 !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug()))
10660 NumVDataDwords = (DMaskLanes + 1) / 2;
10661 else
10662 NumVDataDwords = DMaskLanes;
10663
10664 AdjustRetType = true;
10665 }
10666 }
10667
10668 unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd;
10669 SmallVector<SDValue, 4> VAddrs;
10670
10671 // Check for 16 bit addresses or derivatives and pack if true.
10672 MVT VAddrVT =
10673 Op.getOperand(i: ArgOffset + Intr->GradientStart).getSimpleValueType();
10674 MVT VAddrScalarVT = VAddrVT.getScalarType();
10675 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10676 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10677
10678 VAddrVT = Op.getOperand(i: ArgOffset + Intr->CoordStart).getSimpleValueType();
10679 VAddrScalarVT = VAddrVT.getScalarType();
10680 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10681 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10682
10683 // Push back extra arguments.
10684 for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) {
10685 if (IsA16 && (Op.getOperand(i: ArgOffset + I).getValueType() == MVT::f16)) {
10686 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
10687 // Special handling of bias when A16 is on. Bias is of type half but
10688 // occupies full 32-bit.
10689 SDValue Bias = DAG.getBuildVector(
10690 VT: MVT::v2f16, DL,
10691 Ops: {Op.getOperand(i: ArgOffset + I), DAG.getPOISON(VT: MVT::f16)});
10692 VAddrs.push_back(Elt: Bias);
10693 } else {
10694 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
10695 "Bias needs to be converted to 16 bit in A16 mode");
10696 VAddrs.push_back(Elt: Op.getOperand(i: ArgOffset + I));
10697 }
10698 }
10699
10700 if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) {
10701 // 16 bit gradients are supported, but are tied to the A16 control
10702 // so both gradients and addresses must be 16 bit
10703 LLVM_DEBUG(
10704 dbgs() << "Failed to lower image intrinsic: 16 bit addresses "
10705 "require 16 bit args for both gradients and addresses");
10706 return Op;
10707 }
10708
10709 if (IsA16) {
10710 if (!ST->hasA16()) {
10711 LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not "
10712 "support 16 bit addresses\n");
10713 return Op;
10714 }
10715 }
10716
10717 // We've dealt with incorrect input so we know that if IsA16, IsG16
10718 // are set then we have to compress/pack operands (either address,
10719 // gradient or both)
10720 // In the case where a16 and gradients are tied (no G16 support) then we
10721 // have already verified that both IsA16 and IsG16 are true
10722 if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) {
10723 // Activate g16
10724 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
10725 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
10726 IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16
10727 }
10728
10729 // Add gradients (packed or unpacked)
10730 if (IsG16) {
10731 // Pack the gradients
10732 // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart);
10733 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: GradPackVectorVT, PackedAddrs&: VAddrs,
10734 DimIdx: ArgOffset + Intr->GradientStart,
10735 EndIdx: ArgOffset + Intr->CoordStart, NumGradients: Intr->NumGradients);
10736 } else {
10737 for (unsigned I = ArgOffset + Intr->GradientStart;
10738 I < ArgOffset + Intr->CoordStart; I++)
10739 VAddrs.push_back(Elt: Op.getOperand(i: I));
10740 }
10741
10742 // Add addresses (packed or unpacked)
10743 if (IsA16) {
10744 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: AddrPackVectorVT, PackedAddrs&: VAddrs,
10745 DimIdx: ArgOffset + Intr->CoordStart, EndIdx: VAddrEnd,
10746 NumGradients: 0 /* No gradients */);
10747 } else {
10748 // Add uncompressed address
10749 for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++)
10750 VAddrs.push_back(Elt: Op.getOperand(i: I));
10751 }
10752
10753 // If the register allocator cannot place the address registers contiguously
10754 // without introducing moves, then using the non-sequential address encoding
10755 // is always preferable, since it saves VALU instructions and is usually a
10756 // wash in terms of code size or even better.
10757 //
10758 // However, we currently have no way of hinting to the register allocator that
10759 // MIMG addresses should be placed contiguously when it is possible to do so,
10760 // so force non-NSA for the common 2-address case as a heuristic.
10761 //
10762 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
10763 // allocation when possible.
10764 //
10765 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
10766 // set of the remaining addresses.
10767 const unsigned NSAMaxSize = ST->getNSAMaxSize(HasSampler: BaseOpcode->Sampler);
10768 const bool HasPartialNSAEncoding = ST->hasPartialNSAEncoding();
10769 const bool UseNSA = ST->hasNSAEncoding() &&
10770 VAddrs.size() >= ST->getNSAThreshold(MF) &&
10771 (VAddrs.size() <= NSAMaxSize || HasPartialNSAEncoding);
10772 const bool UsePartialNSA =
10773 UseNSA && HasPartialNSAEncoding && VAddrs.size() > NSAMaxSize;
10774
10775 SDValue VAddr;
10776 if (UsePartialNSA) {
10777 VAddr = getBuildDwordsVector(DAG, DL,
10778 Elts: ArrayRef(VAddrs).drop_front(N: NSAMaxSize - 1));
10779 } else if (!UseNSA) {
10780 VAddr = getBuildDwordsVector(DAG, DL, Elts: VAddrs);
10781 }
10782
10783 SDValue True = DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1);
10784 SDValue False = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1);
10785 SDValue Unorm;
10786 if (!BaseOpcode->Sampler) {
10787 Unorm = True;
10788 } else {
10789 uint64_t UnormConst =
10790 Op.getConstantOperandVal(i: ArgOffset + Intr->UnormIndex);
10791
10792 Unorm = UnormConst ? True : False;
10793 }
10794
10795 SDValue TFE;
10796 SDValue LWE;
10797 SDValue TexFail = Op.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex);
10798 bool IsTexFail = false;
10799 if (!parseTexFail(TexFailCtrl: TexFail, DAG, TFE: &TFE, LWE: &LWE, IsTexFail))
10800 return Op;
10801
10802 if (IsTexFail) {
10803 if (!DMaskLanes) {
10804 // Expecting to get an error flag since TFC is on - and dmask is 0
10805 // Force dmask to be at least 1 otherwise the instruction will fail
10806 DMask = 0x1;
10807 DMaskLanes = 1;
10808 NumVDataDwords = 1;
10809 }
10810 NumVDataDwords += 1;
10811 AdjustRetType = true;
10812 }
10813
10814 // Has something earlier tagged that the return type needs adjusting
10815 // This happens if the instruction is a load or has set TexFailCtrl flags
10816 if (AdjustRetType) {
10817 // NumVDataDwords reflects the true number of dwords required in the return
10818 // type
10819 if (DMaskLanes == 0 && !BaseOpcode->Store) {
10820 // This is a no-op load. This can be eliminated
10821 SDValue Undef = DAG.getPOISON(VT: Op.getValueType());
10822 if (isa<MemSDNode>(Val: Op))
10823 return DAG.getMergeValues(Ops: {Undef, Op.getOperand(i: 0)}, dl: DL);
10824 return Undef;
10825 }
10826
10827 EVT NewVT = NumVDataDwords > 1 ? EVT::getVectorVT(Context&: *DAG.getContext(),
10828 VT: MVT::i32, NumElements: NumVDataDwords)
10829 : MVT::i32;
10830
10831 ResultTypes[0] = NewVT;
10832 if (ResultTypes.size() == 3) {
10833 // Original result was aggregate type used for TexFailCtrl results
10834 // The actual instruction returns as a vector type which has now been
10835 // created. Remove the aggregate result.
10836 ResultTypes.erase(CI: &ResultTypes[1]);
10837 }
10838 }
10839
10840 unsigned CPol = Op.getConstantOperandVal(i: ArgOffset + Intr->CachePolicyIndex);
10841 // Keep GLC only when the atomic's result is actually used.
10842 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
10843 CPol |= AMDGPU::CPol::GLC;
10844 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
10845 AMDGPU::CPol::VOLATILE))
10846 return Op;
10847
10848 SmallVector<SDValue, 26> Ops;
10849 if (BaseOpcode->Store || BaseOpcode->Atomic)
10850 Ops.push_back(Elt: VData); // vdata
10851 if (UsePartialNSA) {
10852 append_range(C&: Ops, R: ArrayRef(VAddrs).take_front(N: NSAMaxSize - 1));
10853 Ops.push_back(Elt: VAddr);
10854 } else if (UseNSA)
10855 append_range(C&: Ops, R&: VAddrs);
10856 else
10857 Ops.push_back(Elt: VAddr);
10858 SDValue Rsrc = Op.getOperand(i: ArgOffset + Intr->RsrcIndex);
10859 EVT RsrcVT = Rsrc.getValueType();
10860 bool IndexedRsrc = IsGFX13 && RsrcVT == MVT::i32;
10861 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32 && !IndexedRsrc)
10862 return Op;
10863 Ops.push_back(Elt: Rsrc);
10864 bool IsSampIndexed = false;
10865 if (BaseOpcode->Sampler) {
10866 SDValue Samp = Op.getOperand(i: ArgOffset + Intr->SampIndex);
10867 EVT SampVT = Samp.getValueType();
10868 IsSampIndexed = IsGFX13 && SampVT == MVT::i32;
10869 if (SampVT != MVT::v4i32 && !IsSampIndexed)
10870 return Op;
10871 Ops.push_back(Elt: Samp);
10872 }
10873 Ops.push_back(Elt: DAG.getTargetConstant(Val: DMask, DL, VT: MVT::i32));
10874 if (IsGFX10Plus)
10875 Ops.push_back(Elt: DAG.getTargetConstant(Val: DimInfo->Encoding, DL, VT: MVT::i32));
10876 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10877 Ops.push_back(Elt: Unorm);
10878 Ops.push_back(Elt: DAG.getTargetConstant(Val: CPol, DL, VT: MVT::i32));
10879 Ops.push_back(Elt: IsA16 && // r128, a16 for gfx9
10880 ST->hasFeature(Feature: AMDGPU::FeatureR128A16)
10881 ? True
10882 : False);
10883 if (IsGFX10Plus)
10884 Ops.push_back(Elt: IsA16 ? True : False);
10885
10886 if (!Subtarget->hasGFX90AInsts())
10887 Ops.push_back(Elt: TFE); // tfe
10888 else if (TFE->getAsZExtVal()) {
10889 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10890 DAG.getMachineFunction().getFunction(),
10891 "TFE is not supported on this GPU", DL.getDebugLoc()));
10892 }
10893
10894 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10895 Ops.push_back(Elt: LWE); // lwe
10896 if (!IsGFX10Plus)
10897 Ops.push_back(Elt: DimInfo->DA ? True : False);
10898 if (BaseOpcode->HasD16)
10899 Ops.push_back(Elt: IsD16 ? True : False);
10900 if (isa<MemSDNode>(Val: Op))
10901 Ops.push_back(Elt: Op.getOperand(i: 0)); // chain
10902
10903 int NumVAddrDwords =
10904 UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32;
10905 int Opcode = -1;
10906
10907 if (IsGFX13) {
10908 Opcode =
10909 AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13, VDataDwords: NumVDataDwords,
10910 VAddrDwords: NumVAddrDwords, IndexedRsrc, IndexedSamp: IsSampIndexed);
10911 } else if (IsGFX12Plus) {
10912 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
10913 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10914 } else if (IsGFX11Plus) {
10915 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10916 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
10917 : AMDGPU::MIMGEncGfx11Default,
10918 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10919 } else if (IsGFX10Plus) {
10920 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10921 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
10922 : AMDGPU::MIMGEncGfx10Default,
10923 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10924 } else {
10925 if (Subtarget->hasGFX90AInsts()) {
10926 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
10927 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10928 if (Opcode == -1) {
10929 return diagnoseUnsupportedImage(
10930 DAG, Op, ResultTypes: OrigResultTypes, DL,
10931 Msg: "requested image instruction is not supported on this GPU");
10932 }
10933 }
10934 if (Opcode == -1 &&
10935 Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10936 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
10937 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10938 if (Opcode == -1)
10939 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
10940 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10941 }
10942 if (Opcode == -1)
10943 return Op;
10944
10945 MachineSDNode *NewNode = DAG.getMachineNode(Opcode, dl: DL, ResultTys: ResultTypes, Ops);
10946 if (auto *MemOp = dyn_cast<MemSDNode>(Val&: Op)) {
10947 MachineMemOperand *MemRef = MemOp->getMemOperand();
10948 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
10949 }
10950
10951 if (BaseOpcode->NoReturn) {
10952 if (BaseOpcode->Atomic)
10953 return DAG.getMergeValues(
10954 Ops: {DAG.getPOISON(VT: OrigResultTypes[0]), SDValue(NewNode, 0)}, dl: DL);
10955
10956 return SDValue(NewNode, 0);
10957 }
10958
10959 if (BaseOpcode->AtomicX2) {
10960 SmallVector<SDValue, 1> Elt;
10961 DAG.ExtractVectorElements(Op: SDValue(NewNode, 0), Args&: Elt, Start: 0, Count: 1);
10962 return DAG.getMergeValues(Ops: {Elt[0], SDValue(NewNode, 1)}, dl: DL);
10963 }
10964
10965 return constructRetValue(DAG, Result: NewNode, ResultTypes: OrigResultTypes, IsTexFail,
10966 Unpacked: Subtarget->hasUnpackedD16VMem(), IsD16, DMaskPop: DMaskLanes,
10967 NumVDataDwords, IsAtomicPacked16Bit, DL);
10968}
10969
10970SDValue SITargetLowering::lowerSBuffer(EVT VT, EVT MemVT, SDLoc DL,
10971 SDValue Chain, SDValue Rsrc,
10972 SDValue Offset, SDValue CachePolicy,
10973 SelectionDAG &DAG,
10974 MachineMemOperand *MMO) const {
10975 MachineFunction &MF = DAG.getMachineFunction();
10976 bool HasChainResult = MMO != nullptr;
10977
10978 // SBUFFER_LOAD only produces values that fill whole SGPRs, apart from the
10979 // subword loads below.
10980 bool IsSubwordLoad = (MemVT == MVT::i8 || MemVT == MVT::i16) &&
10981 Subtarget->hasScalarSubwordLoads();
10982 if ((!isTypeLegal(VT) || VT.getSizeInBits() % 32 != 0) && !IsSubwordLoad) {
10983 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10984 MF.getFunction(), "unsupported s_buffer_load result type",
10985 DL.getDebugLoc()));
10986 EVT ResultTypes[] = {VT, MVT::Other};
10987 return DAG.getErrorMergeValues(
10988 ResultTypes: ArrayRef(ResultTypes, HasChainResult ? 2 : 1), Chain, dl: DL);
10989 }
10990
10991 if (!HasChainResult) {
10992 const DataLayout &DataLayout = DAG.getDataLayout();
10993 Align Alignment =
10994 DataLayout.getABITypeAlign(Ty: MemVT.getTypeForEVT(Context&: *DAG.getContext()));
10995
10996 MMO = MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
10997 F: MachineMemOperand::MOLoad |
10998 MachineMemOperand::MODereferenceable |
10999 MachineMemOperand::MOInvariant,
11000 Size: MemVT.getStoreSize(), BaseAlignment: Alignment);
11001 }
11002
11003 if (!Offset->isDivergent()) {
11004 SDValue Ops[] = {Chain, Rsrc, Offset, CachePolicy};
11005
11006 // Lower llvm.amdgcn.*s.buffer.load.{i,u}N intrinsics. First, generate
11007 // s_buffer_load_u* for signed and unsigned load instructions. Next, DAG
11008 // combiner tries to merge the s_buffer_load_uN with a sext instruction
11009 // (performSignExtendInRegCombine()) and it replaces s_buffer_load_uN with
11010 // s_buffer_load_iN.
11011 auto HandleScalarSubwordLoads = [&](unsigned Opcode) -> SDValue {
11012 SDValue BufferLoad = DAG.getMemIntrinsicNode(
11013 Opcode, dl: DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other), Ops, MemVT, MMO);
11014 SDValue LoadVal = DAG.getAnyExtOrTrunc(
11015 Op: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: BufferLoad), DL, VT);
11016 if (HasChainResult)
11017 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
11018 return LoadVal;
11019 };
11020 if (MemVT == MVT::i8 && Subtarget->hasScalarSubwordLoads())
11021 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_UBYTE);
11022
11023 if (MemVT == MVT::i16 && Subtarget->hasScalarSubwordLoads())
11024 return HandleScalarSubwordLoads(AMDGPUISD::SBUFFER_LOAD_USHORT);
11025
11026 // Widen vec3 load to vec4. Only 32-bit elements have a vec4 pattern.
11027 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
11028 VT.getVectorElementType().getSizeInBits() == 32 &&
11029 !Subtarget->hasScalarDwordx3Loads()) {
11030 EVT WidenedVT =
11031 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
11032 auto WidenedOp = DAG.getMemIntrinsicNode(
11033 Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL, VTList: DAG.getVTList(VT1: WidenedVT, VT2: MVT::Other),
11034 Ops, MemVT: WidenedVT,
11035 MMO: MF.getMachineMemOperand(MMO, Offset: 0, Size: WidenedVT.getStoreSize()));
11036 auto Subvector = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WidenedOp,
11037 N2: DAG.getVectorIdxConstant(Val: 0, DL));
11038 if (HasChainResult)
11039 return DAG.getMergeValues(Ops: {Subvector, WidenedOp.getValue(R: 1)}, dl: DL);
11040 return Subvector;
11041 }
11042
11043 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL,
11044 VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops, MemVT,
11045 MMO);
11046 }
11047
11048 // We have a divergent offset. Emit a MUBUF buffer load instead. We can
11049 // assume that the buffer is unswizzled.
11050 SDValue Ops[] = {
11051 Chain, // Chain
11052 Rsrc, // rsrc
11053 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11054 {}, // voffset
11055 {}, // soffset
11056 {}, // offset
11057 CachePolicy, // cachepolicy
11058 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11059 };
11060 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
11061 Subtarget->hasScalarSubwordLoads()) {
11062 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
11063 SDValue Load = handleByteShortBufferLoads(DAG, LoadVT: MemVT, DL, Ops, MMO);
11064 SDValue LoadVal = DAG.getAnyExtOrTrunc(Op: Load.getOperand(i: 0), DL, VT);
11065 if (HasChainResult)
11066 return DAG.getMergeValues(Ops: {LoadVal, Load.getOperand(i: 1)}, dl: DL);
11067 return LoadVal;
11068 }
11069
11070 SmallVector<SDValue, 4> Loads;
11071 unsigned NumLoads = 1;
11072 MVT LoadVT = VT.getSimpleVT();
11073 unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
11074 assert((LoadVT.getScalarType() == MVT::i32 ||
11075 LoadVT.getScalarType() == MVT::f32));
11076
11077 if (NumElts == 8 || NumElts == 16) {
11078 NumLoads = NumElts / 4;
11079 LoadVT = MVT::getVectorVT(VT: LoadVT.getScalarType(), NumElements: 4);
11080 }
11081
11082 SDVTList VTList = DAG.getVTList(VTs: {LoadVT, MVT::Other});
11083
11084 // Use the alignment to ensure that the required offsets will fit into the
11085 // immediate offsets.
11086 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3],
11087 Alignment: NumLoads > 1 ? Align(16 * NumLoads) : Align(4));
11088
11089 uint64_t InstOffset = Ops[5]->getAsZExtVal();
11090 unsigned LoadSize = LoadVT.getStoreSize();
11091 for (unsigned i = 0; i < NumLoads; ++i) {
11092 Ops[5] = DAG.getTargetConstant(Val: InstOffset + 16 * i, DL, VT: MVT::i32);
11093 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(MMO, Offset: 16 * i, Size: LoadSize);
11094 Loads.push_back(Elt: getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops,
11095 MemVT: LoadVT, MMO: LoadMMO, DAG));
11096 }
11097
11098 if (NumElts == 8 || NumElts == 16) {
11099 SDValue LoadVal = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Loads);
11100 if (HasChainResult) {
11101 SmallVector<SDValue, 4> LoadChains;
11102 for (SDValue Load : Loads)
11103 LoadChains.push_back(Elt: Load.getValue(R: 1));
11104 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: LoadChains);
11105 return DAG.getMergeValues(Ops: {LoadVal, Chain}, dl: DL);
11106 }
11107 return LoadVal;
11108 }
11109
11110 return Loads[0];
11111}
11112
11113SDValue SITargetLowering::lowerWaveID(SelectionDAG &DAG, SDValue Op) const {
11114 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
11115 if (!Subtarget->hasArchitectedSGPRs())
11116 return {};
11117 SDLoc SL(Op);
11118 MVT VT = MVT::i32;
11119 SDValue TTMP8 = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: AMDGPU::TTMP8, VT);
11120 return DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT, N1: TTMP8,
11121 N2: DAG.getConstant(Val: 25, DL: SL, VT), N3: DAG.getConstant(Val: 5, DL: SL, VT));
11122}
11123
11124SDValue SITargetLowering::lowerConstHwRegRead(SelectionDAG &DAG, SDValue Op,
11125 AMDGPU::Hwreg::Id HwReg,
11126 unsigned LowBit,
11127 unsigned Width) const {
11128 SDLoc SL(Op);
11129 using namespace AMDGPU::Hwreg;
11130 return {DAG.getMachineNode(
11131 Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT: MVT::i32,
11132 Op1: DAG.getTargetConstant(Val: HwregEncoding::encode(Values: HwReg, Values: LowBit, Values: Width),
11133 DL: SL, VT: MVT::i32)),
11134 0};
11135}
11136
11137SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
11138 unsigned Dim,
11139 const ArgDescriptor &Arg) const {
11140 SDLoc SL(Op);
11141 MachineFunction &MF = DAG.getMachineFunction();
11142 unsigned MaxID = Subtarget->getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: Dim);
11143 if (MaxID == 0)
11144 return DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
11145
11146 // It's undefined behavior if a function marked with the amdgpu-no-*
11147 // attributes uses the corresponding intrinsic.
11148 if (!Arg)
11149 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
11150
11151 SDValue Val = loadInputValue(DAG, RC: &AMDGPU::VGPR_32RegClass, VT: MVT::i32,
11152 SL: SDLoc(DAG.getEntryNode()), Arg);
11153
11154 // Don't bother inserting AssertZext for packed IDs since we're emitting the
11155 // masking operations anyway.
11156 //
11157 // TODO: We could assert the top bit is 0 for the source copy.
11158 if (Arg.isMasked())
11159 return Val;
11160
11161 // Preserve the known bits after expansion to a copy.
11162 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_width(Value: MaxID));
11163 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Val,
11164 N2: DAG.getValueType(SmallVT));
11165}
11166
11167SDValue SITargetLowering::lowerFromFP8(SDValue Op, bool IsBF8,
11168 SelectionDAG &DAG) const {
11169 SDLoc SL(Op);
11170 SDValue Src = Op.getOperand(i: 0);
11171 EVT DstVT = Op.getValueType();
11172 bool IsF16 = DstVT.getVectorElementType() == MVT::f16;
11173 assert((!IsF16 || Subtarget->hasFP8F16ConversionInsts()) &&
11174 "fp8/bf8 -> f16 conversion requires FP8F16ConversionInsts");
11175
11176 unsigned Opc;
11177 if (IsF16)
11178 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F16_BF8 : AMDGPUISD::CVT_PK_F16_FP8;
11179 else
11180 Opc = IsBF8 ? AMDGPUISD::CVT_PK_F32_BF8 : AMDGPUISD::CVT_PK_F32_FP8;
11181
11182 // Pack the two i8 lanes into the integer type the packed HW node reads. The
11183 // f16 form takes i16 and the f32 form takes i32. v2i8 bitcasts to i16
11184 // directly and the f32 node reads the low half of an any-extended i32.
11185 EVT PackedVT =
11186 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstVT.getScalarSizeInBits());
11187 SDValue AsI16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
11188 SDValue Packed = DAG.getAnyExtOrTrunc(Op: AsI16, DL: SL, VT: PackedVT);
11189 return DAG.getNode(Opcode: Opc, DL: SL, VT: DstVT, Operand: Packed);
11190}
11191
11192SDValue
11193SITargetLowering::LowerCONVERT_FROM_ARBITRARY_FP(SDValue Op,
11194 SelectionDAG &DAG) const {
11195 // Handle the OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 on subtargets
11196 // with matching HW conversions. Other formats use the generic expansion.
11197 APFloatBase::Semantics FPSemantic =
11198 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(i: 1));
11199 const bool IsFP8 = FPSemantic == APFloatBase::S_Float8E4M3FN;
11200 const bool IsBF8 = FPSemantic == APFloatBase::S_Float8E5M2;
11201 const bool IsE5M3 = FPSemantic == APFloatBase::S_Float8E5M3FNU;
11202 const bool HasE5M3ConversionInsts =
11203 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11204 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11205 if (!IsSupported)
11206 return SDValue();
11207
11208 EVT DstVT = Op.getValueType();
11209 // The custom action for a v2i8 source also reaches half conversions on
11210 // targets which only have FP8-to-f32 instructions.
11211 if (DstVT.getScalarType() == MVT::f16 &&
11212 !Subtarget->hasFP8F16ConversionInsts())
11213 return SDValue();
11214
11215 if (IsE5M3) {
11216 if (DstVT.getScalarType() != MVT::f32)
11217 return SDValue();
11218
11219 SDLoc SL(Op);
11220 SDValue Src = Op.getOperand(i: 0);
11221 assert((!DstVT.isVector() || DstVT == MVT::v2f32) &&
11222 "only the v2f32 vector result is custom lowered");
11223
11224 if (DstVT.isVector())
11225 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
11226 Src = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
11227
11228 auto ConvertByte = [&](unsigned ByteSel) {
11229 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_FP8_E5M3, DL: SL, VT: MVT::f32, N1: Src,
11230 N2: DAG.getTargetConstant(Val: ByteSel, DL: SL, VT: MVT::i32));
11231 };
11232
11233 if (!DstVT.isVector())
11234 return ConvertByte(0);
11235 return DAG.getBuildVector(VT: DstVT, DL: SL, Ops: {ConvertByte(0), ConvertByte(1)});
11236 }
11237
11238 if (!DstVT.isVector()) {
11239 SDValue Src = Op.getOperand(i: 0);
11240 if (Src.getValueType() != MVT::i32) {
11241 SDLoc SL(Op);
11242 SDValue SrcI32 = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
11243 return DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: SL, VT: DstVT, N1: SrcI32,
11244 N2: Op.getOperand(i: 1));
11245 }
11246 return Op;
11247 }
11248
11249 EVT EltVT = DstVT.getVectorElementType();
11250 if (EltVT == MVT::f16 || EltVT == MVT::f32)
11251 return lowerFromFP8(Op, IsBF8, DAG);
11252 return SDValue();
11253}
11254
11255SDValue SITargetLowering::lowerToFP8(SDValue Op, bool IsBF8, bool IsE5M3,
11256 SelectionDAG &DAG) const {
11257 SDLoc SL(Op);
11258 SDValue Src = Op.getOperand(i: 0);
11259 EVT ResVT = Op.getValueType();
11260 bool IsF16 = Src.getValueType().getScalarType() == MVT::f16;
11261 assert((!IsF16 || Subtarget->hasF16FP8ConversionInsts()) &&
11262 "f16 -> fp8/bf8 conversion requires F16FP8ConversionInsts");
11263 assert((!ResVT.isVector() || ResVT == MVT::v2i8) &&
11264 "only the v2i8 vector result is custom lowered");
11265
11266 if (IsF16) {
11267 unsigned Opc =
11268 IsBF8 ? AMDGPUISD::CVT_PK_BF8_F16 : AMDGPUISD::CVT_PK_FP8_F16;
11269 SDValue Bytes = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i16, Operand: Src);
11270 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResVT, Operand: Bytes);
11271 }
11272
11273 unsigned Opc = IsBF8 ? AMDGPUISD::CVT_PK_BF8_F32
11274 : IsE5M3 ? AMDGPUISD::CVT_PK_FP8_F32_E5M3
11275 : AMDGPUISD::CVT_PK_FP8_F32;
11276 SDValue PoisonI32 = DAG.getPOISON(VT: MVT::i32);
11277 SDValue WordSel = DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1);
11278
11279 if (!ResVT.isVector()) {
11280 // Convert one lane, the second is unused. Feed it the same source so the
11281 // instruction does not read an undefined register.
11282 SDValue Packed =
11283 DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Src, N2: Src, N3: PoisonI32, N4: WordSel);
11284 return DAG.getAnyExtOrTrunc(Op: Packed, DL: SL, VT: ResVT);
11285 }
11286
11287 SDValue A = DAG.getExtractVectorElt(DL: SL, VT: MVT::f32, Vec: Src, Idx: 0);
11288 SDValue B = DAG.getExtractVectorElt(DL: SL, VT: MVT::f32, Vec: Src, Idx: 1);
11289 SDValue Packed = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: A, N2: B, N3: PoisonI32, N4: WordSel);
11290 SDValue Bytes = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Packed);
11291 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResVT, Operand: Bytes);
11292}
11293
11294SDValue
11295SITargetLowering::LowerCONVERT_TO_ARBITRARY_FP(SDValue Op,
11296 SelectionDAG &DAG) const {
11297 // The OCP FP8 formats (E4M3FN, E5M2) and unsigned E5M3 map to HW conversions
11298 // on subtargets that support them. Everything else uses generic expansion.
11299 APFloatBase::Semantics Sem =
11300 static_cast<APFloatBase::Semantics>(Op.getConstantOperandVal(i: 1));
11301 const bool IsFP8 = Sem == APFloatBase::S_Float8E4M3FN;
11302 const bool IsBF8 = Sem == APFloatBase::S_Float8E5M2;
11303 const bool IsE5M3 = Sem == APFloatBase::S_Float8E5M3FNU;
11304 const bool HasE5M3ConversionInsts =
11305 Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
11306 const bool IsSupported = IsFP8 || IsBF8 || (IsE5M3 && HasE5M3ConversionInsts);
11307 if (!IsSupported)
11308 return SDValue();
11309
11310 // The HW conversions only support nearest-even. The OCP conversions do not
11311 // saturate. The unsigned E5M3 conversion always clamps out-of-range inputs,
11312 // which also refines the non-saturating form where those inputs are poison.
11313 if (static_cast<RoundingMode>(Op.getConstantOperandVal(i: 2)) !=
11314 RoundingMode::NearestTiesToEven)
11315 return SDValue();
11316 if (!IsE5M3 && Op.getConstantOperandVal(i: 3) != 0)
11317 return SDValue();
11318
11319 EVT SrcEltVT = Op.getOperand(i: 0).getValueType().getScalarType();
11320 // The f32 form is built here rather than by a tablegen pattern because the
11321 // HW result is i32 while the node result is i16 after the i8 promotion.
11322 if (SrcEltVT == MVT::f32)
11323 return lowerToFP8(Op, IsBF8, IsE5M3, DAG);
11324 if (!IsE5M3 && SrcEltVT == MVT::f16 &&
11325 Subtarget->hasF16FP8ConversionInsts()) {
11326 // A scalar conversion is selected from the generic node by tablegen, only
11327 // the illegal v2i8 result type needs lowering here.
11328 if (!Op.getValueType().isVector())
11329 return Op;
11330 return lowerToFP8(Op, IsBF8, IsE5M3: false, DAG);
11331 }
11332 return SDValue();
11333}
11334
11335SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11336 SelectionDAG &DAG) const {
11337 MachineFunction &MF = DAG.getMachineFunction();
11338 auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
11339
11340 EVT VT = Op.getValueType();
11341 SDLoc DL(Op);
11342 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
11343
11344 // TODO: Should this propagate fast-math-flags?
11345
11346 switch (IntrinsicID) {
11347 case Intrinsic::amdgcn_wave_reduce_min:
11348 case Intrinsic::amdgcn_wave_reduce_umin:
11349 case Intrinsic::amdgcn_wave_reduce_fmin:
11350 case Intrinsic::amdgcn_wave_reduce_max:
11351 case Intrinsic::amdgcn_wave_reduce_umax:
11352 case Intrinsic::amdgcn_wave_reduce_fmax:
11353 case Intrinsic::amdgcn_wave_reduce_add:
11354 case Intrinsic::amdgcn_wave_reduce_fadd:
11355 case Intrinsic::amdgcn_wave_reduce_sub:
11356 case Intrinsic::amdgcn_wave_reduce_fsub:
11357 case Intrinsic::amdgcn_wave_reduce_and:
11358 case Intrinsic::amdgcn_wave_reduce_or:
11359 case Intrinsic::amdgcn_wave_reduce_xor: {
11360 EVT SrcVT = Op.getOperand(i: 1).getValueType();
11361 if (SrcVT.getFixedSizeInBits() == 16) {
11362 bool IsFPOp = SrcVT.isFloatingPoint();
11363 bool NeedsSignExt = IntrinsicID == Intrinsic::amdgcn_wave_reduce_min ||
11364 IntrinsicID == Intrinsic::amdgcn_wave_reduce_max ||
11365 IntrinsicID == Intrinsic::amdgcn_wave_reduce_add ||
11366 IntrinsicID == Intrinsic::amdgcn_wave_reduce_sub;
11367 unsigned ExtOpc = IsFPOp ? ISD::FP_EXTEND
11368 : NeedsSignExt ? ISD::SIGN_EXTEND
11369 : ISD::ZERO_EXTEND;
11370 auto SrcType = IsFPOp ? MVT::f16 : MVT::i16;
11371 auto ExtType = IsFPOp ? MVT::f32 : MVT::i32;
11372 SDValue ExtendedSrc = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtType, Operand: Op.getOperand(i: 1));
11373 SDValue Strategy = Op.getOperand(i: 2);
11374 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ExtType,
11375 N1: Op.getOperand(i: 0), N2: ExtendedSrc, N3: Strategy);
11376 if (IsFPOp)
11377 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: SrcType, N1: Result,
11378 N2: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32));
11379 else
11380 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcType, Operand: Result);
11381 }
11382 return SDValue();
11383 }
11384 case Intrinsic::amdgcn_implicit_buffer_ptr: {
11385 if (!MFI->getUserSGPRInfo().hasImplicitBufferPtr()) {
11386 const char *Error =
11387 Subtarget->hasArchitectedFlatScratch()
11388 ? "intrinsic not supported on subtarget"
11389 : !Subtarget->isMesa3DOS()
11390 ? "intrinsic not supported for OS"
11391 : "intrinsic not supported for calling convention";
11392 DAG.getContext()->diagnose(
11393 DI: DiagnosticInfoUnsupported(MF.getFunction(), Error, DL.getDebugLoc()));
11394 return DAG.getPOISON(VT);
11395 }
11396
11397 return getPreloadedValue(DAG, MFI: *MFI, VT,
11398 PVID: AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
11399 }
11400 case Intrinsic::amdgcn_dispatch_ptr:
11401 case Intrinsic::amdgcn_queue_ptr: {
11402 if (!Subtarget->isAmdHsaOrMesa(F: MF.getFunction())) {
11403 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11404 MF.getFunction(), "unsupported hsa intrinsic without hsa target",
11405 DL.getDebugLoc()));
11406 return DAG.getPOISON(VT);
11407 }
11408
11409 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
11410 ? AMDGPUFunctionArgInfo::DISPATCH_PTR
11411 : AMDGPUFunctionArgInfo::QUEUE_PTR;
11412 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: RegID);
11413 }
11414 case Intrinsic::amdgcn_implicitarg_ptr: {
11415 if (MFI->isEntryFunction())
11416 return getImplicitArgPtr(DAG, SL: DL);
11417 return getPreloadedValue(DAG, MFI: *MFI, VT,
11418 PVID: AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
11419 }
11420 case Intrinsic::amdgcn_kernarg_segment_ptr: {
11421 if (!AMDGPU::isKernel(F: MF.getFunction())) {
11422 // This only makes sense to call in a kernel, so just lower to null.
11423 return DAG.getConstant(Val: 0, DL, VT);
11424 }
11425
11426 return getPreloadedValue(DAG, MFI: *MFI, VT,
11427 PVID: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
11428 }
11429 case Intrinsic::amdgcn_dispatch_id: {
11430 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: AMDGPUFunctionArgInfo::DISPATCH_ID);
11431 }
11432 case Intrinsic::amdgcn_rcp:
11433 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT, Operand: Op.getOperand(i: 1));
11434 case Intrinsic::amdgcn_rsq:
11435 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11436 case Intrinsic::amdgcn_rsq_legacy:
11437 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11438 return emitRemovedIntrinsicError(DAG, DL, VT);
11439 return SDValue();
11440 case Intrinsic::amdgcn_rcp_legacy:
11441 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
11442 return emitRemovedIntrinsicError(DAG, DL, VT);
11443 return DAG.getNode(Opcode: AMDGPUISD::RCP_LEGACY, DL, VT, Operand: Op.getOperand(i: 1));
11444 case Intrinsic::amdgcn_fma_legacy:
11445 case Intrinsic::amdgcn_sudot4:
11446 case Intrinsic::amdgcn_sudot8:
11447 case Intrinsic::amdgcn_tanh:
11448 return SDValue();
11449 case Intrinsic::amdgcn_rsq_clamp: {
11450 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11451 return DAG.getNode(Opcode: AMDGPUISD::RSQ_CLAMP, DL, VT, Operand: Op.getOperand(i: 1));
11452
11453 Type *Type = VT.getTypeForEVT(Context&: *DAG.getContext());
11454 APFloat Max = APFloat::getLargest(Sem: Type->getFltSemantics());
11455 APFloat Min = APFloat::getLargest(Sem: Type->getFltSemantics(), Negative: true);
11456
11457 SDValue Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11458 SDValue Tmp =
11459 DAG.getNode(Opcode: ISD::FMINNUM, DL, VT, N1: Rsq, N2: DAG.getConstantFP(Val: Max, DL, VT));
11460 return DAG.getNode(Opcode: ISD::FMAXNUM, DL, VT, N1: Tmp,
11461 N2: DAG.getConstantFP(Val: Min, DL, VT));
11462 }
11463 case Intrinsic::r600_read_ngroups_x:
11464 if (Subtarget->isAmdHsaOS())
11465 return emitNonHSAIntrinsicError(DAG, DL, VT);
11466
11467 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11468 Offset: SI::KernelInputOffsets::NGROUPS_X, Alignment: Align(4),
11469 Signed: false);
11470 case Intrinsic::r600_read_ngroups_y:
11471 if (Subtarget->isAmdHsaOS())
11472 return emitNonHSAIntrinsicError(DAG, DL, VT);
11473
11474 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11475 Offset: SI::KernelInputOffsets::NGROUPS_Y, Alignment: Align(4),
11476 Signed: false);
11477 case Intrinsic::r600_read_ngroups_z:
11478 if (Subtarget->isAmdHsaOS())
11479 return emitNonHSAIntrinsicError(DAG, DL, VT);
11480
11481 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11482 Offset: SI::KernelInputOffsets::NGROUPS_Z, Alignment: Align(4),
11483 Signed: false);
11484 case Intrinsic::r600_read_local_size_x:
11485 if (Subtarget->isAmdHsaOS())
11486 return emitNonHSAIntrinsicError(DAG, DL, VT);
11487
11488 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11489 Offset: SI::KernelInputOffsets::LOCAL_SIZE_X);
11490 case Intrinsic::r600_read_local_size_y:
11491 if (Subtarget->isAmdHsaOS())
11492 return emitNonHSAIntrinsicError(DAG, DL, VT);
11493
11494 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11495 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Y);
11496 case Intrinsic::r600_read_local_size_z:
11497 if (Subtarget->isAmdHsaOS())
11498 return emitNonHSAIntrinsicError(DAG, DL, VT);
11499
11500 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11501 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Z);
11502 case Intrinsic::amdgcn_workgroup_id_x:
11503 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11504 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
11505 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
11506 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
11507 case Intrinsic::amdgcn_workgroup_id_y:
11508 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11509 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
11510 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
11511 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
11512 case Intrinsic::amdgcn_workgroup_id_z:
11513 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11514 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
11515 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
11516 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
11517 case Intrinsic::amdgcn_cluster_id_x:
11518 return Subtarget->hasClusters()
11519 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11520 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_X)
11521 : DAG.getPOISON(VT);
11522 case Intrinsic::amdgcn_cluster_id_y:
11523 return Subtarget->hasClusters()
11524 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11525 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y)
11526 : DAG.getPOISON(VT);
11527 case Intrinsic::amdgcn_cluster_id_z:
11528 return Subtarget->hasClusters()
11529 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11530 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z)
11531 : DAG.getPOISON(VT);
11532 case Intrinsic::amdgcn_cluster_workgroup_id_x:
11533 return Subtarget->hasClusters()
11534 ? getPreloadedValue(
11535 DAG, MFI: *MFI, VT,
11536 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X)
11537 : DAG.getPOISON(VT);
11538 case Intrinsic::amdgcn_cluster_workgroup_id_y:
11539 return Subtarget->hasClusters()
11540 ? getPreloadedValue(
11541 DAG, MFI: *MFI, VT,
11542 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y)
11543 : DAG.getPOISON(VT);
11544 case Intrinsic::amdgcn_cluster_workgroup_id_z:
11545 return Subtarget->hasClusters()
11546 ? getPreloadedValue(
11547 DAG, MFI: *MFI, VT,
11548 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z)
11549 : DAG.getPOISON(VT);
11550 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
11551 return Subtarget->hasClusters()
11552 ? lowerConstHwRegRead(DAG, Op, HwReg: AMDGPU::Hwreg::ID_IB_STS2, LowBit: 21, Width: 4)
11553 : SDValue();
11554 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
11555 return Subtarget->hasClusters()
11556 ? getPreloadedValue(
11557 DAG, MFI: *MFI, VT,
11558 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X)
11559 : DAG.getPOISON(VT);
11560 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
11561 return Subtarget->hasClusters()
11562 ? getPreloadedValue(
11563 DAG, MFI: *MFI, VT,
11564 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y)
11565 : DAG.getPOISON(VT);
11566 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
11567 return Subtarget->hasClusters()
11568 ? getPreloadedValue(
11569 DAG, MFI: *MFI, VT,
11570 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z)
11571 : DAG.getPOISON(VT);
11572 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
11573 return Subtarget->hasClusters()
11574 ? getPreloadedValue(
11575 DAG, MFI: *MFI, VT,
11576 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID)
11577 : DAG.getPOISON(VT);
11578 case Intrinsic::amdgcn_wave_id:
11579 return lowerWaveID(DAG, Op);
11580 case Intrinsic::amdgcn_lds_kernel_id: {
11581 if (MFI->isEntryFunction())
11582 return getLDSKernelId(DAG, SL: DL);
11583 return getPreloadedValue(DAG, MFI: *MFI, VT,
11584 PVID: AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
11585 }
11586 case Intrinsic::amdgcn_workitem_id_x:
11587 return lowerWorkitemID(DAG, Op, Dim: 0, Arg: MFI->getArgInfo().WorkItemIDX);
11588 case Intrinsic::amdgcn_workitem_id_y:
11589 return lowerWorkitemID(DAG, Op, Dim: 1, Arg: MFI->getArgInfo().WorkItemIDY);
11590 case Intrinsic::amdgcn_workitem_id_z:
11591 return lowerWorkitemID(DAG, Op, Dim: 2, Arg: MFI->getArgInfo().WorkItemIDZ);
11592 case Intrinsic::amdgcn_wavefrontsize:
11593 return DAG.getConstant(Val: MF.getSubtarget<GCNSubtarget>().getWavefrontSize(),
11594 DL: SDLoc(Op), VT: MVT::i32);
11595 case Intrinsic::amdgcn_s_buffer_load: {
11596 unsigned CPol = Op.getConstantOperandVal(i: 3);
11597 // s_buffer_load, because of how it's optimized, can't be volatile
11598 // so reject ones with the volatile bit set.
11599 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11600 ? AMDGPU::CPol::ALL
11601 : AMDGPU::CPol::ALL_pregfx12))
11602 return Op;
11603 return lowerSBuffer(VT, MemVT: VT, DL, Chain: DAG.getEntryNode(), Rsrc: Op.getOperand(i: 1),
11604 Offset: Op.getOperand(i: 2), CachePolicy: Op.getOperand(i: 3), DAG);
11605 }
11606 case Intrinsic::amdgcn_fdiv_fast:
11607 return lowerFDIV_FAST(Op, DAG);
11608 case Intrinsic::amdgcn_sin:
11609 return DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL, VT, Operand: Op.getOperand(i: 1));
11610
11611 case Intrinsic::amdgcn_cos:
11612 return DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL, VT, Operand: Op.getOperand(i: 1));
11613
11614 case Intrinsic::amdgcn_mul_u24:
11615 return DAG.getNode(Opcode: AMDGPUISD::MUL_U24, DL, VT, N1: Op.getOperand(i: 1),
11616 N2: Op.getOperand(i: 2));
11617 case Intrinsic::amdgcn_mul_i24:
11618 return DAG.getNode(Opcode: AMDGPUISD::MUL_I24, DL, VT, N1: Op.getOperand(i: 1),
11619 N2: Op.getOperand(i: 2));
11620
11621 case Intrinsic::amdgcn_log_clamp: {
11622 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11623 return SDValue();
11624
11625 return emitRemovedIntrinsicError(DAG, DL, VT);
11626 }
11627 case Intrinsic::amdgcn_fract:
11628 return DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: Op.getOperand(i: 1));
11629
11630 case Intrinsic::amdgcn_class: {
11631 SDValue Src = Op.getOperand(i: 1);
11632 EVT SrcVT = Src.getValueType();
11633 bool IsLegal = SrcVT == MVT::f32 || SrcVT == MVT::f64 ||
11634 (SrcVT == MVT::f16 && Subtarget->has16BitInsts());
11635 if (!IsLegal) {
11636 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11637 DAG.getMachineFunction().getFunction(),
11638 "llvm.amdgcn.class only supports f16, f32, and f64",
11639 DL.getDebugLoc()));
11640 return DAG.getPOISON(VT);
11641 }
11642 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT, N1: Src, N2: Op.getOperand(i: 2));
11643 }
11644 case Intrinsic::amdgcn_div_fmas:
11645 return DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL, VT, N1: Op.getOperand(i: 1),
11646 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11647
11648 case Intrinsic::amdgcn_div_fixup:
11649 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL, VT, N1: Op.getOperand(i: 1),
11650 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11651
11652 case Intrinsic::amdgcn_div_scale: {
11653 const ConstantSDNode *Param = cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11654
11655 // Translate to the operands expected by the machine instruction. The
11656 // first parameter must be the same as the first instruction.
11657 SDValue Numerator = Op.getOperand(i: 1);
11658 SDValue Denominator = Op.getOperand(i: 2);
11659
11660 // Note this order is opposite of the machine instruction's operations,
11661 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The
11662 // intrinsic has the numerator as the first operand to match a normal
11663 // division operation.
11664
11665 SDValue Src0 = Param->isAllOnes() ? Numerator : Denominator;
11666
11667 return DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL, VTList: Op->getVTList(), N1: Src0,
11668 N2: Denominator, N3: Numerator);
11669 }
11670 case Intrinsic::amdgcn_ballot:
11671 return lowerBALLOTIntrinsic(TLI: *this, N: Op.getNode(), DAG);
11672 case Intrinsic::amdgcn_fmed3:
11673 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL, VT, N1: Op.getOperand(i: 1),
11674 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), Flags: Op->getFlags());
11675 case Intrinsic::amdgcn_fdot2:
11676 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL, VT, N1: Op.getOperand(i: 1),
11677 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11678 case Intrinsic::amdgcn_fmul_legacy:
11679 return DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL, VT, N1: Op.getOperand(i: 1),
11680 N2: Op.getOperand(i: 2));
11681 case Intrinsic::amdgcn_sbfe:
11682 case Intrinsic::amdgcn_ubfe:
11683 return lowerBFEIntrinsic(Op, DAG, IntrinsicID);
11684 case Intrinsic::amdgcn_cvt_pkrtz:
11685 case Intrinsic::amdgcn_cvt_pknorm_i16:
11686 case Intrinsic::amdgcn_cvt_pknorm_u16:
11687 case Intrinsic::amdgcn_cvt_pk_i16:
11688 case Intrinsic::amdgcn_cvt_pk_u16: {
11689 // FIXME: Stop adding cast if v2f16/v2i16 are legal.
11690 EVT VT = Op.getValueType();
11691 unsigned Opcode;
11692
11693 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
11694 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32;
11695 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
11696 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
11697 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
11698 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
11699 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
11700 Opcode = AMDGPUISD::CVT_PK_I16_I32;
11701 else
11702 Opcode = AMDGPUISD::CVT_PK_U16_U32;
11703
11704 if (isTypeLegal(VT))
11705 return DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11706
11707 SDValue Node =
11708 DAG.getNode(Opcode, DL, VT: MVT::i32, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11709 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Node);
11710 }
11711 case Intrinsic::amdgcn_fmad_ftz:
11712 return DAG.getNode(Opcode: AMDGPUISD::FMAD_FTZ, DL, VT, N1: Op.getOperand(i: 1),
11713 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11714
11715 case Intrinsic::amdgcn_if_break:
11716 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_IF_BREAK, dl: DL, VT,
11717 Op1: Op->getOperand(Num: 1), Op2: Op->getOperand(Num: 2)),
11718 0);
11719
11720 case Intrinsic::amdgcn_groupstaticsize: {
11721 Triple::OSType OS = getTargetMachine().getTargetTriple().getOS();
11722 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL)
11723 return Op;
11724
11725 const Module *M = MF.getFunction().getParent();
11726 const GlobalValue *GV =
11727 Intrinsic::getDeclarationIfExists(M, id: Intrinsic::amdgcn_groupstaticsize);
11728 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: 0,
11729 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11730 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11731 }
11732 case Intrinsic::amdgcn_is_shared:
11733 case Intrinsic::amdgcn_is_private: {
11734 SDLoc SL(Op);
11735 SDValue SrcVec =
11736 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
11737 SDValue SrcHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: SrcVec,
11738 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
11739
11740 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
11741 ? AMDGPUAS::LOCAL_ADDRESS
11742 : AMDGPUAS::PRIVATE_ADDRESS;
11743 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
11744 Subtarget->hasGloballyAddressableScratch()) {
11745 SDValue FlatScratchBaseHi(
11746 DAG.getMachineNode(
11747 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
11748 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, VT: MVT::i32)),
11749 0);
11750 // Test bits 63..58 against the aperture address.
11751 return DAG.getSetCC(
11752 DL: SL, VT: MVT::i1,
11753 LHS: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: SrcHi, N2: FlatScratchBaseHi),
11754 RHS: DAG.getConstant(Val: 1u << 26, DL: SL, VT: MVT::i32), Cond: ISD::SETULT);
11755 }
11756
11757 SDValue Aperture = getSegmentAperture(AS, DL: SL, DAG);
11758 return DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SrcHi, RHS: Aperture, Cond: ISD::SETEQ);
11759 }
11760 case Intrinsic::amdgcn_perm:
11761 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op.getOperand(i: 1),
11762 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11763 case Intrinsic::amdgcn_reloc_constant: {
11764 Module *M = MF.getFunction().getParent();
11765 const MDNode *Metadata = cast<MDNodeSDNode>(Val: Op.getOperand(i: 1))->getMD();
11766 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
11767 auto *RelocSymbol = cast<GlobalVariable>(
11768 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
11769 SDValue GA = DAG.getTargetGlobalAddress(GV: RelocSymbol, DL, VT: MVT::i32, offset: 0,
11770 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11771 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11772 }
11773 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
11774 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
11775 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
11776 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
11777 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
11778 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
11779 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
11780 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
11781 if (Op.getOperand(i: 4).getValueType() == MVT::i32)
11782 return SDValue();
11783
11784 SDLoc SL(Op);
11785 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i32);
11786 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11787 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2),
11788 N4: Op.getOperand(i: 3), N5: IndexKeyi32);
11789 }
11790 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
11791 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
11792 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
11793 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
11794 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
11795 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
11796 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
11797 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
11798 if (Op.getOperand(i: 4).getValueType() == MVT::i64)
11799 return SDValue();
11800
11801 SDLoc SL(Op);
11802 auto IndexKeyi64 =
11803 Op.getOperand(i: 4).getValueType() == MVT::v2i32
11804 ? DAG.getBitcast(VT: MVT::i64, V: Op.getOperand(i: 4))
11805 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i64);
11806 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11807 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11808 Op.getOperand(i: 3), IndexKeyi64, Op.getOperand(i: 5),
11809 Op.getOperand(i: 6)});
11810 }
11811 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
11812 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
11813 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
11814 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
11815 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
11816 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8: {
11817 EVT IndexKeyTy = IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
11818 ? MVT::i64
11819 : MVT::i32;
11820 if (Op.getOperand(i: 6).getValueType() == IndexKeyTy)
11821 return SDValue();
11822
11823 SDLoc SL(Op);
11824 auto IndexKey =
11825 Op.getOperand(i: 6).getValueType().isVector()
11826 ? DAG.getBitcast(VT: IndexKeyTy, V: Op.getOperand(i: 6))
11827 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: IndexKeyTy);
11828 SmallVector<SDValue> Args{
11829 Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11830 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11831 IndexKey, Op.getOperand(i: 7), Op.getOperand(i: 8)};
11832 if (IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8)
11833 Args.push_back(Elt: Op.getOperand(i: 9));
11834 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(), Ops: Args);
11835 }
11836 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
11837 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
11838 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
11839 if (Op.getOperand(i: 6).getValueType() == MVT::i32)
11840 return SDValue();
11841
11842 SDLoc SL(Op);
11843 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: MVT::i32);
11844 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11845 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11846 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11847 IndexKeyi32, Op.getOperand(i: 7)});
11848 }
11849 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
11850 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
11851 unsigned AFmt = (unsigned)Op.getConstantOperandVal(i: 1);
11852 unsigned BFmt = (unsigned)Op.getConstantOperandVal(i: 3);
11853 unsigned AScaleFmt = (unsigned)Op.getConstantOperandVal(i: 8);
11854 unsigned BScaleFmt = (unsigned)Op.getConstantOperandVal(i: 11);
11855 if (!AMDGPU::isValidWMMAScaleFmtCombination(AFmt, AScale: AScaleFmt, BFmt,
11856 BScale: BScaleFmt)) {
11857 DAG.getMachineFunction().getFunction().getContext().emitError(
11858 ErrorStr: "invalid matrix and scale format combination in wmma call");
11859 Op->print(OS&: errs());
11860 errs() << '\n';
11861 }
11862 return SDValue();
11863 }
11864 case Intrinsic::amdgcn_readlane:
11865 case Intrinsic::amdgcn_readfirstlane:
11866 case Intrinsic::amdgcn_writelane:
11867 case Intrinsic::amdgcn_permlane16:
11868 case Intrinsic::amdgcn_permlanex16:
11869 case Intrinsic::amdgcn_permlane64:
11870 case Intrinsic::amdgcn_set_inactive:
11871 case Intrinsic::amdgcn_set_inactive_chain_arg:
11872 case Intrinsic::amdgcn_mov_dpp8:
11873 case Intrinsic::amdgcn_update_dpp:
11874 case Intrinsic::amdgcn_permlane_bcast:
11875 case Intrinsic::amdgcn_permlane_up:
11876 case Intrinsic::amdgcn_permlane_down:
11877 case Intrinsic::amdgcn_permlane_xor:
11878 return lowerLaneOp(TLI: *this, N: Op.getNode(), DAG);
11879 case Intrinsic::amdgcn_dead: {
11880 SmallVector<SDValue, 8> Poisons;
11881 for (const EVT ValTy : Op.getNode()->values())
11882 Poisons.push_back(Elt: DAG.getPOISON(VT: ValTy));
11883 return DAG.getMergeValues(Ops: Poisons, dl: SDLoc(Op));
11884 }
11885 case Intrinsic::amdgcn_wave_shuffle:
11886 return lowerWaveShuffle(TLI: *this, N: Op.getNode(), DAG);
11887 default:
11888 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
11889 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
11890 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: false);
11891
11892 return Op;
11893 }
11894}
11895
11896// On targets not supporting constant in soffset field, turn zero to
11897// SGPR_NULL to avoid generating an extra s_mov with zero.
11898static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG,
11899 const GCNSubtarget *Subtarget) {
11900 if (Subtarget->hasRestrictedSOffset() && isNullConstant(V: SOffset))
11901 return DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32);
11902 return SOffset;
11903}
11904
11905// Lower the arguments of the buffer intrinsic \p Op into the operands of the
11906// corresponding AMDGPUISD buffer node: the chain, \p VData and the cmp operand
11907// if present, then rsrc, vindex, voffset, soffset, offset, [format,]
11908// cachepolicy and idxen.
11909SmallVector<SDValue, 10> SITargetLowering::getBufferIntrinsicOps(
11910 SDValue Op, const AMDGPU::BufferIntrinsicInfo &Info, SelectionDAG &DAG,
11911 SDValue VData) const {
11912 SDLoc DL(Op);
11913 auto GetArg = [&](unsigned Arg) {
11914 return Op.getOperand(i: AMDGPU::getSDOperandIdx(Arg));
11915 };
11916
11917 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: GetArg(Info.RsrcArg), DAG);
11918 auto [VOffset, Offset] =
11919 splitBufferOffsets(Offset: GetArg(*Info.getVOffsetArg()), DAG);
11920 SDValue SOffset =
11921 selectSOffset(SOffset: GetArg(*Info.getSOffsetArg()), DAG, Subtarget);
11922
11923 SmallVector<SDValue, 10> Ops = {Op.getOperand(i: 0)}; // Chain
11924 if (VData)
11925 Ops.push_back(Elt: VData);
11926 if (std::optional<unsigned> CmpArg = Info.getCmpArg())
11927 Ops.push_back(Elt: GetArg(*CmpArg));
11928 Ops.push_back(Elt: Rsrc);
11929 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
11930 Ops.push_back(Elt: VIndexArg ? GetArg(*VIndexArg)
11931 : DAG.getConstant(Val: 0, DL, VT: MVT::i32)); // vindex
11932 Ops.push_back(Elt: VOffset);
11933 Ops.push_back(Elt: SOffset);
11934 Ops.push_back(Elt: Offset);
11935 if (std::optional<unsigned> FormatArg = Info.getFormatArg())
11936 Ops.push_back(Elt: GetArg(*FormatArg));
11937 Ops.push_back(Elt: GetArg(*Info.getAuxArg())); // cachepolicy, swizzled buffer
11938 Ops.push_back(
11939 Elt: DAG.getTargetConstant(Val: VIndexArg.has_value(), DL, VT: MVT::i1)); // idxen
11940 return Ops;
11941}
11942
11943SDValue SITargetLowering::lowerBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
11944 unsigned NewOpcode) const {
11945 SDLoc DL(Op);
11946 const AMDGPU::BufferIntrinsicInfo &Info =
11947 *AMDGPU::lookupBufferIntrinsic(Intr: Op.getConstantOperandVal(i: 1));
11948
11949 SDValue VData = Op.getOperand(i: AMDGPU::getSDOperandIdx(Arg: *Info.getVDataArg()));
11950 if (VData.getValueSizeInBits() != 32 && VData.getValueSizeInBits() != 64) {
11951 SmallVector<EVT, 2> ResultTypes(Op->values());
11952 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
11953 Msg: "unsupported buffer atomic data type");
11954 }
11955 SmallVector<SDValue, 10> Ops = getBufferIntrinsicOps(Op, Info, DAG, VData);
11956
11957 auto *M = cast<MemSDNode>(Val&: Op);
11958
11959 EVT MemVT = VData.getValueType();
11960 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11961 MMO: M->getMemOperand());
11962}
11963
11964// Lower the buffer loads and the compare-and-swap atomics, which are all
11965// INTRINSIC_W_CHAIN nodes. Other atomics need an opcode for their operation,
11966// and so are handled by LowerINTRINSIC_W_CHAIN.
11967SDValue SITargetLowering::lowerBufferIntrinsicWithChain(
11968 SDValue Op, const AMDGPU::BufferIntrinsicInfo &Info,
11969 SelectionDAG &DAG) const {
11970 SDLoc DL(Op);
11971 auto *M = cast<MemSDNode>(Val&: Op);
11972 auto GetArg = [&](unsigned Arg) {
11973 return Op.getOperand(i: AMDGPU::getSDOperandIdx(Arg));
11974 };
11975
11976 switch (Info.Kind) {
11977 case AMDGPU::BufferKindSLoad: {
11978 SDValue CachePolicy = GetArg(*Info.getAuxArg());
11979 if (CachePolicy->getAsZExtVal() &
11980 ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11981 ? AMDGPU::CPol::ALL
11982 : AMDGPU::CPol::ALL_pregfx12))
11983 return Op;
11984
11985 return lowerSBuffer(
11986 VT: Op.getValueType(), MemVT: M->getMemoryVT(), DL, Chain: Op.getOperand(i: 0),
11987 Rsrc: bufferRsrcPtrToVector(MaybePointer: GetArg(Info.RsrcArg), DAG),
11988 Offset: GetArg(*Info.getSOffsetArg()), CachePolicy, DAG, MMO: M->getMemOperand());
11989 }
11990 case AMDGPU::BufferKindLoad:
11991 case AMDGPU::BufferKindAtomicLoad: {
11992 SmallVector<SDValue, 10> Ops = getBufferIntrinsicOps(Op, Info, DAG);
11993 if (!Info.isTyped())
11994 return lowerIntrinsicLoad(M, IsFormat: Info.isFormatFromRsrc(), DAG, Ops);
11995
11996 EVT LoadVT = Op.getValueType();
11997 if (LoadVT.getScalarSizeInBits() == 16)
11998 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
11999 Ops);
12000 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
12001 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
12002 DAG);
12003 }
12004 case AMDGPU::BufferKindAtomicCmpSwap: {
12005 SDValue Src = GetArg(*Info.getVDataArg());
12006 if (Src.getValueSizeInBits() != 32 && Src.getValueSizeInBits() != 64) {
12007 SmallVector<EVT, 2> ResultTypes(Op->values());
12008 return diagnoseUnsupportedImage(DAG, Op, ResultTypes, DL,
12009 Msg: "unsupported buffer atomic data type");
12010 }
12011 SmallVector<SDValue, 10> Ops = getBufferIntrinsicOps(Op, Info, DAG, VData: Src);
12012 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
12013 VTList: Op->getVTList(), Ops, MemVT: Op.getValueType(),
12014 MMO: M->getMemOperand());
12015 }
12016 default:
12017 llvm_unreachable("not a buffer intrinsic with a chain and a result");
12018 }
12019}
12020
12021static void initializeM0ToZeroForClusterLoad(SDValue Op, SelectionDAG &DAG,
12022 SDLoc DL) {
12023 SDNode *N = Op.getNode();
12024 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12025 unsigned NumOperands = N->getNumOperands();
12026 if (N->getOperand(Num: NumOperands - 1) == Zero)
12027 return;
12028 SmallVector<SDValue, 7> Ops(N->ops());
12029 Ops[NumOperands - 1] = Zero; // M0 = 0
12030 DAG.UpdateNodeOperands(N, Ops);
12031}
12032
12033// Try to resolve a named barrier object to a constant barrier ID.
12034// Named barrier IDs are encoded at bits [5:0] of a BARRIER address space
12035// address - other bits of that address are ignored as reserved.
12036static std::optional<uint32_t>
12037getConstantNamedBarrierNumber(SDValue BarOp, SelectionDAG &DAG) {
12038 std::optional<uint64_t> BarVal;
12039 if (auto *C = dyn_cast<ConstantSDNode>(Val&: BarOp)) {
12040 BarVal = C->getZExtValue();
12041 } else if (BarOp->isUndef()) {
12042 // Refine poison to the 0 barrier to turn things into noops, thus reducing
12043 // the risk of hitting an unallocated barrier.
12044 BarVal = 0;
12045 } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: BarOp)) {
12046 const auto *GV = cast<GlobalVariable>(Val: GA->getGlobal());
12047 if (AMDGPU::isNamedBarrier(GV: *GV) &&
12048 AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(GV: *GV,
12049 AS: AMDGPUAS::BARRIER)) {
12050 auto *MFI = DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
12051 // Record the use of the manualy-specified barrier address to make sure
12052 // this function has the right number of resources. This function is named
12053 // weird, and we're calling it here since we're skipping the GlobalAddress
12054 // lowering with this pattern match.
12055 BarVal = MFI->allocateBarrierGlobal(DL: DAG.getDataLayout(), GV: *GV) +
12056 GA->getOffset();
12057 }
12058 }
12059 if (BarVal)
12060 return *BarVal & 0x3F;
12061 return std::nullopt;
12062}
12063
12064SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
12065 SelectionDAG &DAG) const {
12066 unsigned IntrID = Op.getConstantOperandVal(i: 1);
12067 SDLoc DL(Op);
12068
12069 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
12070 AMDGPU::lookupBufferIntrinsic(Intr: IntrID);
12071 BufInfo && BufInfo->Kind != AMDGPU::BufferKindAtomicRMW)
12072 return lowerBufferIntrinsicWithChain(Op, Info: *BufInfo, DAG);
12073
12074 switch (IntrID) {
12075 case Intrinsic::amdgcn_cluster_load_b32:
12076 case Intrinsic::amdgcn_cluster_load_b64:
12077 case Intrinsic::amdgcn_cluster_load_b128: {
12078 if (Subtarget->hasGFX1250_STRICT())
12079 initializeM0ToZeroForClusterLoad(Op, DAG, DL);
12080 return SDValue();
12081 }
12082 case Intrinsic::amdgcn_ds_ordered_add:
12083 case Intrinsic::amdgcn_ds_ordered_swap: {
12084 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12085 SDValue Chain = M->getOperand(Num: 0);
12086 SDValue M0 = M->getOperand(Num: 2);
12087 SDValue Value = M->getOperand(Num: 3);
12088 unsigned IndexOperand = M->getConstantOperandVal(Num: 7);
12089 unsigned WaveRelease = M->getConstantOperandVal(Num: 8);
12090 unsigned WaveDone = M->getConstantOperandVal(Num: 9);
12091
12092 unsigned OrderedCountIndex = IndexOperand & 0x3f;
12093 IndexOperand &= ~0x3f;
12094 unsigned CountDw = 0;
12095
12096 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) {
12097 CountDw = (IndexOperand >> 24) & 0xf;
12098 IndexOperand &= ~(0xf << 24);
12099
12100 if (CountDw < 1 || CountDw > 4) {
12101 const Function &Fn = DAG.getMachineFunction().getFunction();
12102 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12103 Fn, "ds_ordered_count: dword count must be between 1 and 4",
12104 DL.getDebugLoc()));
12105 CountDw = 1;
12106 }
12107 }
12108
12109 if (IndexOperand) {
12110 const Function &Fn = DAG.getMachineFunction().getFunction();
12111 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12112 Fn, "ds_ordered_count: bad index operand", DL.getDebugLoc()));
12113 }
12114
12115 if (WaveDone && !WaveRelease) {
12116 // TODO: Move this to IR verifier
12117 const Function &Fn = DAG.getMachineFunction().getFunction();
12118 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12119 Fn, "ds_ordered_count: wave_done requires wave_release",
12120 DL.getDebugLoc()));
12121 }
12122
12123 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
12124 unsigned ShaderType =
12125 SIInstrInfo::getDSShaderTypeValue(MF: DAG.getMachineFunction());
12126 unsigned Offset0 = OrderedCountIndex << 2;
12127 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
12128
12129 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
12130 Offset1 |= (CountDw - 1) << 6;
12131
12132 if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX11)
12133 Offset1 |= ShaderType << 2;
12134
12135 unsigned Offset = Offset0 | (Offset1 << 8);
12136
12137 SDValue Ops[] = {
12138 Chain, Value, DAG.getTargetConstant(Val: Offset, DL, VT: MVT::i16),
12139 copyToM0(DAG, Chain, DL, V: M0).getValue(R: 1), // Glue
12140 };
12141 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::DS_ORDERED_COUNT, dl: DL,
12142 VTList: M->getVTList(), Ops, MemVT: M->getMemoryVT(),
12143 MMO: M->getMemOperand());
12144 }
12145 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
12146 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
12147 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
12148 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
12149 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
12150 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
12151 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
12152 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
12153 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
12154 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
12155 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
12156 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
12157 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
12158 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
12159 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
12160 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
12161 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
12162 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
12163 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
12164 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
12165 case Intrinsic::amdgcn_raw_buffer_atomic_add:
12166 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
12167 case Intrinsic::amdgcn_struct_buffer_atomic_add:
12168 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
12169 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
12170 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
12171 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
12172 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
12173 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
12174 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
12175 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
12176 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
12177 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
12178 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
12179 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
12180 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
12181 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
12182 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
12183 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
12184 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
12185 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
12186 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
12187 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
12188 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
12189 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
12190 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
12191 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
12192 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
12193 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
12194 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
12195 case Intrinsic::amdgcn_raw_buffer_atomic_and:
12196 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
12197 case Intrinsic::amdgcn_struct_buffer_atomic_and:
12198 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
12199 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
12200 case Intrinsic::amdgcn_raw_buffer_atomic_or:
12201 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
12202 case Intrinsic::amdgcn_struct_buffer_atomic_or:
12203 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
12204 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
12205 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
12206 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
12207 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
12208 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
12209 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
12210 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
12211 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
12212 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
12213 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
12214 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
12215 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
12216 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
12217 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
12218 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
12219 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
12220 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
12221 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
12222 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
12223 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
12224 return lowerBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
12225 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
12226 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
12227 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
12228 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
12229 return lowerBufferAtomicIntrin(Op, DAG,
12230 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
12231 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
12232 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
12233 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12234 SDValue NodePtr = M->getOperand(Num: 2);
12235 SDValue RayExtent = M->getOperand(Num: 3);
12236 SDValue InstanceMask = M->getOperand(Num: 4);
12237 SDValue RayOrigin = M->getOperand(Num: 5);
12238 SDValue RayDir = M->getOperand(Num: 6);
12239 SDValue Offsets = M->getOperand(Num: 7);
12240 SDValue TDescr = M->getOperand(Num: 8);
12241
12242 assert(NodePtr.getValueType() == MVT::i64);
12243 assert(RayDir.getValueType() == MVT::v3f32);
12244
12245 bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
12246 const unsigned NumVDataDwords = 10;
12247 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
12248 int Opcode = AMDGPU::getMIMGOpcode(
12249 BaseOpcode: IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
12250 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
12251 MIMGEncoding: AMDGPU::MIMGEncGfx12, VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12252 assert(Opcode != -1);
12253
12254 SmallVector<SDValue, 7> Ops;
12255 Ops.push_back(Elt: NodePtr);
12256 Ops.push_back(Elt: DAG.getBuildVector(
12257 VT: MVT::v2i32, DL,
12258 Ops: {DAG.getBitcast(VT: MVT::i32, V: RayExtent),
12259 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: InstanceMask)}));
12260 Ops.push_back(Elt: RayOrigin);
12261 Ops.push_back(Elt: RayDir);
12262 Ops.push_back(Elt: Offsets);
12263 Ops.push_back(Elt: TDescr);
12264 Ops.push_back(Elt: M->getChain());
12265
12266 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12267 MachineMemOperand *MemRef = M->getMemOperand();
12268 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12269 return SDValue(NewNode, 0);
12270 }
12271 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
12272 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12273 SDValue NodePtr = M->getOperand(Num: 2);
12274 SDValue RayExtent = M->getOperand(Num: 3);
12275 SDValue RayOrigin = M->getOperand(Num: 4);
12276 SDValue RayDir = M->getOperand(Num: 5);
12277 SDValue RayInvDir = M->getOperand(Num: 6);
12278 SDValue TDescr = M->getOperand(Num: 7);
12279
12280 assert(NodePtr.getValueType() == MVT::i32 ||
12281 NodePtr.getValueType() == MVT::i64);
12282 assert(RayDir.getValueType() == MVT::v3f16 ||
12283 RayDir.getValueType() == MVT::v3f32);
12284
12285 const bool IsGFX11 = AMDGPU::isGFX11(STI: *Subtarget);
12286 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
12287 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12288 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16;
12289 const bool Is64 = NodePtr.getValueType() == MVT::i64;
12290 const unsigned NumVDataDwords = 4;
12291 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
12292 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
12293 const bool UseNSA = (Subtarget->hasNSAEncoding() &&
12294 NumVAddrs <= Subtarget->getNSAMaxSize()) ||
12295 IsGFX12Plus;
12296 const unsigned BaseOpcodes[2][2] = {
12297 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
12298 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
12299 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
12300 int Opcode;
12301 if (UseNSA) {
12302 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
12303 MIMGEncoding: IsGFX12Plus ? AMDGPU::MIMGEncGfx12
12304 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
12305 : AMDGPU::MIMGEncGfx10NSA,
12306 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12307 } else {
12308 assert(!IsGFX12Plus);
12309 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
12310 MIMGEncoding: IsGFX11 ? AMDGPU::MIMGEncGfx11Default
12311 : AMDGPU::MIMGEncGfx10Default,
12312 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
12313 }
12314 assert(Opcode != -1);
12315
12316 SmallVector<SDValue, 16> Ops;
12317
12318 auto packLanes = [&DAG, &Ops, &DL](SDValue Op, bool IsAligned) {
12319 SmallVector<SDValue, 3> Lanes;
12320 DAG.ExtractVectorElements(Op, Args&: Lanes, Start: 0, Count: 3);
12321 if (Lanes[0].getValueSizeInBits() == 32) {
12322 for (unsigned I = 0; I < 3; ++I)
12323 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: Lanes[I]));
12324 } else {
12325 if (IsAligned) {
12326 Ops.push_back(Elt: DAG.getBitcast(
12327 VT: MVT::i32,
12328 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[0], Lanes[1]})));
12329 Ops.push_back(Elt: Lanes[2]);
12330 } else {
12331 SDValue Elt0 = Ops.pop_back_val();
12332 Ops.push_back(Elt: DAG.getBitcast(
12333 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Elt0, Lanes[0]})));
12334 Ops.push_back(Elt: DAG.getBitcast(
12335 VT: MVT::i32,
12336 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[1], Lanes[2]})));
12337 }
12338 }
12339 };
12340
12341 if (UseNSA && IsGFX11Plus) {
12342 Ops.push_back(Elt: NodePtr);
12343 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
12344 Ops.push_back(Elt: RayOrigin);
12345 if (IsA16) {
12346 SmallVector<SDValue, 3> DirLanes, InvDirLanes, MergedLanes;
12347 DAG.ExtractVectorElements(Op: RayDir, Args&: DirLanes, Start: 0, Count: 3);
12348 DAG.ExtractVectorElements(Op: RayInvDir, Args&: InvDirLanes, Start: 0, Count: 3);
12349 for (unsigned I = 0; I < 3; ++I) {
12350 MergedLanes.push_back(Elt: DAG.getBitcast(
12351 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL,
12352 Ops: {DirLanes[I], InvDirLanes[I]})));
12353 }
12354 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v3i32, DL, Ops: MergedLanes));
12355 } else {
12356 Ops.push_back(Elt: RayDir);
12357 Ops.push_back(Elt: RayInvDir);
12358 }
12359 } else {
12360 if (Is64)
12361 DAG.ExtractVectorElements(Op: DAG.getBitcast(VT: MVT::v2i32, V: NodePtr), Args&: Ops, Start: 0,
12362 Count: 2);
12363 else
12364 Ops.push_back(Elt: NodePtr);
12365
12366 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
12367 packLanes(RayOrigin, true);
12368 packLanes(RayDir, true);
12369 packLanes(RayInvDir, false);
12370 }
12371
12372 if (!UseNSA) {
12373 // Build a single vector containing all the operands so far prepared.
12374 if (NumVAddrDwords > 12) {
12375 SDValue Undef = DAG.getPOISON(VT: MVT::i32);
12376 Ops.append(NumInputs: 16 - Ops.size(), Elt: Undef);
12377 }
12378 assert(Ops.size() >= 8 && Ops.size() <= 12);
12379 SDValue MergedOps =
12380 DAG.getBuildVector(VT: MVT::getVectorVT(VT: MVT::i32, NumElements: Ops.size()), DL, Ops);
12381 Ops.clear();
12382 Ops.push_back(Elt: MergedOps);
12383 }
12384
12385 Ops.push_back(Elt: TDescr);
12386 Ops.push_back(Elt: DAG.getTargetConstant(Val: IsA16, DL, VT: MVT::i1));
12387 Ops.push_back(Elt: M->getChain());
12388
12389 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12390 MachineMemOperand *MemRef = M->getMemOperand();
12391 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12392 return SDValue(NewNode, 0);
12393 }
12394 case Intrinsic::amdgcn_global_atomic_fmin_num:
12395 case Intrinsic::amdgcn_global_atomic_fmax_num:
12396 case Intrinsic::amdgcn_flat_atomic_fmin_num:
12397 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12398 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12399 SDValue Ops[] = {
12400 M->getOperand(Num: 0), // Chain
12401 M->getOperand(Num: 2), // Ptr
12402 M->getOperand(Num: 3) // Value
12403 };
12404 unsigned Opcode = 0;
12405 switch (IntrID) {
12406 case Intrinsic::amdgcn_global_atomic_fmin_num:
12407 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
12408 Opcode = ISD::ATOMIC_LOAD_FMIN;
12409 break;
12410 }
12411 case Intrinsic::amdgcn_global_atomic_fmax_num:
12412 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12413 Opcode = ISD::ATOMIC_LOAD_FMAX;
12414 break;
12415 }
12416 default:
12417 llvm_unreachable("unhandled atomic opcode");
12418 }
12419 return DAG.getAtomic(Opcode, dl: SDLoc(Op), MemVT: M->getMemoryVT(), VTList: M->getVTList(),
12420 Ops, MMO: M->getMemOperand());
12421 }
12422 case Intrinsic::amdgcn_s_alloc_vgpr: {
12423 SDValue NumVGPRs = Op.getOperand(i: 2);
12424 if (!NumVGPRs->isDivergent())
12425 return Op;
12426
12427 SDValue ReadFirstLaneID =
12428 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
12429 NumVGPRs = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
12430 N1: ReadFirstLaneID, N2: NumVGPRs);
12431
12432 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, VTList: Op->getVTList(),
12433 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: NumVGPRs);
12434 }
12435 case Intrinsic::amdgcn_s_get_barrier_state:
12436 case Intrinsic::amdgcn_s_get_named_barrier_state: {
12437 SDValue Chain = Op->getOperand(Num: 0);
12438 SmallVector<SDValue, 2> Ops;
12439 unsigned Opc;
12440
12441 std::optional<uint32_t> BarID;
12442 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
12443 BarID = getConstantNamedBarrierNumber(BarOp: Op->getOperand(Num: 2), DAG);
12444 else if (auto *C = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2)))
12445 BarID = C->getZExtValue();
12446
12447 if (BarID) {
12448 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
12449 SDValue K = DAG.getTargetConstant(Val: *BarID, DL, VT: MVT::i32);
12450 Ops.push_back(Elt: K);
12451 Ops.push_back(Elt: Chain);
12452 } else {
12453 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
12454 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
12455 SDValue M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Op->getOperand(Num: 2),
12456 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12457 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12458 } else
12459 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: Op->getOperand(Num: 2)).getValue(R: 0));
12460 }
12461
12462 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12463 return SDValue(NewMI, 0);
12464 }
12465 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
12466 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
12467 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
12468 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12469 SDValue Chain = Op->getOperand(Num: 0);
12470 SDValue Ptr = Op->getOperand(Num: 2);
12471 EVT VT = Op->getValueType(ResNo: 0);
12472 return DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl: DL, MemVT: MII->getMemoryVT(), VT,
12473 Chain, Ptr, MMO: MII->getMemOperand());
12474 }
12475 case Intrinsic::amdgcn_av_load_b8:
12476 case Intrinsic::amdgcn_av_load_b16:
12477 case Intrinsic::amdgcn_av_load_b32:
12478 case Intrinsic::amdgcn_av_load_b64:
12479 case Intrinsic::amdgcn_av_load_b128: {
12480 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12481 SDValue Chain = Op->getOperand(Num: 0);
12482 SDValue Ptr = Op->getOperand(Num: 2);
12483 EVT VT = Op->getValueType(ResNo: 0);
12484 // Lower to a regular ISD::LOAD. The MachineMemOperand carries Monotonic
12485 // ordering and syncscope so that SIMemoryLegalizer sets cache policy bits.
12486 // Address space filtering in the load_global/load_flat PatFrags selects
12487 // the correct GLOBAL vs FLAT instruction.
12488 return DAG.getLoad(VT, dl: DL, Chain, Ptr, MMO: MII->getMemOperand());
12489 }
12490 case Intrinsic::amdgcn_flat_load_monitor_b32:
12491 case Intrinsic::amdgcn_flat_load_monitor_b64:
12492 case Intrinsic::amdgcn_flat_load_monitor_b128: {
12493 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12494 SDValue Chain = Op->getOperand(Num: 0);
12495 SDValue Ptr = Op->getOperand(Num: 2);
12496 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::FLAT_LOAD_MONITOR, dl: DL,
12497 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12498 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12499 }
12500 case Intrinsic::amdgcn_global_load_monitor_b32:
12501 case Intrinsic::amdgcn_global_load_monitor_b64:
12502 case Intrinsic::amdgcn_global_load_monitor_b128: {
12503 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12504 SDValue Chain = Op->getOperand(Num: 0);
12505 SDValue Ptr = Op->getOperand(Num: 2);
12506 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::GLOBAL_LOAD_MONITOR, dl: DL,
12507 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12508 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12509 }
12510 default:
12511
12512 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12513 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID))
12514 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
12515
12516 return SDValue();
12517 }
12518}
12519
12520// Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
12521// dwordx4 if on SI and handle TFE loads.
12522SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
12523 SDVTList VTList,
12524 ArrayRef<SDValue> Ops, EVT MemVT,
12525 MachineMemOperand *MMO,
12526 SelectionDAG &DAG) const {
12527 LLVMContext &C = *DAG.getContext();
12528 MachineFunction &MF = DAG.getMachineFunction();
12529 EVT VT = VTList.VTs[0];
12530
12531 assert(VTList.NumVTs == 2 || VTList.NumVTs == 3);
12532 bool IsTFE = VTList.NumVTs == 3;
12533 if (IsTFE) {
12534 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
12535 unsigned NumOpDWords = NumValueDWords + 1;
12536 EVT OpDWordsVT = EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumOpDWords);
12537 SDVTList OpDWordsVTList = DAG.getVTList(VT1: OpDWordsVT, VT2: VTList.VTs[2]);
12538 MachineMemOperand *OpDWordsMMO =
12539 MF.getMachineMemOperand(MMO, Offset: 0, Size: NumOpDWords * 4);
12540 SDValue Op = getMemIntrinsicNode(Opcode, DL, VTList: OpDWordsVTList, Ops,
12541 MemVT: OpDWordsVT, MMO: OpDWordsMMO, DAG);
12542 auto [Value, Status] = splitTFEValueAndStatus(Op, VT, DL, DAG);
12543 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
12544 }
12545
12546 if (!Subtarget->hasDwordx3LoadStores() &&
12547 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
12548 EVT WidenedVT = EVT::getVectorVT(Context&: C, VT: VT.getVectorElementType(), NumElements: 4);
12549 EVT WidenedMemVT = EVT::getVectorVT(Context&: C, VT: MemVT.getVectorElementType(), NumElements: 4);
12550 MachineMemOperand *WidenedMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 16);
12551 SDVTList WidenedVTList = DAG.getVTList(VT1: WidenedVT, VT2: VTList.VTs[1]);
12552 SDValue Op = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: WidenedVTList, Ops,
12553 MemVT: WidenedMemVT, MMO: WidenedMMO);
12554 SDValue Value = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Op,
12555 N2: DAG.getVectorIdxConstant(Val: 0, DL));
12556 return DAG.getMergeValues(Ops: {Value, SDValue(Op.getNode(), 1)}, dl: DL);
12557 }
12558
12559 return DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops, MemVT, MMO);
12560}
12561
12562SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG,
12563 bool ImageStore) const {
12564 EVT StoreVT = VData.getValueType();
12565
12566 // No change for f16 and legal vector D16 types.
12567 if (!StoreVT.isVector())
12568 return VData;
12569
12570 SDLoc DL(VData);
12571 unsigned NumElements = StoreVT.getVectorNumElements();
12572
12573 if (Subtarget->hasUnpackedD16VMem()) {
12574 // We need to unpack the packed data to store.
12575 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12576 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12577
12578 EVT EquivStoreVT =
12579 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements);
12580 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EquivStoreVT, Operand: IntVData);
12581 return DAG.UnrollVectorOp(N: ZExt.getNode());
12582 }
12583
12584 // The sq block of gfx8.1 does not estimate register use correctly for d16
12585 // image store instructions. The data operand is computed as if it were not a
12586 // d16 image instruction.
12587 if (ImageStore && Subtarget->hasImageStoreD16Bug()) {
12588 // Bitcast to i16
12589 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12590 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12591
12592 // Decompose into scalars
12593 SmallVector<SDValue, 4> Elts;
12594 DAG.ExtractVectorElements(Op: IntVData, Args&: Elts);
12595
12596 // Group pairs of i16 into v2i16 and bitcast to i32
12597 SmallVector<SDValue, 4> PackedElts;
12598 for (unsigned I = 0; I < Elts.size() / 2; I += 1) {
12599 SDValue Pair =
12600 DAG.getBuildVector(VT: MVT::v2i16, DL, Ops: {Elts[I * 2], Elts[I * 2 + 1]});
12601 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12602 PackedElts.push_back(Elt: IntPair);
12603 }
12604 if ((NumElements % 2) == 1) {
12605 // Handle v3i16
12606 unsigned I = Elts.size() / 2;
12607 SDValue Pair = DAG.getBuildVector(VT: MVT::v2i16, DL,
12608 Ops: {Elts[I * 2], DAG.getPOISON(VT: MVT::i16)});
12609 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12610 PackedElts.push_back(Elt: IntPair);
12611 }
12612
12613 // Pad using UNDEF
12614 PackedElts.resize(N: Elts.size(), NV: DAG.getPOISON(VT: MVT::i32));
12615
12616 // Build final vector
12617 EVT VecVT =
12618 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: PackedElts.size());
12619 return DAG.getBuildVector(VT: VecVT, DL, Ops: PackedElts);
12620 }
12621
12622 if (NumElements == 3) {
12623 EVT IntStoreVT =
12624 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreVT.getStoreSizeInBits());
12625 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12626
12627 EVT WidenedStoreVT = EVT::getVectorVT(
12628 Context&: *DAG.getContext(), VT: StoreVT.getVectorElementType(), NumElements: NumElements + 1);
12629 EVT WidenedIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12630 BitWidth: WidenedStoreVT.getStoreSizeInBits());
12631 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenedIntVT, Operand: IntVData);
12632 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WidenedStoreVT, Operand: ZExt);
12633 }
12634
12635 assert(isTypeLegal(StoreVT));
12636 return VData;
12637}
12638
12639static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
12640 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
12641 AMDGPU::lookupBufferIntrinsic(Intr))
12642 return BufInfo->IsAsync;
12643 switch (Intr) {
12644 case Intrinsic::amdgcn_load_async_to_lds:
12645 case Intrinsic::amdgcn_global_load_async_lds:
12646 return true;
12647 }
12648 return false;
12649}
12650
12651// Lower the buffer stores, loads to LDS and prefetches, which are all
12652// INTRINSIC_VOID nodes.
12653SDValue SITargetLowering::lowerBufferIntrinsicVoid(
12654 SDValue Op, const AMDGPU::BufferIntrinsicInfo &Info,
12655 SelectionDAG &DAG) const {
12656 SDLoc DL(Op);
12657 SDValue Chain = Op.getOperand(i: 0);
12658 auto *M = cast<MemSDNode>(Val&: Op);
12659 auto GetArg = [&](unsigned Arg) {
12660 return Op.getOperand(i: AMDGPU::getSDOperandIdx(Arg));
12661 };
12662
12663 switch (Info.Kind) {
12664 case AMDGPU::BufferKindStore: {
12665 SDValue VData = GetArg(*Info.getVDataArg());
12666 EVT VDataVT = VData.getValueType();
12667 EVT EltType = VDataVT.getScalarType();
12668
12669 if (Info.isTyped()) {
12670 bool IsD16 = EltType.getSizeInBits() == 16;
12671 if (IsD16)
12672 VData = handleD16VData(VData, DAG);
12673 SmallVector<SDValue, 10> Ops =
12674 getBufferIntrinsicOps(Op, Info, DAG, VData);
12675 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12676 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12677 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12678 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12679 }
12680
12681 bool IsFormat = Info.isFormatFromRsrc();
12682 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12683
12684 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12685 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12686 DAG.getMachineFunction().getFunction(),
12687 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12688 return Chain;
12689 }
12690
12691 if (IsD16) {
12692 VData = handleD16VData(VData, DAG);
12693 VDataVT = VData.getValueType();
12694 }
12695
12696 if (!isTypeLegal(VT: VDataVT)) {
12697 VData =
12698 DAG.getNode(Opcode: ISD::BITCAST, DL,
12699 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12700 }
12701
12702 SmallVector<SDValue, 10> Ops = getBufferIntrinsicOps(Op, Info, DAG, VData);
12703 unsigned Opc =
12704 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
12705 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12706
12707 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12708 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12709 return handleByteShortBufferStores(DAG, VDataType: VDataVT, DL, Ops: Ops.data(), M);
12710
12711 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12712 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12713 }
12714 case AMDGPU::BufferKindLoadToLDS: {
12715 unsigned Opc;
12716 std::optional<unsigned> VIndexArg = Info.getVIndexArg();
12717 bool HasVIndex = VIndexArg.has_value();
12718 SDValue VOffset = GetArg(*Info.getVOffsetArg());
12719 bool HasVOffset = !isNullConstant(V: VOffset);
12720 unsigned Size = GetArg(*Info.getSizeArg())->getAsZExtVal();
12721
12722 switch (Size) {
12723 default:
12724 return SDValue();
12725 case 1:
12726 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
12727 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
12728 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
12729 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
12730 break;
12731 case 2:
12732 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
12733 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
12734 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
12735 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
12736 break;
12737 case 4:
12738 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
12739 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
12740 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
12741 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
12742 break;
12743 case 12:
12744 if (!Subtarget->hasLDSLoadB96_B128())
12745 return SDValue();
12746 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
12747 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
12748 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
12749 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
12750 break;
12751 case 16:
12752 if (!Subtarget->hasLDSLoadB96_B128())
12753 return SDValue();
12754 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
12755 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
12756 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
12757 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
12758 break;
12759 }
12760
12761 SDValue M0Val = copyToM0(DAG, Chain, DL, V: GetArg(*Info.getLDSPtrArg()));
12762
12763 SmallVector<SDValue, 8> Ops;
12764
12765 if (HasVIndex && HasVOffset)
12766 Ops.push_back(
12767 Elt: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {GetArg(*VIndexArg), VOffset}));
12768 else if (HasVIndex)
12769 Ops.push_back(Elt: GetArg(*VIndexArg));
12770 else if (HasVOffset)
12771 Ops.push_back(Elt: VOffset);
12772
12773 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: GetArg(Info.RsrcArg), DAG);
12774 Ops.push_back(Elt: Rsrc);
12775 Ops.push_back(Elt: GetArg(*Info.getSOffsetArg()));
12776 Ops.push_back(Elt: GetArg(*Info.getImmOffsetArg()));
12777 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12778 unsigned Aux = GetArg(*Info.getAuxArg())->getAsZExtVal();
12779 Ops.push_back(Elt: DAG.getTargetConstant(
12780 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12),
12781 DL, VT: MVT::i16)); // cpol
12782 Ops.push_back(Elt: DAG.getTargetConstant(
12783 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
12784 ? 1
12785 : 0,
12786 DL, VT: MVT::i8)); // swz
12787 Ops.push_back(Elt: DAG.getTargetConstant(Val: Info.IsAsync, DL, VT: MVT::i8)); // async
12788 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12789 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12790
12791 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: M->getVTList(), Ops);
12792 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12793
12794 return SDValue(Load, 0);
12795 }
12796 case AMDGPU::BufferKindPrefetch: {
12797 SDValue Ops[] = {
12798 Chain,
12799 bufferRsrcPtrToVector(MaybePointer: GetArg(Info.RsrcArg), DAG),
12800 GetArg(*Info.getSOffsetArg()), // soffset
12801 GetArg(*Info.getSizeArg()), // length
12802 };
12803
12804 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_PREFETCH_DATA, dl: DL,
12805 VTList: Op->getVTList(), Ops, MemVT: M->getMemoryVT(),
12806 MMO: M->getMemOperand());
12807 }
12808 default:
12809 llvm_unreachable("not a buffer intrinsic without a result");
12810 }
12811}
12812
12813SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12814 SelectionDAG &DAG) const {
12815 SDLoc DL(Op);
12816 SDValue Chain = Op.getOperand(i: 0);
12817 unsigned IntrinsicID = Op.getConstantOperandVal(i: 1);
12818
12819 if (const AMDGPU::BufferIntrinsicInfo *BufInfo =
12820 AMDGPU::lookupBufferIntrinsic(Intr: IntrinsicID))
12821 return lowerBufferIntrinsicVoid(Op, Info: *BufInfo, DAG);
12822
12823 switch (IntrinsicID) {
12824 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
12825 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
12826 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
12827 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
12828 if (Subtarget->hasGFX1250_STRICT())
12829 initializeM0ToZeroForClusterLoad(Op, DAG, DL);
12830 return SDValue();
12831 }
12832 case Intrinsic::amdgcn_exp_compr: {
12833 SDValue Src0 = Op.getOperand(i: 4);
12834 SDValue Src1 = Op.getOperand(i: 5);
12835 // Hack around illegal type on SI by directly selecting it.
12836 if (isTypeLegal(VT: Src0.getValueType()))
12837 return SDValue();
12838
12839 const ConstantSDNode *Done = cast<ConstantSDNode>(Val: Op.getOperand(i: 6));
12840 SDValue Undef = DAG.getPOISON(VT: MVT::f32);
12841 const SDValue Ops[] = {
12842 Op.getOperand(i: 2), // tgt
12843 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src0), // src0
12844 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src1), // src1
12845 Undef, // src2
12846 Undef, // src3
12847 Op.getOperand(i: 7), // vm
12848 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // compr
12849 Op.getOperand(i: 3), // en
12850 Op.getOperand(i: 0) // Chain
12851 };
12852
12853 unsigned Opc = Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
12854 return SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops), 0);
12855 }
12856
12857 // Buffers are handled by LowerBufferFatPointers, and we're going to go
12858 // for "trust me" that the remaining cases are global pointers until
12859 // such time as we can put two mem operands on an intrinsic.
12860 case Intrinsic::amdgcn_load_to_lds:
12861 case Intrinsic::amdgcn_load_async_to_lds:
12862 case Intrinsic::amdgcn_global_load_lds:
12863 case Intrinsic::amdgcn_global_load_async_lds: {
12864 if (!Subtarget->hasVMemToLDSLoad())
12865 return SDValue();
12866
12867 unsigned Opc;
12868 unsigned Size = Op->getConstantOperandVal(Num: 4);
12869 switch (Size) {
12870 default:
12871 return SDValue();
12872 case 1:
12873 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
12874 break;
12875 case 2:
12876 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
12877 break;
12878 case 4:
12879 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
12880 break;
12881 case 12:
12882 if (!Subtarget->hasLDSLoadB96_B128())
12883 return SDValue();
12884 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
12885 break;
12886 case 16:
12887 if (!Subtarget->hasLDSLoadB96_B128())
12888 return SDValue();
12889 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
12890 break;
12891 }
12892
12893 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12894
12895 SmallVector<SDValue, 6> Ops;
12896
12897 SDValue Addr = Op.getOperand(i: 2); // Global ptr
12898 SDValue VOffset;
12899 // Try to split SAddr and VOffset. Global and LDS pointers share the same
12900 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
12901 if (Addr->isDivergent() && Addr->isAnyAdd()) {
12902 SDValue LHS = Addr.getOperand(i: 0);
12903 SDValue RHS = Addr.getOperand(i: 1);
12904
12905 if (LHS->isDivergent())
12906 std::swap(a&: LHS, b&: RHS);
12907
12908 if (!LHS->isDivergent() && RHS.getOpcode() == ISD::ZERO_EXTEND &&
12909 RHS.getOperand(i: 0).getValueType() == MVT::i32) {
12910 // add (i64 sgpr), (zero_extend (i32 vgpr))
12911 Addr = LHS;
12912 VOffset = RHS.getOperand(i: 0);
12913 }
12914 }
12915
12916 Ops.push_back(Elt: Addr);
12917 if (!Addr->isDivergent()) {
12918 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
12919 if (!VOffset)
12920 VOffset =
12921 SDValue(DAG.getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32, dl: DL, VT: MVT::i32,
12922 Op1: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)),
12923 0);
12924 Ops.push_back(Elt: VOffset);
12925 }
12926
12927 Ops.push_back(Elt: Op.getOperand(i: 5)); // Offset
12928
12929 unsigned Aux = Op.getConstantOperandVal(i: 6);
12930 Ops.push_back(Elt: DAG.getTargetConstant(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS, DL,
12931 VT: MVT::i32)); // CPol
12932 Ops.push_back(
12933 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
12934
12935 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12936 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12937
12938 auto *M = cast<MemSDNode>(Val&: Op);
12939 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12940 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12941
12942 return SDValue(Load, 0);
12943 }
12944 case Intrinsic::amdgcn_end_cf:
12945 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_END_CF, dl: DL, VT: MVT::Other,
12946 Op1: Op->getOperand(Num: 2), Op2: Chain),
12947 0);
12948 case Intrinsic::amdgcn_s_barrier_signal_var: {
12949 // Member count of 0 means to re-use a previous member count,
12950 // which, if the named barrier is statically chosen, means we can use
12951 // the immarg form. Otherwisee, fall through to constructiong M0 as for
12952 // s_barrier_init.
12953 SDValue CntOp = Op->getOperand(Num: 3);
12954 auto *CntC = dyn_cast<ConstantSDNode>(Val&: CntOp);
12955 if (CntC && CntC->isZero()) {
12956 SDValue Chain = Op->getOperand(Num: 0);
12957 SDValue BarOp = Op->getOperand(Num: 2);
12958
12959 if (auto BarID = getConstantNamedBarrierNumber(BarOp, DAG)) {
12960 std::array<SDValue, 2> Ops = {
12961 DAG.getTargetConstant(Val: *BarID, DL, VT: MVT::i32), Chain};
12962 auto *NewMI = DAG.getMachineNode(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM, dl: DL,
12963 VTs: Op->getVTList(), Ops);
12964 return SDValue(NewMI, 0);
12965 }
12966 }
12967 [[fallthrough]];
12968 }
12969 case Intrinsic::amdgcn_s_barrier_init: {
12970 // these two intrinsics have two operands: barrier pointer and member count
12971 SDValue Chain = Op->getOperand(Num: 0);
12972 SmallVector<SDValue, 2> Ops;
12973 SDValue BarOp = Op->getOperand(Num: 2);
12974 SDValue CntOp = Op->getOperand(Num: 3);
12975 SDValue M0Val;
12976 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
12977 ? AMDGPU::S_BARRIER_INIT_M0
12978 : AMDGPU::S_BARRIER_SIGNAL_M0;
12979 // extract the BarrierID from bits 0-5 of BarOp
12980 SDValue BarID = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: BarOp,
12981 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12982 // Member count should be put into M0[ShAmt:+CntMask width]
12983 // Barrier ID should be put into M0[5:0]
12984 unsigned CntMask = AMDGPU::getBarrierMemberCountBitMask(
12985 Version: AMDGPU::getIsaVersion(GPU: Subtarget->getCPU()));
12986 SDValue MemberCnt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: CntOp,
12987 N2: DAG.getConstant(Val: CntMask, DL, VT: MVT::i32));
12988 constexpr unsigned ShAmt = 16;
12989 M0Val = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: MemberCnt,
12990 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
12991
12992 M0Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M0Val, N2: BarID);
12993
12994 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12995
12996 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12997 return SDValue(NewMI, 0);
12998 }
12999 case Intrinsic::amdgcn_s_wakeup_barrier: {
13000 if (!Subtarget->hasSWakeupBarrier())
13001 return SDValue();
13002 [[fallthrough]];
13003 }
13004 case Intrinsic::amdgcn_s_barrier_join: {
13005 // these three intrinsics have one operand: barrier pointer
13006 SDValue Chain = Op->getOperand(Num: 0);
13007 SmallVector<SDValue, 2> Ops;
13008 SDValue BarOp = Op->getOperand(Num: 2);
13009 unsigned Opc;
13010
13011 if (auto BarID = getConstantNamedBarrierNumber(BarOp, DAG)) {
13012 switch (IntrinsicID) {
13013 default:
13014 return SDValue();
13015 case Intrinsic::amdgcn_s_barrier_join:
13016 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
13017 break;
13018 case Intrinsic::amdgcn_s_wakeup_barrier:
13019 Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
13020 break;
13021 }
13022 SDValue K = DAG.getTargetConstant(Val: *BarID, DL, VT: MVT::i32);
13023 Ops.push_back(Elt: K);
13024 Ops.push_back(Elt: Chain);
13025 } else {
13026 switch (IntrinsicID) {
13027 default:
13028 return SDValue();
13029 case Intrinsic::amdgcn_s_barrier_join:
13030 Opc = AMDGPU::S_BARRIER_JOIN_M0;
13031 break;
13032 case Intrinsic::amdgcn_s_wakeup_barrier:
13033 Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
13034 break;
13035 }
13036 // extract the BarrierID from bits 0-5 of BarOp, copy to M0[5:0]
13037 SDValue M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: BarOp,
13038 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
13039 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
13040 }
13041
13042 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
13043 return SDValue(NewMI, 0);
13044 }
13045 case Intrinsic::amdgcn_s_prefetch_data:
13046 case Intrinsic::amdgcn_s_prefetch_inst: {
13047 // For non-global address space preserve the chain and remove the call.
13048 if (!AMDGPU::isFlatGlobalAddrSpace(AS: cast<MemSDNode>(Val&: Op)->getAddressSpace()))
13049 return Op.getOperand(i: 0);
13050 return Op;
13051 }
13052 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
13053 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
13054 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
13055 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
13056 SDValue Chain = Op->getOperand(Num: 0);
13057 SDValue Ptr = Op->getOperand(Num: 2);
13058 SDValue Val = Op->getOperand(Num: 3);
13059 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MII->getMemoryVT(), Chain, Ptr: Val,
13060 Val: Ptr, MMO: MII->getMemOperand());
13061 }
13062 case Intrinsic::amdgcn_av_store_b8:
13063 case Intrinsic::amdgcn_av_store_b16:
13064 case Intrinsic::amdgcn_av_store_b32:
13065 case Intrinsic::amdgcn_av_store_b64:
13066 case Intrinsic::amdgcn_av_store_b128: {
13067 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
13068 SDValue Chain = Op->getOperand(Num: 0);
13069 SDValue Ptr = Op->getOperand(Num: 2);
13070 SDValue Val = Op->getOperand(Num: 3);
13071 return DAG.getStore(Chain, dl: DL, Val, Ptr, MMO: MII->getMemOperand());
13072 }
13073 default: {
13074 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
13075 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
13076 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
13077
13078 return Op;
13079 }
13080 }
13081}
13082
13083// Return whether the operation has NoUnsignedWrap property.
13084static bool isNoUnsignedWrap(SDValue Addr) {
13085 return (Addr.getOpcode() == ISD::ADD &&
13086 Addr->getFlags().hasNoUnsignedWrap()) ||
13087 Addr->getOpcode() == ISD::OR;
13088}
13089
13090bool SITargetLowering::shouldPreservePtrArith(const Function &F,
13091 EVT PtrVT) const {
13092 return PtrVT == MVT::i64;
13093}
13094
13095bool SITargetLowering::canTransformPtrArithOutOfBounds(const Function &F,
13096 EVT PtrVT) const {
13097 return true;
13098}
13099
13100// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
13101// offset (the offset that is included in bounds checking and swizzling, to be
13102// split between the instruction's voffset and immoffset fields) and soffset
13103// (the offset that is excluded from bounds checking and swizzling, to go in
13104// the instruction's soffset field). This function takes the first kind of
13105// offset and figures out how to split it between voffset and immoffset.
13106std::pair<SDValue, SDValue>
13107SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
13108 SDLoc DL(Offset);
13109 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
13110 SDValue N0 = Offset;
13111 ConstantSDNode *C1 = nullptr;
13112
13113 if ((C1 = dyn_cast<ConstantSDNode>(Val&: N0)))
13114 N0 = SDValue();
13115 else if (DAG.isBaseWithConstantOffset(Op: N0)) {
13116 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13117 // being added, so we can only safely match a 32-bit addition with no
13118 // unsigned overflow.
13119 bool CheckNUW = Subtarget->hasGFX1250Insts();
13120 if (!CheckNUW || isNoUnsignedWrap(Addr: N0)) {
13121 C1 = cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
13122 N0 = N0.getOperand(i: 0);
13123 }
13124 }
13125
13126 if (C1) {
13127 unsigned ImmOffset = C1->getZExtValue();
13128 // If the immediate value is too big for the immoffset field, put only bits
13129 // that would normally fit in the immoffset field. The remaining value that
13130 // is copied/added for the voffset field is a large power of 2, and it
13131 // stands more chance of being CSEd with the copy/add for another similar
13132 // load/store.
13133 // However, do not do that rounding down if that is a negative
13134 // number, as it appears to be illegal to have a negative offset in the
13135 // vgpr, even if adding the immediate offset makes it positive.
13136 unsigned Overflow = ImmOffset & ~MaxImm;
13137 ImmOffset -= Overflow;
13138 if ((int32_t)Overflow < 0) {
13139 Overflow += ImmOffset;
13140 ImmOffset = 0;
13141 }
13142 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32));
13143 if (Overflow) {
13144 auto OverflowVal = DAG.getConstant(Val: Overflow, DL, VT: MVT::i32);
13145 if (!N0)
13146 N0 = OverflowVal;
13147 else {
13148 SDValue Ops[] = {N0, OverflowVal};
13149 N0 = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, Ops);
13150 }
13151 }
13152 }
13153 if (!N0)
13154 N0 = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13155 if (!C1)
13156 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
13157 return {N0, SDValue(C1, 0)};
13158}
13159
13160// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
13161// the three offsets (voffset, soffset and instoffset) into the SDValue[3] array
13162// pointed to by Offsets.
13163void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
13164 SelectionDAG &DAG, SDValue *Offsets,
13165 Align Alignment) const {
13166 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
13167 SDLoc DL(CombinedOffset);
13168 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CombinedOffset)) {
13169 uint32_t Imm = C->getZExtValue();
13170 uint32_t SOffset, ImmOffset;
13171 if (TII->splitMUBUFOffset(Imm, SOffset, ImmOffset, Alignment)) {
13172 Offsets[0] = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13173 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
13174 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
13175 return;
13176 }
13177 }
13178 if (DAG.isBaseWithConstantOffset(Op: CombinedOffset)) {
13179 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
13180 // being added, so we can only safely match a 32-bit addition with no
13181 // unsigned overflow.
13182 bool CheckNUW = Subtarget->hasGFX1250Insts();
13183 SDValue N0 = CombinedOffset.getOperand(i: 0);
13184 SDValue N1 = CombinedOffset.getOperand(i: 1);
13185 uint32_t SOffset, ImmOffset;
13186 int Offset = cast<ConstantSDNode>(Val&: N1)->getSExtValue();
13187 if (Offset >= 0 && (!CheckNUW || isNoUnsignedWrap(Addr: CombinedOffset)) &&
13188 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
13189 Offsets[0] = N0;
13190 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
13191 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
13192 return;
13193 }
13194 }
13195
13196 SDValue SOffsetZero = Subtarget->hasRestrictedSOffset()
13197 ? DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32)
13198 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13199
13200 Offsets[0] = CombinedOffset;
13201 Offsets[1] = SOffsetZero;
13202 Offsets[2] = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
13203}
13204
13205SDValue SITargetLowering::bufferRsrcPtrToVector(SDValue MaybePointer,
13206 SelectionDAG &DAG) const {
13207 if (!MaybePointer.getValueType().isScalarInteger())
13208 return MaybePointer;
13209
13210 SDValue Rsrc = DAG.getBitcast(VT: MVT::v4i32, V: MaybePointer);
13211 return Rsrc;
13212}
13213
13214// Wrap a global or flat pointer into a buffer intrinsic using the flags
13215// specified in the intrinsic.
13216SDValue SITargetLowering::lowerPointerAsRsrcIntrin(SDNode *Op,
13217 SelectionDAG &DAG) const {
13218 SDLoc Loc(Op);
13219
13220 SDValue Pointer = Op->getOperand(Num: 1);
13221 SDValue Stride = Op->getOperand(Num: 2);
13222 SDValue NumRecords = Op->getOperand(Num: 3);
13223 SDValue Flags = Op->getOperand(Num: 4);
13224
13225 SDValue ExtStride = DAG.getAnyExtOrTrunc(Op: Stride, DL: Loc, VT: MVT::i32);
13226 SDValue Rsrc;
13227
13228 if (Subtarget->getBufferResourceNumRecordsWidth() == 45) {
13229 NumRecords = DAG.getZExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i64);
13230 NumRecords = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i64, N1: NumRecords,
13231 N2: DAG.getConstant(Val: (1ULL << 45) - 1, DL: Loc, VT: MVT::i64));
13232 SDValue Zero = DAG.getConstant(Val: 0, DL: Loc, VT: MVT::i32);
13233 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
13234 // num_records.
13235 SDValue ExtPointer = DAG.getAnyExtOrTrunc(Op: Pointer, DL: Loc, VT: MVT::i64);
13236 SDValue NumRecordsLHS =
13237 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i64, N1: NumRecords,
13238 N2: DAG.getShiftAmountConstant(Val: 57, VT: MVT::i32, DL: Loc));
13239 SDValue LowHalf =
13240 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: ExtPointer, N2: NumRecordsLHS);
13241
13242 // Build the higher 64-bit value, which has the higher 38-bit num_records,
13243 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
13244 SDValue NumRecordsRHS =
13245 DAG.getNode(Opcode: ISD::SRL, DL: Loc, VT: MVT::i64, N1: NumRecords,
13246 N2: DAG.getShiftAmountConstant(Val: 7, VT: MVT::i32, DL: Loc));
13247 SDValue ShiftedStride =
13248 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13249 N2: DAG.getShiftAmountConstant(Val: 12, VT: MVT::i32, DL: Loc));
13250 SDValue ExtShiftedStrideVec =
13251 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedStride);
13252 SDValue ExtShiftedStride =
13253 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedStrideVec);
13254 SDValue ShiftedFlags =
13255 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: Flags,
13256 N2: DAG.getShiftAmountConstant(Val: 28, VT: MVT::i32, DL: Loc));
13257 SDValue ExtShiftedFlagsVec =
13258 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedFlags);
13259 SDValue ExtShiftedFlags =
13260 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedFlagsVec);
13261 SDValue CombinedFields =
13262 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: NumRecordsRHS, N2: ExtShiftedStride);
13263 SDValue HighHalf =
13264 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: CombinedFields, N2: ExtShiftedFlags);
13265
13266 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i64, N1: LowHalf, N2: HighHalf);
13267 } else {
13268 NumRecords = DAG.getZExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i32);
13269 auto [LowHalf, HighHalf] =
13270 DAG.SplitScalar(N: Pointer, DL: Loc, LoVT: MVT::i32, HiVT: MVT::i32);
13271 SDValue Mask = DAG.getConstant(Val: 0x0000ffff, DL: Loc, VT: MVT::i32);
13272 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i32, N1: HighHalf, N2: Mask);
13273 SDValue ShiftedStride =
13274 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13275 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: Loc));
13276 SDValue NewHighHalf =
13277 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i32, N1: Masked, N2: ShiftedStride);
13278
13279 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v4i32, N1: LowHalf, N2: NewHighHalf,
13280 N3: NumRecords, N4: Flags);
13281 }
13282
13283 SDValue RsrcPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i128, Operand: Rsrc);
13284 return RsrcPtr;
13285}
13286
13287// Handle 8 bit and 16 bit buffer loads
13288SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
13289 EVT LoadVT, SDLoc DL,
13290 ArrayRef<SDValue> Ops,
13291 MachineMemOperand *MMO,
13292 bool IsTFE) const {
13293 EVT IntVT = LoadVT.changeTypeToInteger();
13294
13295 if (IsTFE) {
13296 unsigned Opc = (LoadVT.getScalarType() == MVT::i8)
13297 ? AMDGPUISD::BUFFER_LOAD_UBYTE_TFE
13298 : AMDGPUISD::BUFFER_LOAD_USHORT_TFE;
13299 MachineFunction &MF = DAG.getMachineFunction();
13300 MachineMemOperand *OpMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 8);
13301 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i32, VT2: MVT::Other);
13302 SDValue Op = getMemIntrinsicNode(Opcode: Opc, DL, VTList: VTs, Ops, MemVT: MVT::v2i32, MMO: OpMMO, DAG);
13303 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13304 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
13305 SDValue Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13306 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
13307 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Data);
13308 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: Trunc);
13309 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
13310 }
13311
13312 unsigned Opc = LoadVT.getScalarType() == MVT::i8
13313 ? AMDGPUISD::BUFFER_LOAD_UBYTE
13314 : AMDGPUISD::BUFFER_LOAD_USHORT;
13315
13316 SDVTList ResList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
13317 SDValue BufferLoad =
13318 DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: IntVT, MMO);
13319 SDValue LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: BufferLoad);
13320 LoadVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: LoadVal);
13321
13322 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
13323}
13324
13325// Handle 8 bit and 16 bit buffer stores
13326SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
13327 EVT VDataType, SDLoc DL,
13328 SDValue Ops[],
13329 MemSDNode *M) const {
13330 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
13331 Ops[1] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Ops[1]);
13332
13333 SDValue BufferStoreExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Ops[1]);
13334 Ops[1] = BufferStoreExt;
13335 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE
13336 : AMDGPUISD::BUFFER_STORE_SHORT;
13337 ArrayRef<SDValue> OpsRef = ArrayRef(&Ops[0], 9);
13338 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: M->getVTList(), Ops: OpsRef, MemVT: VDataType,
13339 MMO: M->getMemOperand());
13340}
13341
13342static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType,
13343 SDValue Op, const SDLoc &SL, EVT VT) {
13344 if (VT.bitsLT(VT: Op.getValueType()))
13345 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
13346
13347 switch (ExtType) {
13348 case ISD::SEXTLOAD:
13349 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SL, VT, Operand: Op);
13350 case ISD::ZEXTLOAD:
13351 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT, Operand: Op);
13352 case ISD::EXTLOAD:
13353 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT, Operand: Op);
13354 case ISD::NON_EXTLOAD:
13355 return Op;
13356 }
13357
13358 llvm_unreachable("invalid ext type");
13359}
13360
13361// Try to turn 8 and 16-bit scalar loads into SMEM eligible 32-bit loads.
13362// TODO: Skip this on GFX12 which does have scalar sub-dword loads.
13363SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
13364 DAGCombinerInfo &DCI) const {
13365 SelectionDAG &DAG = DCI.DAG;
13366 if (Ld->getAlign() < Align(4) || Ld->isDivergent())
13367 return SDValue();
13368
13369 // FIXME: Constant loads should all be marked invariant.
13370 unsigned AS = Ld->getAddressSpace();
13371 if (AS != AMDGPUAS::CONSTANT_ADDRESS &&
13372 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
13373 (AS != AMDGPUAS::GLOBAL_ADDRESS || !Ld->isInvariant()))
13374 return SDValue();
13375
13376 // Don't do this early, since it may interfere with adjacent load merging for
13377 // illegal types. We can avoid losing alignment information for exotic types
13378 // pre-legalize.
13379 EVT MemVT = Ld->getMemoryVT();
13380 if ((MemVT.isSimple() && !DCI.isAfterLegalizeDAG()) ||
13381 MemVT.getSizeInBits() >= 32)
13382 return SDValue();
13383
13384 SDLoc SL(Ld);
13385
13386 assert((!MemVT.isVector() || Ld->getExtensionType() == ISD::NON_EXTLOAD) &&
13387 "unexpected vector extload");
13388
13389 // TODO: Drop only high part of range.
13390 SDValue Ptr = Ld->getBasePtr();
13391 SDValue NewLoad = DAG.getLoad(
13392 AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT: MVT::i32, dl: SL, Chain: Ld->getChain(), Ptr,
13393 Offset: Ld->getOffset(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i32, Alignment: Ld->getAlign(),
13394 MMOFlags: Ld->getMemOperand()->getFlags(), Metadata: Ld->getAAInfo()); // Drop ranges
13395
13396 EVT TruncVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
13397 if (MemVT.isFloatingPoint()) {
13398 assert(Ld->getExtensionType() == ISD::NON_EXTLOAD &&
13399 "unexpected fp extload");
13400 TruncVT = MemVT.changeTypeToInteger();
13401 }
13402
13403 SDValue Cvt = NewLoad;
13404 if (Ld->getExtensionType() == ISD::SEXTLOAD) {
13405 Cvt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SL, VT: MVT::i32, N1: NewLoad,
13406 N2: DAG.getValueType(TruncVT));
13407 } else if (Ld->getExtensionType() == ISD::ZEXTLOAD ||
13408 Ld->getExtensionType() == ISD::NON_EXTLOAD) {
13409 Cvt = DAG.getZeroExtendInReg(Op: NewLoad, DL: SL, VT: TruncVT);
13410 } else {
13411 assert(Ld->getExtensionType() == ISD::EXTLOAD);
13412 }
13413
13414 EVT VT = Ld->getValueType(ResNo: 0);
13415 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
13416
13417 DCI.AddToWorklist(N: Cvt.getNode());
13418
13419 // We may need to handle exotic cases, such as i16->i64 extloads, so insert
13420 // the appropriate extension from the 32-bit load.
13421 Cvt = getLoadExtOrTrunc(DAG, ExtType: Ld->getExtensionType(), Op: Cvt, SL, VT: IntVT);
13422 DCI.AddToWorklist(N: Cvt.getNode());
13423
13424 // Handle conversion back to floating point if necessary.
13425 Cvt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Cvt);
13426
13427 return DAG.getMergeValues(Ops: {Cvt, NewLoad.getValue(R: 1)}, dl: SL);
13428}
13429
13430static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO,
13431 const SIMachineFunctionInfo &Info) {
13432 // TODO: Should check if the address can definitely not access stack.
13433 if (Info.isEntryFunction())
13434 return Info.getUserSGPRInfo().hasFlatScratchInit();
13435 return true;
13436}
13437
13438// Lower a VGPR ("as memory") load or store to a REG_LOAD / REG_STORE node
13439// carrying the dword index. TODO: sub-dword accesses.
13440SDValue SITargetLowering::lowerLoadStoreVGPR(SDValue Op,
13441 SelectionDAG &DAG) const {
13442 SDLoc DL(Op);
13443 MemSDNode *MemOp = cast<MemSDNode>(Val&: Op);
13444 EVT MemVT = MemOp->getMemoryVT();
13445 unsigned BitWidth = MemVT.getSizeInBits();
13446
13447 // Each caller replaces the node, so this is diagnosed once.
13448 auto reportUnsupported = [&]() -> SDValue {
13449 const Function &F = DAG.getMachineFunction().getFunction();
13450 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
13451 F,
13452 "unsupported access of VGPR 'as memory' address space (13); only "
13453 "dword-aligned whole-dword loads and stores are implemented",
13454 DL.getDebugLoc()));
13455 SmallVector<EVT, 2> ResultTypes(Op->values());
13456 return DAG.getErrorMergeValues(ResultTypes, Chain: MemOp->getChain(), dl: DL);
13457 };
13458
13459 if (BitWidth < 32)
13460 return reportUnsupported();
13461
13462 // The index is the pointer >> 2, so an under-aligned access would silently
13463 // reach the containing dword rather than the bytes asked for.
13464 if (MemOp->getAlign() < Align(4))
13465 return reportUnsupported();
13466 auto *Store = dyn_cast<StoreSDNode>(Val: MemOp);
13467 if (Store && Store->isTruncatingStore())
13468 return reportUnsupported();
13469 // Only a width some VGPR tuple has is looked up. A whole-dword access of any
13470 // other width is done as one of <N x i32>, which type legalization splits.
13471 auto *Load = dyn_cast<LoadSDNode>(Val: MemOp);
13472 if (!Subtarget->getRegisterInfo()->getVGPRClassForBitWidth(BitWidth) ||
13473 !AMDGPUMI::VLoadStoreIdxInst::isSupportedBitWidth(Bits: BitWidth)) {
13474 if (BitWidth % 32 != 0)
13475 return reportUnsupported();
13476 EVT ViewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: MemVT);
13477 if (Store) {
13478 if (MemVT == ViewVT)
13479 return SDValue();
13480 return DAG.getStore(Chain: Store->getChain(), dl: DL,
13481 Val: DAG.getBitcast(VT: ViewVT, V: Store->getValue()),
13482 Ptr: Store->getBasePtr(), MMO: Store->getMemOperand());
13483 }
13484 if (MemVT == ViewVT && Load->getExtensionType() == ISD::NON_EXTLOAD)
13485 return SDValue();
13486 // The range is for the elements of MemVT, not for dwords.
13487 Load->getMemOperand()->clearRanges();
13488 SDValue NewLoad = DAG.getLoad(VT: ViewVT, dl: DL, Chain: Load->getChain(),
13489 Ptr: Load->getBasePtr(), MMO: Load->getMemOperand());
13490 SDValue Value = getLoadExtOrTrunc(DAG, ExtType: Load->getExtensionType(),
13491 Op: DAG.getBitcast(VT: MemVT, V: NewLoad), SL: DL,
13492 VT: Load->getValueType(ResNo: 0));
13493 return DAG.getMergeValues(Ops: {Value, NewLoad.getValue(R: 1)}, dl: DL);
13494 }
13495
13496 SDValue Chain = MemOp->getChain();
13497 SDValue Index = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: MemOp->getBasePtr(),
13498 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
13499
13500 // View as i32 / <N x i32> when the memory type is not register legal.
13501 EVT RegVT = MemVT;
13502 if (!isTypeLegal(VT: RegVT)) {
13503 unsigned NumDwords = BitWidth / 32;
13504 RegVT = NumDwords == 1
13505 ? EVT(MVT::i32)
13506 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumDwords);
13507 }
13508
13509 if (Store) {
13510 SDValue Value = DAG.getBitcast(VT: RegVT, V: Store->getValue());
13511 return DAG.getMemIntrinsicNode(
13512 Opcode: AMDGPUISD::REG_STORE, dl: DL, VTList: DAG.getVTList(VT: MVT::Other),
13513 Ops: {Chain, Value, Index}, MemVT, MMO: Store->getMemOperand());
13514 }
13515
13516 SDValue NewLoad = DAG.getMemIntrinsicNode(
13517 Opcode: AMDGPUISD::REG_LOAD, dl: DL, VTList: DAG.getVTList(VT1: RegVT, VT2: MVT::Other), Ops: {Chain, Index},
13518 MemVT, MMO: Load->getMemOperand());
13519 SDValue Value = DAG.getBitcast(VT: MemVT, V: NewLoad);
13520 // The combiner folds an extension of a whole-dword load into the load.
13521 Value = getLoadExtOrTrunc(DAG, ExtType: Load->getExtensionType(), Op: Value, SL: DL,
13522 VT: Load->getValueType(ResNo: 0));
13523 if (Value == NewLoad)
13524 return NewLoad;
13525 return DAG.getMergeValues(Ops: {Value, NewLoad.getValue(R: 1)}, dl: DL);
13526}
13527
13528SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
13529 SDLoc DL(Op);
13530 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
13531 assert(Load->getExtensionType() == ISD::NON_EXTLOAD &&
13532 "Unexpected custom extending load!");
13533 EVT MemVT = Load->getMemoryVT();
13534 MachineMemOperand *MMO = Load->getMemOperand();
13535
13536 if (MemVT.getSizeInBits() < 32) {
13537 // Legalize uniform 16-bit loads to i16 = trunc (zextload i16->i32)
13538 // to match subword load patterns.
13539 // Only do this for loads that can use scalar subword load instructions.
13540 if (!MemVT.isVector() && MemVT.getSizeInBits() == 16 &&
13541 isTypeLegal(VT: MemVT) && Subtarget->hasScalarSubwordLoads() &&
13542 isUniformLoad(Load)) {
13543 SDValue Chain = Load->getChain();
13544 SDValue BasePtr = Load->getBasePtr();
13545
13546 // Load as i16 and zero-extend to i32 (matches S_LOAD_U16 behavior)
13547 SDValue NewLD = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: DL, VT: MVT::i32, Chain,
13548 Ptr: BasePtr, MemVT: MVT::i16, MMO);
13549
13550 // Truncate back to i16
13551 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: NewLD);
13552
13553 // For f16/bf16, bitcast from i16 to the original fp type
13554 SDValue Result = (MemVT == MVT::i16)
13555 ? Trunc
13556 : DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MemVT, Operand: Trunc);
13557
13558 SDValue Ops[] = {Result, NewLD.getValue(R: 1)};
13559 return DAG.getMergeValues(Ops, dl: DL);
13560 }
13561
13562 if (MemVT == MVT::i16 && isTypeLegal(VT: MVT::i16))
13563 return SDValue();
13564
13565 // FIXME: Copied from PPC
13566 // First, load into 32 bits, then truncate to 1 bit.
13567
13568 SDValue Chain = Load->getChain();
13569 SDValue BasePtr = Load->getBasePtr();
13570
13571 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
13572
13573 SDValue NewLD = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i32, Chain, Ptr: BasePtr,
13574 MemVT: RealMemVT, MMO);
13575
13576 if (!MemVT.isVector()) {
13577 SDValue Ops[] = {DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: NewLD),
13578 NewLD.getValue(R: 1)};
13579
13580 return DAG.getMergeValues(Ops, dl: DL);
13581 }
13582
13583 SmallVector<SDValue, 3> Elts;
13584 for (unsigned I = 0, N = MemVT.getVectorNumElements(); I != N; ++I) {
13585 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: NewLD,
13586 N2: DAG.getConstant(Val: I, DL, VT: MVT::i32));
13587
13588 Elts.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Elt));
13589 }
13590
13591 SDValue Ops[] = {DAG.getBuildVector(VT: MemVT, DL, Ops: Elts), NewLD.getValue(R: 1)};
13592
13593 return DAG.getMergeValues(Ops, dl: DL);
13594 }
13595
13596 assert(MemVT.isVector() &&
13597 Op.getValueType().getVectorElementType() == MVT::i32 &&
13598 "Custom lowering for non-i32 vectors hasn't been implemented.");
13599
13600 Align Alignment = Load->getAlign();
13601 unsigned AS = Load->getAddressSpace();
13602 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13603 AS == AMDGPUAS::FLAT_ADDRESS &&
13604 Alignment.value() < MemVT.getStoreSize() && MemVT.getSizeInBits() > 32) {
13605 return SplitVectorLoad(Op, DAG);
13606 }
13607
13608 MachineFunction &MF = DAG.getMachineFunction();
13609 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
13610 // If there is a possibility that flat instruction access scratch memory
13611 // then we need to use the same legalization rules we use for private.
13612 if (AS == AMDGPUAS::FLAT_ADDRESS &&
13613 !Subtarget->hasMultiDwordFlatScratchAddressing())
13614 AS = addressMayBeAccessedAsPrivate(MMO: Load->getMemOperand(), Info: *MFI)
13615 ? AMDGPUAS::PRIVATE_ADDRESS
13616 : AMDGPUAS::GLOBAL_ADDRESS;
13617
13618 unsigned NumElements = MemVT.getVectorNumElements();
13619
13620 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13621 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13622 (AS == AMDGPUAS::GLOBAL_ADDRESS && Load->isSimple() &&
13623 (Load->isInvariant() || isMemOpHasNoClobberedMemOperand(N: Load)))) {
13624 if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) &&
13625 Alignment >= Align(4) && NumElements < 32) {
13626 if (MemVT.isPow2VectorType() ||
13627 (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
13628 return SDValue();
13629 return WidenOrSplitVectorLoad(Op, DAG);
13630 }
13631 // Non-uniform loads will be selected to MUBUF instructions, so they
13632 // have the same legalization requirements as global and private
13633 // loads.
13634 //
13635 }
13636 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13637 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13638 AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
13639 if (NumElements > 4)
13640 return SplitVectorLoad(Op, DAG);
13641 // v3 loads not supported on SI.
13642 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13643 return WidenOrSplitVectorLoad(Op, DAG);
13644
13645 // v3 and v4 loads are supported for private and global memory.
13646 return SDValue();
13647 }
13648 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
13649 // Depending on the setting of the private_element_size field in the
13650 // resource descriptor, we can only make private accesses up to a certain
13651 // size.
13652 switch (Subtarget->getMaxPrivateElementSize()) {
13653 case 4: {
13654 auto [Op0, Op1] = scalarizeVectorLoad(LD: Load, DAG);
13655 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13656 }
13657 case 8:
13658 if (NumElements > 2)
13659 return SplitVectorLoad(Op, DAG);
13660 return SDValue();
13661 case 16:
13662 // Same as global/flat
13663 if (NumElements > 4)
13664 return SplitVectorLoad(Op, DAG);
13665 // v3 loads not supported on SI.
13666 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13667 return WidenOrSplitVectorLoad(Op, DAG);
13668
13669 return SDValue();
13670 default:
13671 llvm_unreachable("unsupported private_element_size");
13672 }
13673 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
13674 unsigned Fast = 0;
13675 auto Flags = Load->getMemOperand()->getFlags();
13676 if (allowsMisalignedMemoryAccessesImpl(Size: MemVT.getSizeInBits(), AddrSpace: AS,
13677 Alignment: Load->getAlign(), Flags, IsFast: &Fast) &&
13678 Fast > 1)
13679 return SDValue();
13680
13681 return SplitVectorLoad(Op, DAG);
13682 }
13683
13684 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
13685 VT: MemVT, MMO: *Load->getMemOperand())) {
13686 auto [Op0, Op1] = expandUnalignedLoad(LD: Load, DAG);
13687 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13688 }
13689
13690 return SDValue();
13691}
13692
13693SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
13694 EVT VT = Op.getValueType();
13695 if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256 ||
13696 VT.getSizeInBits() == 512)
13697 return splitTernaryVectorOp(Op, DAG);
13698
13699 assert(VT.getSizeInBits() == 64);
13700
13701 SDLoc DL(Op);
13702 SDValue Cond = DAG.getFreeze(V: Op.getOperand(i: 0));
13703
13704 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13705 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
13706
13707 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
13708 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 2));
13709
13710 SDValue Lo0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: Zero);
13711 SDValue Lo1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: Zero);
13712
13713 SDValue Lo = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Lo0, RHS: Lo1);
13714
13715 SDValue Hi0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: One);
13716 SDValue Hi1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: One);
13717
13718 SDValue Hi = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Hi0, RHS: Hi1);
13719
13720 SDValue Res = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Lo, Hi});
13721 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
13722}
13723
13724// Catch division cases where we can use shortcuts with rcp and rsq
13725// instructions.
13726SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
13727 SelectionDAG &DAG) const {
13728 SDLoc SL(Op);
13729 SDValue LHS = Op.getOperand(i: 0);
13730 SDValue RHS = Op.getOperand(i: 1);
13731 EVT VT = Op.getValueType();
13732 const SDNodeFlags Flags = Op->getFlags();
13733
13734 bool AllowInaccurateRcp = Flags.hasApproximateFuncs();
13735
13736 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
13737 // Without !fpmath accuracy information, we can't do more because we don't
13738 // know exactly whether rcp is accurate enough to meet !fpmath requirement.
13739 // f16 is always accurate enough
13740 if (!AllowInaccurateRcp && VT != MVT::f16 && VT != MVT::bf16)
13741 return SDValue();
13742
13743 if (CLHS->isOne()) {
13744 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
13745 // the CI documentation has a worst case error of 1 ulp.
13746 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
13747 // use it as long as we aren't trying to use denormals.
13748 //
13749 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
13750
13751 // 1.0 / sqrt(x) -> rsq(x)
13752
13753 // XXX - Is afn sufficient to do this for f64? The maximum ULP
13754 // error seems really high at 2^29 ULP.
13755 // 1.0 / x -> rcp(x)
13756 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13757 }
13758
13759 // Same as for 1.0, but expand the sign out of the constant.
13760 if (CLHS->isMinusOne()) {
13761 // -1.0 / x -> fneg (rcp x)
13762 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13763 return DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Rcp);
13764 }
13765 }
13766
13767 // For f16 and bf16 require afn or arcp.
13768 // For f32 require afn.
13769 if (!AllowInaccurateRcp &&
13770 ((VT != MVT::f16 && VT != MVT::bf16) || !Flags.hasAllowReciprocal()))
13771 return SDValue();
13772
13773 // Turn into multiply by the reciprocal.
13774 // x / y -> x * (1.0 / y)
13775 SDValue Recip = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13776 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LHS, N2: Recip, Flags);
13777}
13778
13779SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op,
13780 SelectionDAG &DAG) const {
13781 SDLoc SL(Op);
13782 SDValue X = Op.getOperand(i: 0);
13783 SDValue Y = Op.getOperand(i: 1);
13784 EVT VT = Op.getValueType();
13785 const SDNodeFlags Flags = Op->getFlags();
13786
13787 bool AllowInaccurateDiv = Flags.hasApproximateFuncs();
13788 if (!AllowInaccurateDiv)
13789 return SDValue();
13790
13791 const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: X);
13792 bool IsNegRcp = CLHS && CLHS->isMinusOne();
13793
13794 // Pull out the negation so it folds for free into the source modifiers.
13795 if (IsNegRcp)
13796 X = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13797
13798 SDValue NegY = IsNegRcp ? Y : DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y);
13799 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13800
13801 SDValue R = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: Y);
13802 if (IsNegRcp)
13803 R = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: R);
13804
13805 SDValue Tmp0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13806
13807 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp0, N2: R, N3: R);
13808 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13809 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp1, N2: R, N3: R);
13810
13811 // Skip the last 2 correction terms for reciprocal.
13812 if (IsNegRcp || (CLHS && CLHS->isOne()))
13813 return R;
13814
13815 SDValue Ret = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: R);
13816 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: Ret, N3: X);
13817 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp2, N2: R, N3: Ret);
13818}
13819
13820static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13821 EVT VT, SDValue A, SDValue B, SDValue GlueChain,
13822 SDNodeFlags Flags) {
13823 if (GlueChain->getNumValues() <= 1) {
13824 return DAG.getNode(Opcode, DL: SL, VT, N1: A, N2: B, Flags);
13825 }
13826
13827 assert(GlueChain->getNumValues() == 3);
13828
13829 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13830 switch (Opcode) {
13831 default:
13832 llvm_unreachable("no chain equivalent for opcode");
13833 case ISD::FMUL:
13834 Opcode = AMDGPUISD::FMUL_W_CHAIN;
13835 break;
13836 }
13837
13838 return DAG.getNode(Opcode, DL: SL, VTList,
13839 Ops: {GlueChain.getValue(R: 1), A, B, GlueChain.getValue(R: 2)},
13840 Flags);
13841}
13842
13843static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13844 EVT VT, SDValue A, SDValue B, SDValue C,
13845 SDValue GlueChain, SDNodeFlags Flags) {
13846 if (GlueChain->getNumValues() <= 1) {
13847 return DAG.getNode(Opcode, DL: SL, VT, Ops: {A, B, C}, Flags);
13848 }
13849
13850 assert(GlueChain->getNumValues() == 3);
13851
13852 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13853 switch (Opcode) {
13854 default:
13855 llvm_unreachable("no chain equivalent for opcode");
13856 case ISD::FMA:
13857 Opcode = AMDGPUISD::FMA_W_CHAIN;
13858 break;
13859 }
13860
13861 return DAG.getNode(Opcode, DL: SL, VTList,
13862 Ops: {GlueChain.getValue(R: 1), A, B, C, GlueChain.getValue(R: 2)},
13863 Flags);
13864}
13865
13866SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
13867 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13868 return FastLowered;
13869
13870 SDLoc SL(Op);
13871 EVT VT = Op.getValueType();
13872 SDValue LHS = Op.getOperand(i: 0);
13873 SDValue RHS = Op.getOperand(i: 1);
13874
13875 SDValue LHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: LHS);
13876 SDValue RHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: RHS);
13877
13878 if (VT == MVT::bf16) {
13879 SDValue ExtDiv =
13880 DAG.getNode(Opcode: ISD::FDIV, DL: SL, VT: MVT::f32, N1: LHSExt, N2: RHSExt, Flags: Op->getFlags());
13881 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::bf16, N1: ExtDiv,
13882 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13883 }
13884
13885 assert(VT == MVT::f16);
13886
13887 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
13888 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
13889 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
13890 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
13891 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13892 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
13893 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13894 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
13895 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
13896 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
13897 // q16.u = opx(V_CVT_F16_F32, q32.u);
13898 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
13899
13900 // We will use ISD::FMA on targets that don't support ISD::FMAD.
13901 unsigned FMADOpCode =
13902 isOperationLegal(Op: ISD::FMAD, VT: MVT::f32) ? ISD::FMAD : ISD::FMA;
13903 SDValue NegRHSExt = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHSExt);
13904 SDValue Rcp =
13905 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: RHSExt, Flags: Op->getFlags());
13906 SDValue Quot =
13907 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHSExt, N2: Rcp, Flags: Op->getFlags());
13908 SDValue Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13909 Flags: Op->getFlags());
13910 Quot = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, N3: Quot, Flags: Op->getFlags());
13911 Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13912 Flags: Op->getFlags());
13913 SDValue Tmp = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, Flags: Op->getFlags());
13914 SDValue TmpCast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Tmp);
13915 TmpCast = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TmpCast,
13916 N2: DAG.getConstant(Val: 0xff800000, DL: SL, VT: MVT::i32));
13917 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: TmpCast);
13918 Quot = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f32, N1: Tmp, N2: Quot, Flags: Op->getFlags());
13919 SDValue RDst = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Quot,
13920 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13921 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f16, N1: RDst, N2: RHS, N3: LHS,
13922 Flags: Op->getFlags());
13923}
13924
13925// Faster 2.5 ULP division that does not support denormals.
13926SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
13927 SDNodeFlags Flags = Op->getFlags();
13928 SDLoc SL(Op);
13929 SDValue LHS = Op.getOperand(i: 1);
13930 SDValue RHS = Op.getOperand(i: 2);
13931
13932 // TODO: The combiner should probably handle elimination of redundant fabs.
13933 SDValue r1 = DAG.SignBitIsZeroFP(Op: RHS)
13934 ? RHS
13935 : DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f32, Operand: RHS, Flags);
13936
13937 const APFloat K0Val(0x1p+96f);
13938 const SDValue K0 = DAG.getConstantFP(Val: K0Val, DL: SL, VT: MVT::f32);
13939
13940 const APFloat K1Val(0x1p-32f);
13941 const SDValue K1 = DAG.getConstantFP(Val: K1Val, DL: SL, VT: MVT::f32);
13942
13943 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
13944
13945 EVT SetCCVT =
13946 getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT: MVT::f32);
13947
13948 SDValue r2 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: r1, RHS: K0, Cond: ISD::SETOGT);
13949
13950 SDValue r3 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: r2, N2: K1, N3: One, Flags);
13951
13952 r1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: RHS, N2: r3, Flags);
13953
13954 // rcp does not support denormals.
13955 SDValue r0 = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: r1, Flags);
13956
13957 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHS, N2: r0, Flags);
13958
13959 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: r3, N2: Mul, Flags);
13960}
13961
13962// Returns immediate value for setting the F32 denorm mode when using the
13963// S_DENORM_MODE instruction.
13964static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG,
13965 const SIMachineFunctionInfo *Info,
13966 const GCNSubtarget *ST) {
13967 assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE");
13968 uint32_t DPDenormModeDefault = Info->getMode().fpDenormModeDPValue();
13969 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
13970 return DAG.getTargetConstant(Val: Mode, DL: SDLoc(), VT: MVT::i32);
13971}
13972
13973SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
13974 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13975 return FastLowered;
13976
13977 // The selection matcher assumes anything with a chain selecting to a
13978 // mayRaiseFPException machine instruction. Since we're introducing a chain
13979 // here, we need to explicitly report nofpexcept for the regular fdiv
13980 // lowering.
13981 SDNodeFlags Flags = Op->getFlags();
13982 Flags.setNoFPExcept(true);
13983
13984 SDLoc SL(Op);
13985 SDValue LHS = Op.getOperand(i: 0);
13986 SDValue RHS = Op.getOperand(i: 1);
13987
13988 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
13989
13990 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f32, VT2: MVT::i1);
13991
13992 SDValue DenominatorScaled =
13993 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {RHS, RHS, LHS}, Flags);
13994 SDValue NumeratorScaled =
13995 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {LHS, RHS, LHS}, Flags);
13996
13997 // Denominator is scaled to not be denormal, so using rcp is ok.
13998 SDValue ApproxRcp =
13999 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
14000 SDValue NegDivScale0 =
14001 DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
14002
14003 using namespace AMDGPU::Hwreg;
14004 const unsigned Denorm32Reg = HwregEncoding::encode(Values: ID_MODE, Values: 4, Values: 2);
14005 const SDValue BitField = DAG.getTargetConstant(Val: Denorm32Reg, DL: SL, VT: MVT::i32);
14006
14007 const MachineFunction &MF = DAG.getMachineFunction();
14008 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
14009 const DenormalMode DenormMode = Info->getMode().FP32Denormals;
14010
14011 const bool PreservesDenormals = DenormMode == DenormalMode::getIEEE();
14012 const bool HasDynamicDenormals =
14013 (DenormMode.Input == DenormalMode::Dynamic) ||
14014 (DenormMode.Output == DenormalMode::Dynamic);
14015
14016 SDValue SavedDenormMode;
14017
14018 if (!PreservesDenormals) {
14019 // Note we can't use the STRICT_FMA/STRICT_FMUL for the non-strict FDIV
14020 // lowering. The chain dependence is insufficient, and we need glue. We do
14021 // not need the glue variants in a strictfp function.
14022
14023 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
14024
14025 SDValue Glue = DAG.getEntryNode();
14026 if (HasDynamicDenormals) {
14027 SDNode *GetReg = DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32, dl: SL,
14028 VTs: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Glue),
14029 Ops: {BitField, Glue});
14030 SavedDenormMode = SDValue(GetReg, 0);
14031
14032 Glue = DAG.getMergeValues(
14033 Ops: {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, dl: SL);
14034 }
14035
14036 SDNode *EnableDenorm;
14037 if (Subtarget->hasDenormModeInst()) {
14038 const SDValue EnableDenormValue =
14039 getSPDenormModeValue(FP_DENORM_FLUSH_NONE, DAG, Info, ST: Subtarget);
14040
14041 EnableDenorm = DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs, N1: Glue,
14042 N2: EnableDenormValue)
14043 .getNode();
14044 } else {
14045 const SDValue EnableDenormValue =
14046 DAG.getConstant(FP_DENORM_FLUSH_NONE, DL: SL, VT: MVT::i32);
14047 EnableDenorm = DAG.getMachineNode(Opcode: AMDGPU::S_SETREG_B32, dl: SL, VTs: BindParamVTs,
14048 Ops: {EnableDenormValue, BitField, Glue});
14049 }
14050
14051 SDValue Ops[3] = {NegDivScale0, SDValue(EnableDenorm, 0),
14052 SDValue(EnableDenorm, 1)};
14053
14054 NegDivScale0 = DAG.getMergeValues(Ops, dl: SL);
14055 }
14056
14057 SDValue Fma0 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0,
14058 B: ApproxRcp, C: One, GlueChain: NegDivScale0, Flags);
14059
14060 SDValue Fma1 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma0, B: ApproxRcp,
14061 C: ApproxRcp, GlueChain: Fma0, Flags);
14062
14063 SDValue Mul = getFPBinOp(DAG, Opcode: ISD::FMUL, SL, VT: MVT::f32, A: NumeratorScaled, B: Fma1,
14064 GlueChain: Fma1, Flags);
14065
14066 SDValue Fma2 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Mul,
14067 C: NumeratorScaled, GlueChain: Mul, Flags);
14068
14069 SDValue Fma3 =
14070 getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma2, B: Fma1, C: Mul, GlueChain: Fma2, Flags);
14071
14072 SDValue Fma4 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Fma3,
14073 C: NumeratorScaled, GlueChain: Fma3, Flags);
14074
14075 if (!PreservesDenormals) {
14076 SDNode *DisableDenorm;
14077 if (!HasDynamicDenormals && Subtarget->hasDenormModeInst()) {
14078 const SDValue DisableDenormValue = getSPDenormModeValue(
14079 FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, Info, ST: Subtarget);
14080
14081 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
14082 DisableDenorm =
14083 DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs,
14084 N1: Fma4.getValue(R: 1), N2: DisableDenormValue, N3: Fma4.getValue(R: 2))
14085 .getNode();
14086 } else {
14087 assert(HasDynamicDenormals == (bool)SavedDenormMode);
14088 const SDValue DisableDenormValue =
14089 HasDynamicDenormals
14090 ? SavedDenormMode
14091 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, DL: SL, VT: MVT::i32);
14092
14093 DisableDenorm = DAG.getMachineNode(
14094 Opcode: AMDGPU::S_SETREG_B32, dl: SL, VT: MVT::Other,
14095 Ops: {DisableDenormValue, BitField, Fma4.getValue(R: 1), Fma4.getValue(R: 2)});
14096 }
14097
14098 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
14099 N1: SDValue(DisableDenorm, 0), N2: DAG.getRoot());
14100 DAG.setRoot(OutputChain);
14101 }
14102
14103 SDValue Scale = NumeratorScaled.getValue(R: 1);
14104 SDValue Fmas = DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f32,
14105 Ops: {Fma4, Fma1, Fma3, Scale}, Flags);
14106
14107 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f32, N1: Fmas, N2: RHS, N3: LHS, Flags);
14108}
14109
14110SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
14111 if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG))
14112 return FastLowered;
14113
14114 SDLoc SL(Op);
14115 SDValue X = Op.getOperand(i: 0);
14116 SDValue Y = Op.getOperand(i: 1);
14117
14118 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
14119
14120 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f64, VT2: MVT::i1);
14121
14122 SDValue DivScale0 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: Y, N2: Y, N3: X);
14123
14124 SDValue NegDivScale0 = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f64, Operand: DivScale0);
14125
14126 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f64, Operand: DivScale0);
14127
14128 SDValue Fma0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Rcp, N3: One);
14129
14130 SDValue Fma1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Rcp, N2: Fma0, N3: Rcp);
14131
14132 SDValue Fma2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Fma1, N3: One);
14133
14134 SDValue DivScale1 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: X, N2: Y, N3: X);
14135
14136 SDValue Fma3 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Fma1, N2: Fma2, N3: Fma1);
14137 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f64, N1: DivScale1, N2: Fma3);
14138
14139 SDValue Fma4 =
14140 DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Mul, N3: DivScale1);
14141
14142 SDValue Scale;
14143
14144 if (!Subtarget->hasUsableDivScaleConditionOutput()) {
14145 // Workaround a hardware bug on SI where the condition output from div_scale
14146 // is not usable.
14147
14148 const SDValue Hi = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
14149
14150 // Figure out if the scale to use for div_fmas.
14151 SDValue NumBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: X);
14152 SDValue DenBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Y);
14153 SDValue Scale0BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale0);
14154 SDValue Scale1BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale1);
14155
14156 SDValue NumHi =
14157 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: NumBC, N2: Hi);
14158 SDValue DenHi =
14159 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: DenBC, N2: Hi);
14160
14161 SDValue Scale0Hi =
14162 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale0BC, N2: Hi);
14163 SDValue Scale1Hi =
14164 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale1BC, N2: Hi);
14165
14166 SDValue CmpDen = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: DenHi, RHS: Scale0Hi, Cond: ISD::SETEQ);
14167 SDValue CmpNum = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: NumHi, RHS: Scale1Hi, Cond: ISD::SETEQ);
14168 Scale = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: CmpNum, N2: CmpDen);
14169 } else {
14170 Scale = DivScale1.getValue(R: 1);
14171 }
14172
14173 SDValue Fmas =
14174 DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f64, N1: Fma4, N2: Fma3, N3: Mul, N4: Scale);
14175
14176 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f64, N1: Fmas, N2: Y, N3: X);
14177}
14178
14179SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
14180 EVT VT = Op.getValueType();
14181
14182 if (VT == MVT::f32)
14183 return LowerFDIV32(Op, DAG);
14184
14185 if (VT == MVT::f64)
14186 return LowerFDIV64(Op, DAG);
14187
14188 if (VT == MVT::f16 || VT == MVT::bf16)
14189 return LowerFDIV16(Op, DAG);
14190
14191 llvm_unreachable("Unexpected type for fdiv");
14192}
14193
14194SDValue SITargetLowering::LowerFFREXP(SDValue Op, SelectionDAG &DAG) const {
14195 SDLoc dl(Op);
14196 SDValue Val = Op.getOperand(i: 0);
14197 EVT VT = Val.getValueType();
14198 EVT ResultExpVT = Op->getValueType(ResNo: 1);
14199 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
14200
14201 SDValue Mant = DAG.getNode(
14202 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT,
14203 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_mant, DL: dl, VT: MVT::i32), N2: Val);
14204
14205 SDValue Exp = DAG.getNode(
14206 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: InstrExpVT,
14207 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_exp, DL: dl, VT: MVT::i32), N2: Val);
14208
14209 if (Subtarget->hasFractBug()) {
14210 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val);
14211 SDValue Inf =
14212 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
14213
14214 SDValue IsFinite = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Fabs, RHS: Inf, Cond: ISD::SETOLT);
14215 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: InstrExpVT);
14216 Exp = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InstrExpVT, N1: IsFinite, N2: Exp, N3: Zero);
14217 Mant = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: IsFinite, N2: Mant, N3: Val);
14218 }
14219
14220 SDValue CastExp = DAG.getSExtOrTrunc(Op: Exp, DL: dl, VT: ResultExpVT);
14221 return DAG.getMergeValues(Ops: {Mant, CastExp}, dl);
14222}
14223
14224SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
14225 SDLoc DL(Op);
14226 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
14227 assert(!Store->isTruncatingStore() && "Unexpected custom truncating store!");
14228 EVT VT = Store->getMemoryVT();
14229
14230 if (VT == MVT::i1) {
14231 return DAG.getTruncStore(
14232 Chain: Store->getChain(), dl: DL,
14233 Val: DAG.getSExtOrTrunc(Op: Store->getValue(), DL, VT: MVT::i32),
14234 Ptr: Store->getBasePtr(), SVT: MVT::i1, MMO: Store->getMemOperand());
14235 }
14236
14237 assert(VT.isVector() &&
14238 Store->getValue().getValueType().getScalarType() == MVT::i32);
14239
14240 unsigned AS = Store->getAddressSpace();
14241 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
14242 AS == AMDGPUAS::FLAT_ADDRESS &&
14243 Store->getAlign().value() < VT.getStoreSize() &&
14244 VT.getSizeInBits() > 32) {
14245 return SplitVectorStore(Op, DAG);
14246 }
14247
14248 MachineFunction &MF = DAG.getMachineFunction();
14249 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
14250 // If there is a possibility that flat instruction access scratch memory
14251 // then we need to use the same legalization rules we use for private.
14252 if (AS == AMDGPUAS::FLAT_ADDRESS &&
14253 !Subtarget->hasMultiDwordFlatScratchAddressing())
14254 AS = addressMayBeAccessedAsPrivate(MMO: Store->getMemOperand(), Info: *MFI)
14255 ? AMDGPUAS::PRIVATE_ADDRESS
14256 : AMDGPUAS::GLOBAL_ADDRESS;
14257
14258 unsigned NumElements = VT.getVectorNumElements();
14259 if (AS == AMDGPUAS::FLAT_ADDRESS || AMDGPU::isExtendedGlobalAddrSpace(AS)) {
14260 if (NumElements > 4)
14261 return SplitVectorStore(Op, DAG);
14262 // v3 stores not supported on SI.
14263 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
14264 return SplitVectorStore(Op, DAG);
14265
14266 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
14267 VT, MMO: *Store->getMemOperand()))
14268 return expandUnalignedStore(ST: Store, DAG);
14269
14270 return SDValue();
14271 }
14272 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
14273 switch (Subtarget->getMaxPrivateElementSize()) {
14274 case 4:
14275 return scalarizeVectorStore(ST: Store, DAG);
14276 case 8:
14277 if (NumElements > 2)
14278 return SplitVectorStore(Op, DAG);
14279 return SDValue();
14280 case 16:
14281 if (NumElements > 4 ||
14282 (NumElements == 3 && !Subtarget->hasFlatScratchEnabled()))
14283 return SplitVectorStore(Op, DAG);
14284 return SDValue();
14285 default:
14286 llvm_unreachable("unsupported private_element_size");
14287 }
14288 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
14289 unsigned Fast = 0;
14290 auto Flags = Store->getMemOperand()->getFlags();
14291 if (allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace: AS,
14292 Alignment: Store->getAlign(), Flags, IsFast: &Fast) &&
14293 Fast > 1)
14294 return SDValue();
14295
14296 return SplitVectorStore(Op, DAG);
14297 }
14298
14299 // Probably an invalid store. If so we'll end up emitting a selection error.
14300 return SDValue();
14301}
14302
14303// Avoid the full correct expansion for f32 sqrt when promoting from f16.
14304SDValue SITargetLowering::lowerFSQRTF16(SDValue Op, SelectionDAG &DAG) const {
14305 SDLoc SL(Op);
14306 assert(!Subtarget->has16BitInsts());
14307 SDNodeFlags Flags = Op->getFlags();
14308 SDValue Ext =
14309 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op.getOperand(i: 0), Flags);
14310
14311 SDValue SqrtID = DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL: SL, VT: MVT::i32);
14312 SDValue Sqrt =
14313 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::f32, N1: SqrtID, N2: Ext, Flags);
14314
14315 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Sqrt,
14316 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
14317}
14318
14319SDValue SITargetLowering::lowerFSQRTF32(SDValue Op, SelectionDAG &DAG) const {
14320 SDLoc DL(Op);
14321 SDNodeFlags Flags = Op->getFlags();
14322 MVT VT = Op.getValueType().getSimpleVT();
14323 const SDValue X = Op.getOperand(i: 0);
14324
14325 if (allowApproxFunc(DAG, Flags)) {
14326 // Instruction is 1ulp but ignores denormals.
14327 return DAG.getNode(
14328 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
14329 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32), N2: X, Flags);
14330 }
14331
14332 SDValue ScaleThreshold = DAG.getConstantFP(Val: 0x1.0p-96f, DL, VT);
14333 SDValue NeedScale = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleThreshold, Cond: ISD::SETOLT);
14334
14335 SDValue ScaleUpFactor = DAG.getConstantFP(Val: 0x1.0p+32f, DL, VT);
14336
14337 SDValue ScaledX = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: ScaleUpFactor, Flags);
14338
14339 SDValue SqrtX =
14340 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledX, N3: X, Flags);
14341
14342 SDValue SqrtS;
14343 if (needsDenormHandlingF32(DAG, Src: X, Flags)) {
14344 SDValue SqrtID =
14345 DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32);
14346 SqrtS = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: SqrtID, N2: SqrtX, Flags);
14347
14348 SDValue SqrtSAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: SqrtS);
14349 SDValue SqrtSNextDownInt =
14350 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14351 N2: DAG.getAllOnesConstant(DL, VT: MVT::i32));
14352 SDValue SqrtSNextDown = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextDownInt);
14353
14354 SDValue NegSqrtSNextDown =
14355 DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextDown, Flags);
14356
14357 SDValue SqrtVP =
14358 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextDown, N2: SqrtS, N3: SqrtX, Flags);
14359
14360 SDValue SqrtSNextUpInt = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14361 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
14362 SDValue SqrtSNextUp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextUpInt);
14363
14364 SDValue NegSqrtSNextUp = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextUp, Flags);
14365 SDValue SqrtVS =
14366 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextUp, N2: SqrtS, N3: SqrtX, Flags);
14367
14368 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
14369 SDValue SqrtVPLE0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVP, RHS: Zero, Cond: ISD::SETOLE);
14370
14371 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPLE0, N2: SqrtSNextDown, N3: SqrtS,
14372 Flags);
14373
14374 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVS, RHS: Zero, Cond: ISD::SETOGT);
14375 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPVSGT0, N2: SqrtSNextUp, N3: SqrtS,
14376 Flags);
14377 } else {
14378 SDValue SqrtR = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: SqrtX, Flags);
14379
14380 SqrtS = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtX, N2: SqrtR, Flags);
14381
14382 SDValue Half = DAG.getConstantFP(Val: 0.5f, DL, VT);
14383 SDValue SqrtH = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtR, N2: Half, Flags);
14384 SDValue NegSqrtH = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtH, Flags);
14385
14386 SDValue SqrtE = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtH, N2: SqrtS, N3: Half, Flags);
14387 SqrtH = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtH, N2: SqrtE, N3: SqrtH, Flags);
14388 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtS, N2: SqrtE, N3: SqrtS, Flags);
14389
14390 SDValue NegSqrtS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtS, Flags);
14391 SDValue SqrtD =
14392 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtS, N2: SqrtS, N3: SqrtX, Flags);
14393 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtD, N2: SqrtH, N3: SqrtS, Flags);
14394 }
14395
14396 SDValue ScaleDownFactor = DAG.getConstantFP(Val: 0x1.0p-16f, DL, VT);
14397
14398 SDValue ScaledDown =
14399 DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtS, N2: ScaleDownFactor, Flags);
14400
14401 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledDown, N3: SqrtS, Flags);
14402 SDValue IsZeroOrInf =
14403 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14404 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14405
14406 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtS, Flags);
14407}
14408
14409SDValue SITargetLowering::lowerFSQRTF64(SDValue Op, SelectionDAG &DAG) const {
14410 // For double type, the SQRT and RSQ instructions don't have required
14411 // precision, we apply Goldschmidt's algorithm to improve the result:
14412 //
14413 // y0 = rsq(x)
14414 // g0 = x * y0
14415 // h0 = 0.5 * y0
14416 //
14417 // r0 = 0.5 - h0 * g0
14418 // g1 = g0 * r0 + g0
14419 // h1 = h0 * r0 + h0
14420 //
14421 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
14422 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
14423 // h2 = h1 * r1 + h1
14424 //
14425 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
14426 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
14427 //
14428 // sqrt(x) = g3
14429
14430 SDNodeFlags Flags = Op->getFlags();
14431
14432 SDLoc DL(Op);
14433
14434 SDValue X = Op.getOperand(i: 0);
14435 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
14436
14437 SDValue SqrtX = X;
14438 SDValue Scaling;
14439 if (!Flags.hasApproximateFuncs()) {
14440 SDValue ScaleConstant = DAG.getConstantFP(Val: 0x1.0p-767, DL, VT: MVT::f64);
14441 Scaling = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleConstant, Cond: ISD::SETOLT);
14442
14443 // Scale up input if it is too small.
14444 SDValue ScaleUpFactor = DAG.getConstant(Val: 256, DL, VT: MVT::i32);
14445 SDValue ScaleUp =
14446 DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling, N2: ScaleUpFactor, N3: ZeroInt);
14447 SqrtX = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: X, N2: ScaleUp, Flags);
14448 }
14449
14450 SDValue SqrtY = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT: MVT::f64, Operand: SqrtX);
14451
14452 SDValue SqrtS0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtX, N2: SqrtY);
14453
14454 SDValue Half = DAG.getConstantFP(Val: 0.5, DL, VT: MVT::f64);
14455 SDValue SqrtH0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtY, N2: Half);
14456
14457 SDValue NegSqrtH0 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtH0);
14458 SDValue SqrtR0 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtH0, N2: SqrtS0, N3: Half);
14459
14460 SDValue SqrtH1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtH0, N2: SqrtR0, N3: SqrtH0);
14461
14462 SDValue SqrtS1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtS0, N2: SqrtR0, N3: SqrtS0);
14463
14464 SDValue NegSqrtS1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS1);
14465 SDValue SqrtD0 =
14466 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS1, N2: SqrtS1, N3: SqrtX);
14467
14468 SDValue SqrtS2 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD0, N2: SqrtH1, N3: SqrtS1);
14469
14470 SDValue SqrtRet = SqrtS2;
14471 if (!Flags.hasApproximateFuncs()) {
14472 SDValue NegSqrtS2 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS2);
14473 SDValue SqrtD1 =
14474 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS2, N2: SqrtS2, N3: SqrtX);
14475
14476 SqrtRet = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD1, N2: SqrtH1, N3: SqrtS2);
14477
14478 SDValue ScaleDownFactor = DAG.getSignedConstant(Val: -128, DL, VT: MVT::i32);
14479 SDValue ScaleDown = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling,
14480 N2: ScaleDownFactor, N3: ZeroInt);
14481 SqrtRet = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: SqrtRet, N2: ScaleDown, Flags);
14482 }
14483
14484 // TODO: Check for DAZ and expand to subnormals
14485
14486 SDValue IsZeroOrInf;
14487 if (Flags.hasNoInfs()) {
14488 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
14489 IsZeroOrInf = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtX, RHS: Zero, Cond: ISD::SETOEQ);
14490 } else {
14491 IsZeroOrInf =
14492 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14493 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14494 }
14495
14496 // If x is +INF, +0, or -0, use its original value
14497 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f64, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtRet,
14498 Flags);
14499}
14500
14501SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
14502 SDLoc DL(Op);
14503 EVT VT = Op.getValueType();
14504 SDValue Arg = Op.getOperand(i: 0);
14505 SDValue TrigVal;
14506
14507 // Propagate fast-math flags so that the multiply we introduce can be folded
14508 // if Arg is already the result of a multiply by constant.
14509 auto Flags = Op->getFlags();
14510
14511 // AMDGPUISD nodes of vector type must be unrolled here since
14512 // they will not be expanded elsewhere.
14513 auto UnrollIfVec = [&DAG](SDValue V) -> SDValue {
14514 if (!V.getValueType().isVector())
14515 return V;
14516
14517 return DAG.UnrollVectorOp(N: cast<SDNode>(Val&: V));
14518 };
14519
14520 SDValue OneOver2Pi = DAG.getConstantFP(Val: 0.5 * numbers::inv_pi, DL, VT);
14521
14522 if (Subtarget->hasTrigReducedRange()) {
14523 SDValue MulVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14524 TrigVal = UnrollIfVec(DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: MulVal, Flags));
14525 } else {
14526 TrigVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14527 }
14528
14529 switch (Op.getOpcode()) {
14530 case ISD::FCOS:
14531 TrigVal = DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14532 break;
14533 case ISD::FSIN:
14534 TrigVal = DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14535 break;
14536 default:
14537 llvm_unreachable("Wrong trig opcode");
14538 }
14539
14540 return UnrollIfVec(TrigVal);
14541}
14542
14543SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
14544 SelectionDAG &DAG) const {
14545 AtomicSDNode *AtomicNode = cast<AtomicSDNode>(Val&: Op);
14546 assert(AtomicNode->isCompareAndSwap());
14547 unsigned AS = AtomicNode->getAddressSpace();
14548
14549 // No custom lowering required for local address space
14550 if (!AMDGPU::isFlatGlobalAddrSpace(AS))
14551 return Op;
14552
14553 // Non-local address space requires custom lowering for atomic compare
14554 // and swap; cmp and swap should be in a v2i32 or v2i64 in case of _X2
14555 SDLoc DL(Op);
14556 SDValue ChainIn = Op.getOperand(i: 0);
14557 SDValue Addr = Op.getOperand(i: 1);
14558 SDValue Old = Op.getOperand(i: 2);
14559 SDValue New = Op.getOperand(i: 3);
14560 EVT VT = Op.getValueType();
14561 MVT SimpleVT = VT.getSimpleVT();
14562 MVT VecType = MVT::getVectorVT(VT: SimpleVT, NumElements: 2);
14563
14564 SDValue NewOld = DAG.getBuildVector(VT: VecType, DL, Ops: {New, Old});
14565 SDValue Ops[] = {ChainIn, Addr, NewOld};
14566
14567 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::ATOMIC_CMP_SWAP, dl: DL,
14568 VTList: Op->getVTList(), Ops, MemVT: VT,
14569 MMO: AtomicNode->getMemOperand());
14570}
14571
14572//===----------------------------------------------------------------------===//
14573// Custom DAG optimizations
14574//===----------------------------------------------------------------------===//
14575
14576SDValue
14577SITargetLowering::performUCharToFloatCombine(SDNode *N,
14578 DAGCombinerInfo &DCI) const {
14579 EVT VT = N->getValueType(ResNo: 0);
14580 EVT ScalarVT = VT.getScalarType();
14581 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
14582 return SDValue();
14583
14584 SelectionDAG &DAG = DCI.DAG;
14585 SDLoc DL(N);
14586
14587 SDValue Src = N->getOperand(Num: 0);
14588 EVT SrcVT = Src.getValueType();
14589
14590 // TODO: We could try to match extracting the higher bytes, which would be
14591 // easier if i8 vectors weren't promoted to i32 vectors, particularly after
14592 // types are legalized. v4i8 -> v4f32 is probably the only case to worry
14593 // about in practice.
14594 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
14595 if (DAG.MaskedValueIsZero(Op: Src, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 24))) {
14596 SDValue Cvt = DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0, DL, VT: MVT::f32, Operand: Src);
14597 DCI.AddToWorklist(N: Cvt.getNode());
14598
14599 // For the f16 case, fold to a cast to f32 and then cast back to f16.
14600 if (ScalarVT != MVT::f32) {
14601 Cvt = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Cvt,
14602 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
14603 }
14604 return Cvt;
14605 }
14606 }
14607
14608 return SDValue();
14609}
14610
14611SDValue SITargetLowering::performFCopySignCombine(SDNode *N,
14612 DAGCombinerInfo &DCI) const {
14613 SDValue MagnitudeOp = N->getOperand(Num: 0);
14614 SDValue SignOp = N->getOperand(Num: 1);
14615
14616 // The generic combine for fcopysign + fp cast is too conservative with
14617 // vectors, and also gets confused by the splitting we will perform here, so
14618 // peek through FP casts.
14619 if (SignOp.getOpcode() == ISD::FP_EXTEND ||
14620 SignOp.getOpcode() == ISD::FP_ROUND)
14621 SignOp = SignOp.getOperand(i: 0);
14622
14623 SelectionDAG &DAG = DCI.DAG;
14624 SDLoc DL(N);
14625 EVT SignVT = SignOp.getValueType();
14626
14627 // f64 fcopysign is really an f32 copysign on the high bits, so replace the
14628 // lower half with a copy.
14629 // fcopysign f64:x, _:y -> x.lo32, (fcopysign (f32 x.hi32), _:y)
14630 EVT MagVT = MagnitudeOp.getValueType();
14631
14632 unsigned NumElts = MagVT.isVector() ? MagVT.getVectorNumElements() : 1;
14633
14634 if (MagVT.getScalarType() == MVT::f64) {
14635 EVT F32VT = MagVT.isVector()
14636 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14637 : MVT::v2f32;
14638
14639 SDValue MagAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: MagnitudeOp);
14640
14641 SmallVector<SDValue, 8> NewElts;
14642 for (unsigned I = 0; I != NumElts; ++I) {
14643 SDValue MagLo =
14644 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14645 N2: DAG.getConstant(Val: 2 * I, DL, VT: MVT::i32));
14646 SDValue MagHi =
14647 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14648 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14649
14650 SDValue SignOpElt =
14651 MagVT.isVector()
14652 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SignVT.getScalarType(),
14653 N1: SignOp, N2: DAG.getConstant(Val: I, DL, VT: MVT::i32))
14654 : SignOp;
14655
14656 SDValue HiOp =
14657 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: MVT::f32, N1: MagHi, N2: SignOpElt);
14658
14659 SDValue Vector =
14660 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2f32, N1: MagLo, N2: HiOp);
14661
14662 SDValue NewElt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Vector);
14663 NewElts.push_back(Elt: NewElt);
14664 }
14665
14666 if (NewElts.size() == 1)
14667 return NewElts[0];
14668
14669 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MagVT, Ops: NewElts);
14670 }
14671
14672 if (SignVT.getScalarType() != MVT::f64)
14673 return SDValue();
14674
14675 // Reduce width of sign operand, we only need the highest bit.
14676 //
14677 // fcopysign f64:x, f64:y ->
14678 // fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1)
14679 // TODO: In some cases it might make sense to go all the way to f16.
14680
14681 EVT F32VT = MagVT.isVector()
14682 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14683 : MVT::v2f32;
14684
14685 SDValue SignAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: SignOp);
14686
14687 SmallVector<SDValue, 8> F32Signs;
14688 for (unsigned I = 0; I != NumElts; ++I) {
14689 // Take sign from odd elements of cast vector
14690 SDValue SignAsF32 =
14691 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: SignAsVector,
14692 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14693 F32Signs.push_back(Elt: SignAsF32);
14694 }
14695
14696 SDValue NewSign =
14697 NumElts == 1
14698 ? F32Signs.back()
14699 : DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL,
14700 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: NumElts),
14701 Ops: F32Signs);
14702
14703 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
14704 N2: NewSign);
14705}
14706
14707// (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2)
14708// (shl (or x, c1), c2) -> add (shl x, c2), (shl c1, c2) iff x and c1 share no
14709// bits
14710
14711// This is a variant of
14712// (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2),
14713//
14714// The normal DAG combiner will do this, but only if the add has one use since
14715// that would increase the number of instructions.
14716//
14717// This prevents us from seeing a constant offset that can be folded into a
14718// memory instruction's addressing mode. If we know the resulting add offset of
14719// a pointer can be folded into an addressing offset, we can replace the pointer
14720// operand with the add of new constant offset. This eliminates one of the uses,
14721// and may allow the remaining use to also be simplified.
14722//
14723SDValue SITargetLowering::performSHLPtrCombine(SDNode *N, unsigned AddrSpace,
14724 EVT MemVT,
14725 DAGCombinerInfo &DCI) const {
14726 SDValue N0 = N->getOperand(Num: 0);
14727 SDValue N1 = N->getOperand(Num: 1);
14728
14729 // We only do this to handle cases where it's profitable when there are
14730 // multiple uses of the add, so defer to the standard combine.
14731 if ((!N0->isAnyAdd() && N0.getOpcode() != ISD::OR) || N0->hasOneUse())
14732 return SDValue();
14733
14734 const ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val&: N1);
14735 if (!CN1)
14736 return SDValue();
14737
14738 const ConstantSDNode *CAdd = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
14739 if (!CAdd)
14740 return SDValue();
14741
14742 SelectionDAG &DAG = DCI.DAG;
14743
14744 if (N0->getOpcode() == ISD::OR &&
14745 !DAG.haveNoCommonBitsSet(A: N0.getOperand(i: 0), B: N0.getOperand(i: 1)))
14746 return SDValue();
14747
14748 // If the resulting offset is too large, we can't fold it into the
14749 // addressing mode offset.
14750 APInt Offset = CAdd->getAPIntValue() << CN1->getAPIntValue();
14751 Type *Ty = MemVT.getTypeForEVT(Context&: *DCI.DAG.getContext());
14752
14753 AddrMode AM;
14754 AM.HasBaseReg = true;
14755 AM.BaseOffs = Offset.getSExtValue();
14756 if (!isLegalAddressingMode(DL: DCI.DAG.getDataLayout(), AM, Ty, AS: AddrSpace))
14757 return SDValue();
14758
14759 SDLoc SL(N);
14760 EVT VT = N->getValueType(ResNo: 0);
14761
14762 SDValue ShlX = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: N0.getOperand(i: 0), N2: N1);
14763 SDValue COffset = DAG.getConstant(Val: Offset, DL: SL, VT);
14764
14765 SDNodeFlags Flags;
14766 Flags.setNoUnsignedWrap(
14767 N->getFlags().hasNoUnsignedWrap() &&
14768 (N0.getOpcode() == ISD::OR || N0->getFlags().hasNoUnsignedWrap()));
14769
14770 // Use ISD::ADD even if the original operation was ISD::PTRADD, since we can't
14771 // be sure that the new left operand is a proper base pointer.
14772 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ShlX, N2: COffset, Flags);
14773}
14774
14775/// MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset
14776/// by the chain and intrinsic ID. Theoretically we would also need to check the
14777/// specific intrinsic, but they all place the pointer operand first.
14778static unsigned getBasePtrIndex(const MemSDNode *N) {
14779 switch (N->getOpcode()) {
14780 case ISD::STORE:
14781 case ISD::INTRINSIC_W_CHAIN:
14782 case ISD::INTRINSIC_VOID:
14783 return 2;
14784 default:
14785 return 1;
14786 }
14787}
14788
14789SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
14790 DAGCombinerInfo &DCI) const {
14791 SelectionDAG &DAG = DCI.DAG;
14792
14793 unsigned PtrIdx = getBasePtrIndex(N);
14794 SDValue Ptr = N->getOperand(Num: PtrIdx);
14795
14796 // TODO: We could also do this for multiplies.
14797 if (Ptr.getOpcode() == ISD::SHL) {
14798 SDValue NewPtr = performSHLPtrCombine(N: Ptr.getNode(), AddrSpace: N->getAddressSpace(),
14799 MemVT: N->getMemoryVT(), DCI);
14800 if (NewPtr) {
14801 SmallVector<SDValue, 8> NewOps(N->ops());
14802
14803 NewOps[PtrIdx] = NewPtr;
14804 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
14805 }
14806 }
14807
14808 return SDValue();
14809}
14810
14811static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val) {
14812 return (Opc == ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
14813 (Opc == ISD::OR && (Val == 0xffffffff || Val == 0)) ||
14814 (Opc == ISD::XOR && Val == 0);
14815}
14816
14817// Break up 64-bit bit operation of a constant into two 32-bit and/or/xor. This
14818// will typically happen anyway for a VALU 64-bit and. This exposes other 32-bit
14819// integer combine opportunities since most 64-bit operations are decomposed
14820// this way. TODO: We won't want this for SALU especially if it is an inline
14821// immediate.
14822SDValue SITargetLowering::splitBinaryBitConstantOp(
14823 DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
14824 const ConstantSDNode *CRHS) const {
14825 uint64_t Val = CRHS->getZExtValue();
14826 uint32_t ValLo = Lo_32(Value: Val);
14827 uint32_t ValHi = Hi_32(Value: Val);
14828 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14829
14830 if ((bitOpWithConstantIsReducible(Opc, Val: ValLo) ||
14831 bitOpWithConstantIsReducible(Opc, Val: ValHi)) ||
14832 (CRHS->hasOneUse() && !TII->isInlineConstant(Imm: CRHS->getAPIntValue()))) {
14833 // We have 64-bit scalar and/or/xor, but do not have vector forms.
14834 if (Subtarget->has64BitLiterals() && CRHS->hasOneUse() &&
14835 !CRHS->user_begin()->isDivergent())
14836 return SDValue();
14837
14838 // If we need to materialize a 64-bit immediate, it will be split up later
14839 // anyway. Avoid creating the harder to understand 64-bit immediate
14840 // materialization.
14841 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
14842 }
14843
14844 return SDValue();
14845}
14846
14847bool llvm::isBoolSGPR(SDValue V) {
14848 if (V.getValueType() != MVT::i1)
14849 return false;
14850 switch (V.getOpcode()) {
14851 default:
14852 break;
14853 case ISD::SETCC:
14854 case ISD::IS_FPCLASS:
14855 case AMDGPUISD::FP_CLASS:
14856 return true;
14857 case ISD::AND:
14858 case ISD::OR:
14859 case ISD::XOR:
14860 return isBoolSGPR(V: V.getOperand(i: 0)) && isBoolSGPR(V: V.getOperand(i: 1));
14861 case ISD::SADDO:
14862 case ISD::UADDO:
14863 case ISD::SSUBO:
14864 case ISD::USUBO:
14865 case ISD::SMULO:
14866 case ISD::UMULO:
14867 return V.getResNo() == 1;
14868 case ISD::INTRINSIC_WO_CHAIN: {
14869 unsigned IntrinsicID = V.getConstantOperandVal(i: 0);
14870 switch (IntrinsicID) {
14871 case Intrinsic::amdgcn_is_shared:
14872 case Intrinsic::amdgcn_is_private:
14873 return true;
14874 default:
14875 return false;
14876 }
14877
14878 return false;
14879 }
14880 }
14881 return false;
14882}
14883
14884static bool isBoolSGPRExt(SDValue V) {
14885 return (V.getOpcode() == ISD::SIGN_EXTEND ||
14886 V.getOpcode() == ISD::ANY_EXTEND) &&
14887 isBoolSGPR(V: V.getOperand(i: 0));
14888}
14889
14890// If a constant has all zeroes or all ones within each byte return it.
14891// Otherwise return 0.
14892static uint32_t getConstantPermuteMask(uint32_t C) {
14893 // 0xff for any zero byte in the mask
14894 uint32_t ZeroByteMask = 0;
14895 if (!(C & 0x000000ff))
14896 ZeroByteMask |= 0x000000ff;
14897 if (!(C & 0x0000ff00))
14898 ZeroByteMask |= 0x0000ff00;
14899 if (!(C & 0x00ff0000))
14900 ZeroByteMask |= 0x00ff0000;
14901 if (!(C & 0xff000000))
14902 ZeroByteMask |= 0xff000000;
14903 uint32_t NonZeroByteMask = ~ZeroByteMask; // 0xff for any non-zero byte
14904 if ((NonZeroByteMask & C) != NonZeroByteMask)
14905 return 0; // Partial bytes selected.
14906 return C;
14907}
14908
14909// Check if a node selects whole bytes from its operand 0 starting at a byte
14910// boundary while masking the rest. Returns select mask as in the v_perm_b32
14911// or -1 if not succeeded.
14912// Note byte select encoding:
14913// value 0-3 selects corresponding source byte;
14914// value 0xc selects zero;
14915// value 0xff selects 0xff.
14916static uint32_t getPermuteMask(SDValue V) {
14917 assert(V.getValueSizeInBits() == 32);
14918
14919 if (V.getNumOperands() != 2)
14920 return ~0;
14921
14922 ConstantSDNode *N1 = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
14923 if (!N1)
14924 return ~0;
14925
14926 uint32_t C = N1->getZExtValue();
14927
14928 switch (V.getOpcode()) {
14929 default:
14930 break;
14931 case ISD::AND:
14932 if (uint32_t ConstMask = getConstantPermuteMask(C))
14933 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
14934 break;
14935
14936 case ISD::OR:
14937 if (uint32_t ConstMask = getConstantPermuteMask(C))
14938 return (0x03020100 & ~ConstMask) | ConstMask;
14939 break;
14940
14941 case ISD::SHL:
14942 if (C % 8)
14943 return ~0;
14944
14945 return uint32_t((0x030201000c0c0c0cull << C) >> 32);
14946
14947 case ISD::SRL:
14948 if (C % 8)
14949 return ~0;
14950
14951 return uint32_t(0x0c0c0c0c03020100ull >> C);
14952 }
14953
14954 return ~0;
14955}
14956
14957SDValue SITargetLowering::performAndCombine(SDNode *N,
14958 DAGCombinerInfo &DCI) const {
14959 if (DCI.isBeforeLegalize())
14960 return SDValue();
14961
14962 SelectionDAG &DAG = DCI.DAG;
14963 EVT VT = N->getValueType(ResNo: 0);
14964 SDValue LHS = N->getOperand(Num: 0);
14965 SDValue RHS = N->getOperand(Num: 1);
14966
14967 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
14968 if (VT == MVT::i64 && CRHS) {
14969 if (SDValue Split =
14970 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::AND, LHS, CRHS))
14971 return Split;
14972 }
14973
14974 if (CRHS && VT == MVT::i32) {
14975 // and (srl x, c), mask => shl (bfe x, nb + c, mask >> nb), nb
14976 // nb = number of trailing zeroes in mask
14977 // It can be optimized out using SDWA for GFX8+ in the SDWA peephole pass,
14978 // given that we are selecting 8 or 16 bit fields starting at byte boundary.
14979 uint64_t Mask = CRHS->getZExtValue();
14980 unsigned Bits = llvm::popcount(Value: Mask);
14981 if (getSubtarget()->hasSDWA() && LHS->getOpcode() == ISD::SRL &&
14982 (Bits == 8 || Bits == 16) && isShiftedMask_64(Value: Mask) && !(Mask & 1)) {
14983 if (auto *CShift = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1))) {
14984 unsigned Shift = CShift->getZExtValue();
14985 unsigned NB = CRHS->getAPIntValue().countr_zero();
14986 unsigned Offset = NB + Shift;
14987 if ((Offset & (Bits - 1)) == 0) { // Starts at a byte or word boundary.
14988 SDLoc SL(N);
14989 SDValue BFE =
14990 DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32, N1: LHS->getOperand(Num: 0),
14991 N2: DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32),
14992 N3: DAG.getConstant(Val: Bits, DL: SL, VT: MVT::i32));
14993 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
14994 SDValue Ext = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT, N1: BFE,
14995 N2: DAG.getValueType(NarrowVT));
14996 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(LHS), VT, N1: Ext,
14997 N2: DAG.getConstant(Val: NB, DL: SDLoc(CRHS), VT: MVT::i32));
14998 return Shl;
14999 }
15000 }
15001 }
15002
15003 // and (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15004 if (LHS.hasOneUse() && LHS.getOpcode() == AMDGPUISD::PERM &&
15005 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
15006 uint32_t Sel = getConstantPermuteMask(C: Mask);
15007 if (!Sel)
15008 return SDValue();
15009
15010 // Select 0xc for all zero bytes
15011 Sel = (LHS.getConstantOperandVal(i: 2) & Sel) | (~Sel & 0x0c0c0c0c);
15012 SDLoc DL(N);
15013 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15014 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15015 }
15016 }
15017
15018 // and (trunc (srl x, c)), mask => srl (trunc x), c
15019 // if mask keeps exactly the bits that srl leaves in the low half
15020 if (CRHS && VT == MVT::i16 && LHS.getOpcode() == ISD::TRUNCATE &&
15021 LHS.hasOneUse() && LHS.getOperand(i: 0).getOpcode() == ISD::SRL) {
15022 auto *Shift = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 0).getOperand(i: 1));
15023 if (Shift && Shift->getZExtValue() < 16 &&
15024 CRHS->getAPIntValue().isMask(numBits: 16 - Shift->getZExtValue())) {
15025 SDLoc SL(N);
15026 SDValue Trunc =
15027 DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: LHS.getOperand(i: 0).getOperand(i: 0));
15028 return DAG.getNode(
15029 Opcode: ISD::SRL, DL: SL, VT, N1: Trunc,
15030 N2: DAG.getShiftAmountConstant(Val: Shift->getZExtValue(), VT, DL: SL));
15031 }
15032 }
15033
15034 // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
15035 // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
15036 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
15037 ISD::CondCode LCC = LHS.getOperandCondCode(i: 2);
15038 ISD::CondCode RCC = RHS.getOperandCondCode(i: 2);
15039
15040 SDValue X = LHS.getOperand(i: 0);
15041 SDValue Y = RHS.getOperand(i: 0);
15042 if (Y.getOpcode() != ISD::FABS || Y.getOperand(i: 0) != X ||
15043 !isTypeLegal(VT: X.getValueType()))
15044 return SDValue();
15045
15046 if (LCC == ISD::SETO) {
15047 if (X != LHS.getOperand(i: 1))
15048 return SDValue();
15049
15050 if (RCC == ISD::SETUNE) {
15051 const ConstantFPSDNode *C1 =
15052 dyn_cast<ConstantFPSDNode>(Val: RHS.getOperand(i: 1));
15053 if (!C1 || !C1->isInfinity() || C1->isNegative())
15054 return SDValue();
15055
15056 const uint32_t Mask = SIInstrFlags::N_NORMAL |
15057 SIInstrFlags::N_SUBNORMAL | SIInstrFlags::N_ZERO |
15058 SIInstrFlags::P_ZERO | SIInstrFlags::P_SUBNORMAL |
15059 SIInstrFlags::P_NORMAL;
15060
15061 static_assert(
15062 ((~(SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN |
15063 SIInstrFlags::N_INFINITY | SIInstrFlags::P_INFINITY)) &
15064 0x3ff) == Mask,
15065 "mask not equal");
15066
15067 SDLoc DL(N);
15068 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: X,
15069 N2: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
15070 }
15071 }
15072 }
15073
15074 if (RHS.getOpcode() == ISD::SETCC && LHS.getOpcode() == AMDGPUISD::FP_CLASS)
15075 std::swap(a&: LHS, b&: RHS);
15076
15077 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15078 RHS.hasOneUse()) {
15079 ISD::CondCode LCC = LHS.getOperandCondCode(i: 2);
15080 // and (fcmp seto), (fp_class x, mask) -> fp_class x, mask & ~(p_nan |
15081 // n_nan) and (fcmp setuo), (fp_class x, mask) -> fp_class x, mask & (p_nan
15082 // | n_nan)
15083 const ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
15084 if ((LCC == ISD::SETO || LCC == ISD::SETUO) && Mask &&
15085 (RHS.getOperand(i: 0) == LHS.getOperand(i: 0) &&
15086 LHS.getOperand(i: 0) == LHS.getOperand(i: 1))) {
15087 const unsigned OrdMask = SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN;
15088 unsigned NewMask = LCC == ISD::SETO ? Mask->getZExtValue() & ~OrdMask
15089 : Mask->getZExtValue() & OrdMask;
15090
15091 SDLoc DL(N);
15092 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: RHS.getOperand(i: 0),
15093 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
15094 }
15095 }
15096
15097 if (VT == MVT::i32 && (RHS.getOpcode() == ISD::SIGN_EXTEND ||
15098 LHS.getOpcode() == ISD::SIGN_EXTEND)) {
15099 // and x, (sext cc from i1) => select cc, x, 0
15100 if (RHS.getOpcode() != ISD::SIGN_EXTEND)
15101 std::swap(a&: LHS, b&: RHS);
15102 if (isBoolSGPR(V: RHS.getOperand(i: 0)))
15103 return DAG.getSelect(DL: SDLoc(N), VT: MVT::i32, Cond: RHS.getOperand(i: 0), LHS,
15104 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32));
15105 }
15106
15107 // and (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15108 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15109 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15110 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15111 uint32_t LHSMask = getPermuteMask(V: LHS);
15112 uint32_t RHSMask = getPermuteMask(V: RHS);
15113 if (LHSMask != ~0u && RHSMask != ~0u) {
15114 // Canonicalize the expression in an attempt to have fewer unique masks
15115 // and therefore fewer registers used to hold the masks.
15116 if (LHSMask > RHSMask) {
15117 std::swap(a&: LHSMask, b&: RHSMask);
15118 std::swap(a&: LHS, b&: RHS);
15119 }
15120
15121 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15122 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15123 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15124 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15125
15126 // Check of we need to combine values from two sources within a byte.
15127 if (!(LHSUsedLanes & RHSUsedLanes) &&
15128 // If we select high and lower word keep it for SDWA.
15129 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15130 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15131 // Each byte in each mask is either selector mask 0-3, or has higher
15132 // bits set in either of masks, which can be 0xff for 0xff or 0x0c for
15133 // zero. If 0x0c is in either mask it shall always be 0x0c. Otherwise
15134 // mask which is not 0xff wins. By anding both masks we have a correct
15135 // result except that 0x0c shall be corrected to give 0x0c only.
15136 uint32_t Mask = LHSMask & RHSMask;
15137 for (unsigned I = 0; I < 32; I += 8) {
15138 uint32_t ByteSel = 0xff << I;
15139 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
15140 Mask &= (0x0c << I) & 0xffffffff;
15141 }
15142
15143 // Add 4 to each active LHS lane. It will not affect any existing 0xff
15144 // or 0x0c.
15145 uint32_t Sel = Mask | (LHSUsedLanes & 0x04040404);
15146 SDLoc DL(N);
15147
15148 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15149 N2: RHS.getOperand(i: 0),
15150 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15151 }
15152 }
15153 }
15154
15155 return SDValue();
15156}
15157
15158// A key component of v_perm is a mapping between byte position of the src
15159// operands, and the byte position of the dest. To provide such, we need: 1. the
15160// node that provides x byte of the dest of the OR, and 2. the byte of the node
15161// used to provide that x byte. calculateByteProvider finds which node provides
15162// a certain byte of the dest of the OR, and calculateSrcByte takes that node,
15163// and finds an ultimate src and byte position For example: The supported
15164// LoadCombine pattern for vector loads is as follows
15165// t1
15166// or
15167// / \
15168// t2 t3
15169// zext shl
15170// | | \
15171// t4 t5 16
15172// or anyext
15173// / \ |
15174// t6 t7 t8
15175// srl shl or
15176// / | / \ / \
15177// t9 t10 t11 t12 t13 t14
15178// trunc* 8 trunc* 8 and and
15179// | | / | | \
15180// t15 t16 t17 t18 t19 t20
15181// trunc* 255 srl -256
15182// | / \
15183// t15 t15 16
15184//
15185// *In this example, the truncs are from i32->i16
15186//
15187// calculateByteProvider would find t6, t7, t13, and t14 for bytes 0-3
15188// respectively. calculateSrcByte would find (given node) -> ultimate src &
15189// byteposition: t6 -> t15 & 1, t7 -> t16 & 0, t13 -> t15 & 0, t14 -> t15 & 3.
15190// After finding the mapping, we can combine the tree into vperm t15, t16,
15191// 0x05000407
15192
15193// Find the source and byte position from a node.
15194// \p DestByte is the byte position of the dest of the or that the src
15195// ultimately provides. \p SrcIndex is the byte of the src that maps to this
15196// dest of the or byte. \p Depth tracks how many recursive iterations we have
15197// performed.
15198static const std::optional<ByteProvider<SDValue>>
15199calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
15200 unsigned Depth = 0) {
15201 // We may need to recursively traverse a series of SRLs
15202 if (Depth >= 6)
15203 return std::nullopt;
15204
15205 if (Op.getValueSizeInBits() < 8)
15206 return std::nullopt;
15207
15208 if (Op.getValueType().isVector())
15209 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
15210
15211 switch (Op->getOpcode()) {
15212 case ISD::TRUNCATE: {
15213 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
15214 }
15215
15216 case ISD::ANY_EXTEND:
15217 case ISD::SIGN_EXTEND:
15218 case ISD::ZERO_EXTEND:
15219 case ISD::SIGN_EXTEND_INREG: {
15220 SDValue NarrowOp = Op->getOperand(Num: 0);
15221 auto NarrowVT = NarrowOp.getValueType();
15222 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG) {
15223 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15224 NarrowVT = VTSign->getVT();
15225 }
15226 if (!NarrowVT.isByteSized())
15227 return std::nullopt;
15228 uint64_t NarrowByteWidth = NarrowVT.getStoreSize();
15229
15230 if (SrcIndex >= NarrowByteWidth)
15231 return std::nullopt;
15232 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
15233 }
15234
15235 case ISD::SRA:
15236 case ISD::SRL: {
15237 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15238 if (!ShiftOp)
15239 return std::nullopt;
15240
15241 uint64_t BitShift = ShiftOp->getZExtValue();
15242
15243 if (BitShift % 8 != 0)
15244 return std::nullopt;
15245
15246 uint64_t NewSrcIndex = SrcIndex + BitShift / 8;
15247 if (NewSrcIndex >= Op.getScalarValueSizeInBits() / 8)
15248 return std::nullopt;
15249
15250 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex: NewSrcIndex,
15251 Depth: Depth + 1);
15252 }
15253
15254 default: {
15255 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
15256 }
15257 }
15258 llvm_unreachable("fully handled switch");
15259}
15260
15261// For a byte position in the result of an Or, traverse the tree and find the
15262// node (and the byte of the node) which ultimately provides this {Or,
15263// BytePosition}. \p Op is the operand we are currently examining. \p Index is
15264// the byte position of the Op that corresponds with the originally requested
15265// byte of the Or \p Depth tracks how many recursive iterations we have
15266// performed. \p StartingIndex is the originally requested byte of the Or
15267static const std::optional<ByteProvider<SDValue>>
15268calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
15269 unsigned StartingIndex = 0) {
15270 // Finding Src tree of RHS of or typically requires at least 1 additional
15271 // depth
15272 if (Depth > 6)
15273 return std::nullopt;
15274
15275 unsigned BitWidth = Op.getScalarValueSizeInBits();
15276 if (BitWidth % 8 != 0)
15277 return std::nullopt;
15278 if (Index > BitWidth / 8 - 1)
15279 return std::nullopt;
15280
15281 bool IsVec = Op.getValueType().isVector();
15282 switch (Op.getOpcode()) {
15283 case ISD::OR: {
15284 if (IsVec)
15285 return std::nullopt;
15286
15287 auto RHS = calculateByteProvider(Op: Op.getOperand(i: 1), Index, Depth: Depth + 1,
15288 StartingIndex);
15289 if (!RHS)
15290 return std::nullopt;
15291 auto LHS = calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15292 StartingIndex);
15293 if (!LHS)
15294 return std::nullopt;
15295 // A well formed Or will have two ByteProviders for each byte, one of which
15296 // is constant zero
15297 if (!LHS->isConstantZero() && !RHS->isConstantZero())
15298 return std::nullopt;
15299 if (!LHS || LHS->isConstantZero())
15300 return RHS;
15301 if (!RHS || RHS->isConstantZero())
15302 return LHS;
15303 return std::nullopt;
15304 }
15305
15306 case ISD::AND: {
15307 if (IsVec)
15308 return std::nullopt;
15309
15310 auto *BitMaskOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15311 if (!BitMaskOp)
15312 return std::nullopt;
15313
15314 uint32_t BitMask = BitMaskOp->getZExtValue();
15315 // Bits we expect for our StartingIndex
15316 uint32_t IndexMask = 0xFF << (Index * 8);
15317
15318 if ((IndexMask & BitMask) != IndexMask) {
15319 // If the result of the and partially provides the byte, then it
15320 // is not well formatted
15321 if (IndexMask & BitMask)
15322 return std::nullopt;
15323 return ByteProvider<SDValue>::getConstantZero();
15324 }
15325
15326 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex, SrcIndex: Index);
15327 }
15328
15329 case ISD::FSHR: {
15330 if (IsVec)
15331 return std::nullopt;
15332
15333 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
15334 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15335 if (!ShiftOp || Op.getValueType().isVector())
15336 return std::nullopt;
15337
15338 uint64_t BitsProvided = Op.getValueSizeInBits();
15339 if (BitsProvided % 8 != 0)
15340 return std::nullopt;
15341
15342 uint64_t BitShift = ShiftOp->getAPIntValue().urem(RHS: BitsProvided);
15343 if (BitShift % 8)
15344 return std::nullopt;
15345
15346 uint64_t ConcatSizeInBytes = BitsProvided / 4;
15347 uint64_t ByteShift = BitShift / 8;
15348
15349 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
15350 uint64_t BytesProvided = BitsProvided / 8;
15351 SDValue NextOp = Op.getOperand(i: NewIndex >= BytesProvided ? 0 : 1);
15352 NewIndex %= BytesProvided;
15353 return calculateByteProvider(Op: NextOp, Index: NewIndex, Depth: Depth + 1, StartingIndex);
15354 }
15355
15356 case ISD::SRA:
15357 case ISD::SRL: {
15358 if (IsVec)
15359 return std::nullopt;
15360
15361 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15362 if (!ShiftOp)
15363 return std::nullopt;
15364
15365 uint64_t BitShift = ShiftOp->getZExtValue();
15366 if (BitShift % 8)
15367 return std::nullopt;
15368
15369 auto BitsProvided = Op.getScalarValueSizeInBits();
15370 if (BitsProvided % 8 != 0)
15371 return std::nullopt;
15372
15373 uint64_t BytesProvided = BitsProvided / 8;
15374 uint64_t ByteShift = BitShift / 8;
15375 if (Index + ByteShift < BytesProvided)
15376 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex,
15377 SrcIndex: Index + ByteShift);
15378 // SRA's out-of-range bytes are sign bits, not constant zero.
15379 if (Op.getOpcode() == ISD::SRA)
15380 return std::nullopt;
15381 return ByteProvider<SDValue>::getConstantZero();
15382 }
15383
15384 case ISD::SHL: {
15385 if (IsVec)
15386 return std::nullopt;
15387
15388 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15389 if (!ShiftOp)
15390 return std::nullopt;
15391
15392 uint64_t BitShift = ShiftOp->getZExtValue();
15393 if (BitShift % 8 != 0)
15394 return std::nullopt;
15395 uint64_t ByteShift = BitShift / 8;
15396
15397 // If we are shifting by an amount greater than (or equal to)
15398 // the index we are trying to provide, then it provides 0s. If not,
15399 // then this bytes are not definitively 0s, and the corresponding byte
15400 // of interest is Index - ByteShift of the src
15401 return Index < ByteShift
15402 ? ByteProvider<SDValue>::getConstantZero()
15403 : calculateByteProvider(Op: Op.getOperand(i: 0), Index: Index - ByteShift,
15404 Depth: Depth + 1, StartingIndex);
15405 }
15406 case ISD::ANY_EXTEND:
15407 case ISD::SIGN_EXTEND:
15408 case ISD::ZERO_EXTEND:
15409 case ISD::SIGN_EXTEND_INREG:
15410 case ISD::AssertZext:
15411 case ISD::AssertSext: {
15412 if (IsVec)
15413 return std::nullopt;
15414
15415 SDValue NarrowOp = Op->getOperand(Num: 0);
15416 unsigned NarrowBitWidth = NarrowOp.getValueSizeInBits();
15417 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG ||
15418 Op->getOpcode() == ISD::AssertZext ||
15419 Op->getOpcode() == ISD::AssertSext) {
15420 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15421 NarrowBitWidth = VTSign->getVT().getSizeInBits();
15422 }
15423 if (NarrowBitWidth % 8 != 0)
15424 return std::nullopt;
15425 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15426
15427 if (Index >= NarrowByteWidth)
15428 return Op.getOpcode() == ISD::ZERO_EXTEND
15429 ? std::optional<ByteProvider<SDValue>>(
15430 ByteProvider<SDValue>::getConstantZero())
15431 : std::nullopt;
15432 return calculateByteProvider(Op: NarrowOp, Index, Depth: Depth + 1, StartingIndex);
15433 }
15434
15435 case ISD::TRUNCATE: {
15436 if (IsVec)
15437 return std::nullopt;
15438
15439 uint64_t NarrowByteWidth = BitWidth / 8;
15440
15441 if (NarrowByteWidth >= Index) {
15442 return calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15443 StartingIndex);
15444 }
15445
15446 return std::nullopt;
15447 }
15448
15449 case ISD::CopyFromReg: {
15450 if (BitWidth / 8 > Index)
15451 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15452
15453 return std::nullopt;
15454 }
15455
15456 case ISD::LOAD: {
15457 auto *L = cast<LoadSDNode>(Val: Op.getNode());
15458
15459 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
15460 if (NarrowBitWidth % 8 != 0)
15461 return std::nullopt;
15462 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15463
15464 // If the width of the load does not reach byte we are trying to provide for
15465 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
15466 // question
15467 if (Index >= NarrowByteWidth) {
15468 return L->getExtensionType() == ISD::ZEXTLOAD
15469 ? std::optional<ByteProvider<SDValue>>(
15470 ByteProvider<SDValue>::getConstantZero())
15471 : std::nullopt;
15472 }
15473
15474 if (NarrowByteWidth > Index) {
15475 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15476 }
15477
15478 return std::nullopt;
15479 }
15480
15481 case ISD::BSWAP: {
15482 if (IsVec)
15483 return std::nullopt;
15484
15485 return calculateByteProvider(Op: Op->getOperand(Num: 0), Index: BitWidth / 8 - Index - 1,
15486 Depth: Depth + 1, StartingIndex);
15487 }
15488
15489 case ISD::EXTRACT_VECTOR_ELT: {
15490 auto *IdxOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15491 if (!IdxOp)
15492 return std::nullopt;
15493 auto VecIdx = IdxOp->getZExtValue();
15494 auto ScalarSize = Op.getScalarValueSizeInBits();
15495 if (ScalarSize < 32)
15496 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
15497 return calculateSrcByte(Op: ScalarSize >= 32 ? Op : Op.getOperand(i: 0),
15498 DestByte: StartingIndex, SrcIndex: Index);
15499 }
15500
15501 case AMDGPUISD::PERM: {
15502 if (IsVec)
15503 return std::nullopt;
15504
15505 auto *PermMask = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15506 if (!PermMask)
15507 return std::nullopt;
15508
15509 auto IdxMask =
15510 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
15511 if (IdxMask > 0x07 && IdxMask != 0x0c)
15512 return std::nullopt;
15513
15514 auto NextOp = Op.getOperand(i: IdxMask > 0x03 ? 0 : 1);
15515 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
15516
15517 return IdxMask != 0x0c ? calculateSrcByte(Op: NextOp, DestByte: StartingIndex, SrcIndex: NextIndex)
15518 : ByteProvider<SDValue>(
15519 ByteProvider<SDValue>::getConstantZero());
15520 }
15521
15522 default: {
15523 return std::nullopt;
15524 }
15525 }
15526
15527 llvm_unreachable("fully handled switch");
15528}
15529
15530// Returns true if the Operand is a scalar and is 16 bits
15531static bool isExtendedFrom16Bits(SDValue &Operand) {
15532
15533 switch (Operand.getOpcode()) {
15534 case ISD::ANY_EXTEND:
15535 case ISD::SIGN_EXTEND:
15536 case ISD::ZERO_EXTEND: {
15537 auto OpVT = Operand.getOperand(i: 0).getValueType();
15538 return !OpVT.isVector() && OpVT.getSizeInBits() == 16;
15539 }
15540 case ISD::LOAD: {
15541 LoadSDNode *L = cast<LoadSDNode>(Val: Operand.getNode());
15542 auto ExtType = cast<LoadSDNode>(Val: L)->getExtensionType();
15543 if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::SEXTLOAD ||
15544 ExtType == ISD::EXTLOAD) {
15545 auto MemVT = L->getMemoryVT();
15546 return !MemVT.isVector() && MemVT.getSizeInBits() == 16;
15547 }
15548 return L->getMemoryVT().getSizeInBits() == 16;
15549 }
15550 default:
15551 return false;
15552 }
15553}
15554
15555// Returns true if the mask matches consecutive bytes, and the first byte
15556// begins at a power of 2 byte offset from 0th byte
15557static bool addresses16Bits(int Mask) {
15558 int Low8 = Mask & 0xff;
15559 int Hi8 = (Mask & 0xff00) >> 8;
15560
15561 assert(Low8 < 8 && Hi8 < 8);
15562 // Are the bytes contiguous in the order of increasing addresses.
15563 bool IsConsecutive = (Hi8 - Low8 == 1);
15564 // Is the first byte at location that is aligned for 16 bit instructions.
15565 // A counter example is taking 2 consecutive bytes starting at the 8th bit.
15566 // In this case, we still need code to extract the 16 bit operand, so it
15567 // is better to use i8 v_perm
15568 bool Is16Aligned = !(Low8 % 2);
15569
15570 return IsConsecutive && Is16Aligned;
15571}
15572
15573// Do not lower into v_perm if the operands are actually 16 bit
15574// and the selected bits (based on PermMask) correspond with two
15575// easily addressable 16 bit operands.
15576static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op,
15577 SDValue &OtherOp) {
15578 int Low16 = PermMask & 0xffff;
15579 int Hi16 = (PermMask & 0xffff0000) >> 16;
15580
15581 auto TempOp = peekThroughBitcasts(V: Op);
15582 auto TempOtherOp = peekThroughBitcasts(V: OtherOp);
15583
15584 auto OpIs16Bit =
15585 TempOp.getValueSizeInBits() == 16 || isExtendedFrom16Bits(Operand&: TempOp);
15586 if (!OpIs16Bit)
15587 return true;
15588
15589 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
15590 isExtendedFrom16Bits(Operand&: TempOtherOp);
15591 if (!OtherOpIs16Bit)
15592 return true;
15593
15594 // Do we cleanly address both
15595 return !addresses16Bits(Mask: Low16) || !addresses16Bits(Mask: Hi16);
15596}
15597
15598static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src,
15599 unsigned DWordOffset) {
15600 SDValue Ret;
15601
15602 auto TypeSize = Src.getValueSizeInBits().getFixedValue();
15603 // ByteProvider must be at least 8 bits
15604 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
15605
15606 if (TypeSize <= 32)
15607 return DAG.getBitcastedAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
15608
15609 if (Src.getValueType().isVector()) {
15610 auto ScalarTySize = Src.getScalarValueSizeInBits();
15611 auto ScalarTy = Src.getValueType().getScalarType();
15612 if (ScalarTySize == 32) {
15613 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Src,
15614 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15615 }
15616 if (ScalarTySize > 32) {
15617 Ret = DAG.getNode(
15618 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ScalarTy, N1: Src,
15619 N2: DAG.getConstant(Val: DWordOffset / (ScalarTySize / 32), DL: SL, VT: MVT::i32));
15620 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
15621 if (ShiftVal)
15622 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Ret.getValueType(), N1: Ret,
15623 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15624 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15625 }
15626
15627 assert(ScalarTySize < 32);
15628 if (TypeSize % 32 == 0) {
15629 assert(DWordOffset < TypeSize / 32);
15630 SDValue Cast = DAG.getBitcast(
15631 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: TypeSize / 32), V: Src);
15632 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
15633 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15634 }
15635
15636 auto NumElements = TypeSize / ScalarTySize;
15637 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
15638 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
15639 auto NumElementsIn32 = 32 / ScalarTySize;
15640 auto NumAvailElements = DWordOffset < Trunc32Elements
15641 ? NumElementsIn32
15642 : NumElements - NormalizedTrunc;
15643
15644 SmallVector<SDValue, 4> VecSrcs;
15645 DAG.ExtractVectorElements(Op: Src, Args&: VecSrcs, Start: DWordOffset * NumElementsIn32,
15646 Count: NumAvailElements);
15647
15648 Ret = DAG.getBuildVector(
15649 VT: MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ScalarTySize), NumElements: NumAvailElements), DL: SL,
15650 Ops: VecSrcs);
15651 return Ret = DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15652 }
15653
15654 /// Scalar Type
15655 auto ShiftVal = 32 * DWordOffset;
15656 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Src.getValueType(), N1: Src,
15657 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15658 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15659}
15660
15661static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15662 SelectionDAG &DAG = DCI.DAG;
15663 [[maybe_unused]] EVT VT = N->getValueType(ResNo: 0);
15664 SmallVector<ByteProvider<SDValue>, 8> PermNodes;
15665
15666 // VT is known to be MVT::i32, so we need to provide 4 bytes.
15667 assert(VT == MVT::i32);
15668 for (int i = 0; i < 4; i++) {
15669 // Find the ByteProvider that provides the ith byte of the result of OR
15670 std::optional<ByteProvider<SDValue>> P =
15671 calculateByteProvider(Op: SDValue(N, 0), Index: i, Depth: 0, /*StartingIndex = */ i);
15672 // TODO support constantZero
15673 if (!P || P->isConstantZero())
15674 return SDValue();
15675
15676 PermNodes.push_back(Elt: *P);
15677 }
15678 if (PermNodes.size() != 4)
15679 return SDValue();
15680
15681 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
15682 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
15683 uint64_t PermMask = 0x00000000;
15684 for (size_t i = 0; i < PermNodes.size(); i++) {
15685 auto PermOp = PermNodes[i];
15686 // Since the mask is applied to Src1:Src2, Src1 bytes must be offset
15687 // by sizeof(Src2) = 4
15688 int SrcByteAdjust = 4;
15689
15690 // If the Src uses a byte from a different DWORD, then it corresponds
15691 // with a difference source
15692 if (!PermOp.hasSameSrc(Other: PermNodes[FirstSrc.first]) ||
15693 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
15694 if (SecondSrc)
15695 if (!PermOp.hasSameSrc(Other: PermNodes[SecondSrc->first]) ||
15696 ((PermOp.SrcOffset / 4) != SecondSrc->second))
15697 return SDValue();
15698
15699 // Set the index of the second distinct Src node
15700 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
15701 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
15702 SrcByteAdjust = 0;
15703 }
15704 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
15705 assert(!DAG.getDataLayout().isBigEndian());
15706 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
15707 }
15708 SDLoc DL(N);
15709 SDValue Op = *PermNodes[FirstSrc.first].Src;
15710 Op = getDWordFromOffset(DAG, SL: DL, Src: Op, DWordOffset: FirstSrc.second);
15711 assert(Op.getValueSizeInBits() == 32);
15712
15713 // Check that we are not just extracting the bytes in order from an op
15714 if (!SecondSrc) {
15715 int Low16 = PermMask & 0xffff;
15716 int Hi16 = (PermMask & 0xffff0000) >> 16;
15717
15718 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
15719 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
15720
15721 // The perm op would really just produce Op. So combine into Op
15722 if (WellFormedLow && WellFormedHi)
15723 return DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: 32), V: Op);
15724 }
15725
15726 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src : Op;
15727
15728 if (SecondSrc) {
15729 OtherOp = getDWordFromOffset(DAG, SL: DL, Src: OtherOp, DWordOffset: SecondSrc->second);
15730 assert(OtherOp.getValueSizeInBits() == 32);
15731 }
15732
15733 // Check that we haven't just recreated the same FSHR node.
15734 if (N->getOpcode() == ISD::FSHR &&
15735 (N->getOperand(Num: 0) == Op || N->getOperand(Num: 0) == OtherOp) &&
15736 (N->getOperand(Num: 1) == Op || N->getOperand(Num: 1) == OtherOp))
15737 return SDValue();
15738
15739 if (hasNon16BitAccesses(PermMask, Op, OtherOp)) {
15740
15741 assert(Op.getValueType().isByteSized() &&
15742 OtherOp.getValueType().isByteSized());
15743
15744 // If the ultimate src is less than 32 bits, then we will only be
15745 // using bytes 0: Op.getValueSizeInBytes() - 1 in the or.
15746 // CalculateByteProvider would not have returned Op as source if we
15747 // used a byte that is outside its ValueType. Thus, we are free to
15748 // ANY_EXTEND as the extended bits are dont-cares.
15749 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, VT: MVT::i32);
15750 OtherOp = DAG.getBitcastedAnyExtOrTrunc(Op: OtherOp, DL, VT: MVT::i32);
15751
15752 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op, N2: OtherOp,
15753 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
15754 }
15755 return SDValue();
15756}
15757
15758SDValue SITargetLowering::performOrCombine(SDNode *N,
15759 DAGCombinerInfo &DCI) const {
15760 SelectionDAG &DAG = DCI.DAG;
15761 SDValue LHS = N->getOperand(Num: 0);
15762 SDValue RHS = N->getOperand(Num: 1);
15763
15764 EVT VT = N->getValueType(ResNo: 0);
15765 if (VT == MVT::i1) {
15766 // or (fp_class x, c1), (fp_class x, c2) -> fp_class x, (c1 | c2)
15767 if (LHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15768 RHS.getOpcode() == AMDGPUISD::FP_CLASS) {
15769 SDValue Src = LHS.getOperand(i: 0);
15770 if (Src != RHS.getOperand(i: 0))
15771 return SDValue();
15772
15773 const ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
15774 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
15775 if (!CLHS || !CRHS)
15776 return SDValue();
15777
15778 // Only 10 bits are used.
15779 static const uint32_t MaxMask = 0x3ff;
15780
15781 uint32_t NewMask =
15782 (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask;
15783 SDLoc DL(N);
15784 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: Src,
15785 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
15786 }
15787
15788 return SDValue();
15789 }
15790
15791 if (VT == MVT::i32 && !DCI.isBeforeLegalize()) {
15792 for (unsigned I : {1u, 0u}) {
15793 SDValue Ext = N->getOperand(Num: I);
15794 // The select may need an e64 cndmask, so only fold when it eliminates
15795 // the extension.
15796 if (!Ext.hasOneUse() || !isBoolSGPRExt(V: Ext))
15797 continue;
15798 // A divergent select still needs to materialize a uniform condition.
15799 // Sharing that condition can also require converting it back to SCC.
15800 SDValue Cond = Ext.getOperand(i: 0);
15801 if (N->isDivergent() && !Cond->isDivergent() && !Cond.hasOneUse())
15802 continue;
15803
15804 // or x, (sext/anyext cc from i1) => select cc, -1, x
15805 // Any-extended bits can be chosen to match sign extension.
15806 SDLoc DL(N);
15807 return DAG.getSelect(DL, VT: MVT::i32, Cond,
15808 LHS: DAG.getAllOnesConstant(DL, VT: MVT::i32),
15809 RHS: N->getOperand(Num: 1 - I));
15810 }
15811 }
15812
15813 // or (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15814 if (isa<ConstantSDNode>(Val: RHS) && LHS.hasOneUse() &&
15815 LHS.getOpcode() == AMDGPUISD::PERM &&
15816 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
15817 uint32_t Sel = getConstantPermuteMask(C: N->getConstantOperandVal(Num: 1));
15818 if (!Sel)
15819 return SDValue();
15820
15821 Sel |= LHS.getConstantOperandVal(i: 2);
15822 SDLoc DL(N);
15823 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15824 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15825 }
15826
15827 // or (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15828 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15829 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15830 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15831
15832 // If all the uses of an or need to extract the individual elements, do not
15833 // attempt to lower into v_perm
15834 auto usesCombinedOperand = [](SDNode *OrUse) {
15835 // If we have any non-vectorized use, then it is a candidate for v_perm
15836 if (OrUse->getOpcode() != ISD::BITCAST ||
15837 !OrUse->getValueType(ResNo: 0).isVector())
15838 return true;
15839
15840 // If we have any non-vectorized use, then it is a candidate for v_perm
15841 for (auto *VUser : OrUse->users()) {
15842 if (!VUser->getValueType(ResNo: 0).isVector())
15843 return true;
15844
15845 // If the use of a vector is a store, then combining via a v_perm
15846 // is beneficial.
15847 // TODO -- whitelist more uses
15848 for (auto VectorwiseOp : {ISD::STORE, ISD::CopyToReg, ISD::CopyFromReg})
15849 if (VUser->getOpcode() == VectorwiseOp)
15850 return true;
15851 }
15852 return false;
15853 };
15854
15855 if (!any_of(Range: N->users(), P: usesCombinedOperand))
15856 return SDValue();
15857
15858 uint32_t LHSMask = getPermuteMask(V: LHS);
15859 uint32_t RHSMask = getPermuteMask(V: RHS);
15860
15861 if (LHSMask != ~0u && RHSMask != ~0u) {
15862 // Canonicalize the expression in an attempt to have fewer unique masks
15863 // and therefore fewer registers used to hold the masks.
15864 if (LHSMask > RHSMask) {
15865 std::swap(a&: LHSMask, b&: RHSMask);
15866 std::swap(a&: LHS, b&: RHS);
15867 }
15868
15869 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15870 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15871 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15872 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15873
15874 // Check of we need to combine values from two sources within a byte.
15875 if (!(LHSUsedLanes & RHSUsedLanes) &&
15876 // If we select high and lower word keep it for SDWA.
15877 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15878 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15879 // Kill zero bytes selected by other mask. Zero value is 0xc.
15880 LHSMask &= ~RHSUsedLanes;
15881 RHSMask &= ~LHSUsedLanes;
15882 // Add 4 to each active LHS lane
15883 LHSMask |= LHSUsedLanes & 0x04040404;
15884 // Combine masks
15885 uint32_t Sel = LHSMask | RHSMask;
15886 SDLoc DL(N);
15887
15888 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15889 N2: RHS.getOperand(i: 0),
15890 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15891 }
15892 }
15893 if (LHSMask == ~0u || RHSMask == ~0u) {
15894 if (SDValue Perm = matchPERM(N, DCI))
15895 return Perm;
15896 }
15897 }
15898
15899 // Detect identity v2i32 OR and replace with identity source node.
15900 // Specifically an Or that has operands constructed from the same source node
15901 // via extract_vector_elt and build_vector. I.E.
15902 // v2i32 or(
15903 // v2i32 build_vector(
15904 // i32 extract_elt(%IdentitySrc, 0),
15905 // i32 0
15906 // ),
15907 // v2i32 build_vector(
15908 // i32 0,
15909 // i32 extract_elt(%IdentitySrc, 1)
15910 // ) )
15911 // =>
15912 // v2i32 %IdentitySrc
15913
15914 if (VT == MVT::v2i32 && LHS->getOpcode() == ISD::BUILD_VECTOR &&
15915 RHS->getOpcode() == ISD::BUILD_VECTOR) {
15916
15917 ConstantSDNode *LC = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1));
15918 ConstantSDNode *RC = dyn_cast<ConstantSDNode>(Val: RHS->getOperand(Num: 0));
15919
15920 // Test for and normalise build vectors.
15921 if (LC && RC && LC->getZExtValue() == 0 && RC->getZExtValue() == 0) {
15922
15923 // Get the extract_vector_element operands.
15924 SDValue LEVE = LHS->getOperand(Num: 0);
15925 SDValue REVE = RHS->getOperand(Num: 1);
15926
15927 if (LEVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15928 REVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
15929 // Check that different elements from the same vector are
15930 // extracted.
15931 if (LEVE->getOperand(Num: 0) == REVE->getOperand(Num: 0) &&
15932 LEVE->getOperand(Num: 1) != REVE->getOperand(Num: 1)) {
15933 SDValue IdentitySrc = LEVE.getOperand(i: 0);
15934 return IdentitySrc;
15935 }
15936 }
15937 }
15938 }
15939
15940 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
15941 return SDValue();
15942
15943 // TODO: This could be a generic combine with a predicate for extracting the
15944 // high half of an integer being free.
15945
15946 // (or i64:x, (zero_extend i32:y)) ->
15947 // i64 (bitcast (v2i32 build_vector (or i32:y, lo_32(x)), hi_32(x)))
15948 if (LHS.getOpcode() == ISD::ZERO_EXTEND &&
15949 RHS.getOpcode() != ISD::ZERO_EXTEND)
15950 std::swap(a&: LHS, b&: RHS);
15951
15952 if (RHS.getOpcode() == ISD::ZERO_EXTEND) {
15953 SDValue ExtSrc = RHS.getOperand(i: 0);
15954 EVT SrcVT = ExtSrc.getValueType();
15955 if (SrcVT == MVT::i32) {
15956 SDLoc SL(N);
15957 auto [LowLHS, HiBits] = split64BitValue(Op: LHS, DAG);
15958 SDValue LowOr = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: LowLHS, N2: ExtSrc);
15959
15960 DCI.AddToWorklist(N: LowOr.getNode());
15961 DCI.AddToWorklist(N: HiBits.getNode());
15962
15963 SDValue Vec =
15964 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: LowOr, N2: HiBits);
15965 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
15966 }
15967 }
15968
15969 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
15970 if (CRHS) {
15971 if (SDValue Split = splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::OR,
15972 LHS: N->getOperand(Num: 0), CRHS))
15973 return Split;
15974 }
15975
15976 return SDValue();
15977}
15978
15979SDValue SITargetLowering::performXorCombine(SDNode *N,
15980 DAGCombinerInfo &DCI) const {
15981 if (SDValue RV = reassociateScalarOps(N, DAG&: DCI.DAG))
15982 return RV;
15983
15984 SDValue LHS = N->getOperand(Num: 0);
15985 SDValue RHS = N->getOperand(Num: 1);
15986
15987 const ConstantSDNode *CRHS = isConstOrConstSplat(N: RHS);
15988 SelectionDAG &DAG = DCI.DAG;
15989
15990 EVT VT = N->getValueType(ResNo: 0);
15991 if (CRHS && VT == MVT::i64) {
15992 if (SDValue Split =
15993 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::XOR, LHS, CRHS))
15994 return Split;
15995 }
15996
15997 // v2i32 (xor (vselect cc, x, y), K) ->
15998 // (v2i32 svelect cc, (xor x, K), (xor y, K)) This enables the xor to be
15999 // replaced with source modifiers when the select is lowered to CNDMASK.
16000 unsigned Opc = LHS.getOpcode();
16001 if (((Opc == ISD::VSELECT && VT == MVT::v2i32) ||
16002 (Opc == ISD::SELECT && VT == MVT::i64)) &&
16003 CRHS && CRHS->getAPIntValue().isSignMask()) {
16004 SDValue CC = LHS->getOperand(Num: 0);
16005 SDValue TRUE = LHS->getOperand(Num: 1);
16006 SDValue FALSE = LHS->getOperand(Num: 2);
16007 SDValue XTrue = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: TRUE, N2: RHS);
16008 SDValue XFalse = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: FALSE, N2: RHS);
16009 SDValue XSelect =
16010 DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT, N1: CC, N2: XTrue, N3: XFalse);
16011 return XSelect;
16012 }
16013
16014 // Make sure to apply the 64-bit constant splitting fold before trying to fold
16015 // fneg-like xors into 64-bit select.
16016 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) {
16017 // This looks like an fneg, try to fold as a source modifier.
16018 if (CRHS && CRHS->getAPIntValue().isSignMask() &&
16019 shouldFoldFNegIntoSrc(FNeg: N, FNegSrc: LHS)) {
16020 // xor (select c, a, b), 0x80000000 ->
16021 // bitcast (select c, (fneg (bitcast a)), (fneg (bitcast b)))
16022 SDLoc DL(N);
16023 SDValue CastLHS =
16024 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 1));
16025 SDValue CastRHS =
16026 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 2));
16027 SDValue FNegLHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastLHS);
16028 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastRHS);
16029 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f32,
16030 N1: LHS->getOperand(Num: 0), N2: FNegLHS, N3: FNegRHS);
16031 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: NewSelect);
16032 }
16033 }
16034
16035 return SDValue();
16036}
16037
16038SDValue
16039SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
16040 DAGCombinerInfo &DCI) const {
16041 if (!Subtarget->has16BitInsts() ||
16042 DCI.getDAGCombineLevel() < AfterLegalizeTypes)
16043 return SDValue();
16044
16045 EVT VT = N->getValueType(ResNo: 0);
16046 if (VT != MVT::i32)
16047 return SDValue();
16048
16049 SDValue Src = N->getOperand(Num: 0);
16050 if (Src.getValueType() != MVT::i16)
16051 return SDValue();
16052
16053 if (!Src->hasOneUse())
16054 return SDValue();
16055
16056 if (SDValue Ext = performExtendTruncShiftCombine(N, DCI))
16057 return Ext;
16058
16059 // TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
16060 // possible we're missing out on some combine opportunities, but we'd need to
16061 // weigh the cost of extracting the byte from the upper dwords.
16062
16063 std::optional<ByteProvider<SDValue>> BP0 =
16064 calculateByteProvider(Op: SDValue(N, 0), Index: 0, Depth: 0, StartingIndex: 0);
16065 if (!BP0 || BP0->SrcOffset >= 4 || !BP0->Src)
16066 return SDValue();
16067 SDValue V0 = *BP0->Src;
16068
16069 std::optional<ByteProvider<SDValue>> BP1 =
16070 calculateByteProvider(Op: SDValue(N, 0), Index: 1, Depth: 0, StartingIndex: 1);
16071 if (!BP1 || BP1->SrcOffset >= 4 || !BP1->Src)
16072 return SDValue();
16073
16074 SDValue V1 = *BP1->Src;
16075
16076 if (V0 == V1)
16077 return SDValue();
16078
16079 SelectionDAG &DAG = DCI.DAG;
16080 SDLoc DL(N);
16081 uint32_t PermMask = 0x0c0c0c0c;
16082 if (V0) {
16083 V0 = getDWordFromOffset(DAG, SL: DL, Src: V0, /*DWordOffset=*/0);
16084 PermMask = (PermMask & ~0xFF) | (BP0->SrcOffset + 4);
16085 }
16086
16087 if (V1) {
16088 V1 = getDWordFromOffset(DAG, SL: DL, Src: V1, /*DWordOffset=*/0);
16089 PermMask = (PermMask & ~(0xFF << 8)) | (BP1->SrcOffset << 8);
16090 }
16091
16092 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: V0, N2: V1,
16093 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
16094}
16095
16096// zext (srl (trunc x), c) => and (srl x, c), lowbits
16097// aext (srl (trunc x), c) => and (srl x, c), lowbits
16098// sext (sra (trunc x), 8) => sext_inreg (srl x, 8), i8
16099SDValue
16100SITargetLowering::performExtendTruncShiftCombine(SDNode *N,
16101 DAGCombinerInfo &DCI) const {
16102 if (DCI.getDAGCombineLevel() < AfterLegalizeTypes)
16103 return SDValue();
16104
16105 const unsigned Opc = N->getOpcode();
16106 const bool IsSigned = Opc == ISD::SIGN_EXTEND;
16107 SDValue Src = N->getOperand(Num: 0);
16108 if (N->getValueType(ResNo: 0) != MVT::i32 || Src.getValueType() != MVT::i16 ||
16109 Src.getOpcode() != (IsSigned ? ISD::SRA : ISD::SRL) || !Src.hasOneUse() ||
16110 Src.getOperand(i: 0).getOpcode() != ISD::TRUNCATE)
16111 return SDValue();
16112
16113 // A uniform i16 srl is promoted to i32 later and yields the same nodes.
16114 // Skip it so that the node order, and with it the schedule, stays the same.
16115 if (!IsSigned && !Src->isDivergent())
16116 return SDValue();
16117
16118 SDValue X = Src.getOperand(i: 0).getOperand(i: 0);
16119 auto *Shift = dyn_cast<ConstantSDNode>(Val: Src.getOperand(i: 1));
16120 if (X.getValueType() != MVT::i32 || !Shift || Shift->getZExtValue() == 0 ||
16121 Shift->getZExtValue() >= 16 || (IsSigned && Shift->getZExtValue() != 8))
16122 return SDValue();
16123
16124 SelectionDAG &DAG = DCI.DAG;
16125 SDLoc SL(N);
16126 const unsigned Width = 16 - Shift->getZExtValue();
16127 SDValue Srl = DAG.getNode(
16128 Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: X,
16129 N2: DAG.getShiftAmountConstant(Val: Shift->getZExtValue(), VT: MVT::i32, DL: SL));
16130 if (IsSigned)
16131 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SL, VT: MVT::i32, N1: Srl,
16132 N2: DAG.getValueType(MVT::i8));
16133 return DAG.getNode(
16134 Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Srl,
16135 N2: DAG.getConstant(Val: maskTrailingOnes<uint32_t>(N: Width), DL: SL, VT: MVT::i32));
16136}
16137
16138SDValue
16139SITargetLowering::performSignExtendInRegCombine(SDNode *N,
16140 DAGCombinerInfo &DCI) const {
16141 SDValue Src = N->getOperand(Num: 0);
16142 auto *VTSign = cast<VTSDNode>(Val: N->getOperand(Num: 1));
16143
16144 // Combine s_buffer_load_u8 or s_buffer_load_u16 with sext and replace them
16145 // with s_buffer_load_i8 and s_buffer_load_i16 respectively.
16146 if (((Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE &&
16147 VTSign->getVT() == MVT::i8) ||
16148 (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_USHORT &&
16149 VTSign->getVT() == MVT::i16))) {
16150 assert(Subtarget->hasScalarSubwordLoads() &&
16151 "s_buffer_load_{u8, i8} are supported "
16152 "in GFX12 (or newer) architectures.");
16153 unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
16154 ? AMDGPUISD::SBUFFER_LOAD_BYTE
16155 : AMDGPUISD::SBUFFER_LOAD_SHORT;
16156 SDLoc DL(N);
16157 SDVTList ResList =
16158 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
16159 SDValue Ops[] = {
16160 Src.getOperand(i: 0), // Chain
16161 Src.getOperand(i: 1), // source register
16162 Src.getOperand(i: 2), // offset
16163 Src.getOperand(i: 3) // cachePolicy
16164 };
16165 auto *M = cast<MemSDNode>(Val&: Src);
16166 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
16167 Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
16168 return DCI.DAG.getMergeValues(Ops: {BufferLoad, BufferLoad.getValue(R: 1)}, dl: DL);
16169 }
16170 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
16171 VTSign->getVT() == MVT::i8) ||
16172 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT &&
16173 VTSign->getVT() == MVT::i16)) &&
16174 Src.hasOneUse()) {
16175 auto *M = cast<MemSDNode>(Val&: Src);
16176 SDValue Ops[] = {Src.getOperand(i: 0), // Chain
16177 Src.getOperand(i: 1), // rsrc
16178 Src.getOperand(i: 2), // vindex
16179 Src.getOperand(i: 3), // voffset
16180 Src.getOperand(i: 4), // soffset
16181 Src.getOperand(i: 5), // offset
16182 Src.getOperand(i: 6), Src.getOperand(i: 7)};
16183 // replace with BUFFER_LOAD_BYTE/SHORT
16184 SDVTList ResList =
16185 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
16186 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE)
16187 ? AMDGPUISD::BUFFER_LOAD_BYTE
16188 : AMDGPUISD::BUFFER_LOAD_SHORT;
16189 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
16190 Opcode: Opc, dl: SDLoc(N), VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
16191 return DCI.DAG.getMergeValues(
16192 Ops: {BufferLoadSignExt, BufferLoadSignExt.getValue(R: 1)}, dl: SDLoc(N));
16193 }
16194 return SDValue();
16195}
16196
16197SDValue SITargetLowering::performClassCombine(SDNode *N,
16198 DAGCombinerInfo &DCI) const {
16199 SelectionDAG &DAG = DCI.DAG;
16200 SDValue Mask = N->getOperand(Num: 1);
16201
16202 // fp_class x, 0 -> false
16203 if (isNullConstant(V: Mask))
16204 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i1);
16205
16206 if (N->getOperand(Num: 0).isUndef())
16207 return DAG.getUNDEF(VT: MVT::i1);
16208
16209 return SDValue();
16210}
16211
16212SDValue SITargetLowering::performRcpCombine(SDNode *N,
16213 DAGCombinerInfo &DCI) const {
16214 EVT VT = N->getValueType(ResNo: 0);
16215 SDValue N0 = N->getOperand(Num: 0);
16216
16217 if (N0.isUndef()) {
16218 return DCI.DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()),
16219 DL: SDLoc(N), VT);
16220 }
16221
16222 // TODO: Could handle f32 + amdgcn.sqrt but probably never reaches here.
16223 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) &&
16224 N->getFlags().hasAllowContract() && N0->getFlags().hasAllowContract()) {
16225 return DCI.DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(N), VT, Operand: N0.getOperand(i: 0),
16226 Flags: N->getFlags());
16227 }
16228
16229 return AMDGPUTargetLowering::performRcpCombine(N, DCI);
16230}
16231
16232bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
16233 SDNodeFlags UserFlags,
16234 unsigned MaxDepth) const {
16235 EVT VT = Op.getValueType();
16236 assert(VT.isFloatingPoint() &&
16237 "expected a floating-point value to query canonicality of");
16238 return isCanonicalized(DAG, Op, QueryVT: VT.getScalarType(), UserFlags, MaxDepth);
16239}
16240
16241bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
16242 EVT QueryVT, SDNodeFlags UserFlags,
16243 unsigned MaxDepth) const {
16244 assert(QueryVT.isFloatingPoint() && !QueryVT.isVector() &&
16245 "QueryVT must be a floating-point scalar type");
16246 EVT VT = Op.getValueType();
16247 if (VT.isFloatingPoint() && VT.getScalarType() != QueryVT)
16248 return false;
16249
16250 unsigned Opcode = Op.getOpcode();
16251 if (Opcode == ISD::FCANONICALIZE)
16252 return true;
16253
16254 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16255 const auto &F = CFP->getValueAPF();
16256 if (F.isNaN() && F.isSignaling())
16257 return false;
16258 if (!F.isDenormal())
16259 return true;
16260
16261 DenormalMode Mode =
16262 DAG.getMachineFunction().getDenormalMode(FPType: F.getSemantics());
16263 return Mode == DenormalMode::getIEEE();
16264 }
16265
16266 // If source is a result of another standard FP operation it is already in
16267 // canonical form.
16268 if (MaxDepth == 0)
16269 return false;
16270
16271 switch (Opcode) {
16272 // These will flush denorms if required.
16273 case ISD::FADD:
16274 case ISD::FSUB:
16275 case ISD::FMUL:
16276 case ISD::FCEIL:
16277 case ISD::FFLOOR:
16278 case ISD::FMA:
16279 case ISD::FMAD:
16280 case ISD::FSQRT:
16281 case ISD::FDIV:
16282 case ISD::FREM:
16283 case ISD::FP_ROUND:
16284 case ISD::FP_EXTEND:
16285 case ISD::FP16_TO_FP:
16286 case ISD::FP_TO_FP16:
16287 case ISD::BF16_TO_FP:
16288 case ISD::FP_TO_BF16:
16289 case ISD::FLDEXP:
16290 case AMDGPUISD::FMUL_LEGACY:
16291 case AMDGPUISD::FMAD_FTZ:
16292 case AMDGPUISD::RCP:
16293 case AMDGPUISD::RSQ:
16294 case AMDGPUISD::RSQ_CLAMP:
16295 case AMDGPUISD::RCP_LEGACY:
16296 case AMDGPUISD::RCP_IFLAG:
16297 case AMDGPUISD::LOG:
16298 case AMDGPUISD::EXP:
16299 case AMDGPUISD::DIV_SCALE:
16300 case AMDGPUISD::DIV_FMAS:
16301 case AMDGPUISD::DIV_FIXUP:
16302 case AMDGPUISD::FRACT:
16303 case AMDGPUISD::CVT_PKRTZ_F16_F32:
16304 case AMDGPUISD::CVT_F32_UBYTE0:
16305 case AMDGPUISD::CVT_F32_UBYTE1:
16306 case AMDGPUISD::CVT_F32_UBYTE2:
16307 case AMDGPUISD::CVT_F32_UBYTE3:
16308 case AMDGPUISD::FP_TO_FP16:
16309 case AMDGPUISD::SIN_HW:
16310 case AMDGPUISD::COS_HW:
16311 return true;
16312
16313 // It can/will be lowered or combined as a bit operation.
16314 // Need to check their input recursively to handle.
16315 case ISD::FNEG:
16316 case ISD::FABS:
16317 case ISD::FCOPYSIGN:
16318 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16319 MaxDepth: MaxDepth - 1);
16320
16321 case ISD::AND:
16322 if (Op.getValueType() == MVT::i32) {
16323 // Be careful as we only know it is a bitcast floating point type. It
16324 // could be f32, v2f16, we have no way of knowing. Luckily the constant
16325 // value that we optimize for, which comes up in fp32 to bf16 conversions,
16326 // is valid to optimize for all types.
16327 if (auto *RHS = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
16328 if (RHS->getZExtValue() == 0xffff0000) {
16329 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16330 MaxDepth: MaxDepth - 1);
16331 }
16332 }
16333 }
16334 break;
16335
16336 case ISD::FSIN:
16337 case ISD::FCOS:
16338 case ISD::FSINCOS:
16339 return Op.getValueType().getScalarType() != MVT::f16;
16340
16341 case ISD::FMINNUM:
16342 case ISD::FMAXNUM:
16343 case ISD::FMINNUM_IEEE:
16344 case ISD::FMAXNUM_IEEE:
16345 case ISD::FMINIMUM:
16346 case ISD::FMAXIMUM:
16347 case ISD::FMINIMUMNUM:
16348 case ISD::FMAXIMUMNUM:
16349 case AMDGPUISD::CLAMP:
16350 case AMDGPUISD::FMED3:
16351 case AMDGPUISD::FMAX3:
16352 case AMDGPUISD::FMIN3:
16353 case AMDGPUISD::FMAXIMUM3:
16354 case AMDGPUISD::FMINIMUM3: {
16355 // FIXME: Shouldn't treat the generic operations different based these.
16356 // However, we aren't really required to flush the result from
16357 // minnum/maxnum..
16358
16359 // snans will be quieted, so we only need to worry about denormals.
16360 if (Subtarget->supportsMinMaxDenormModes() ||
16361 // FIXME: denormalsEnabledForType is broken for dynamic
16362 denormalsEnabledForType(DAG, VT: Op.getValueType()))
16363 return true;
16364
16365 // Flushing may be required.
16366 // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. For such
16367 // targets need to check their input recursively.
16368
16369 // FIXME: Does this apply with clamp? It's implemented with max.
16370 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
16371 if (!isCanonicalized(DAG, Op: Op.getOperand(i: I), QueryVT, UserFlags,
16372 MaxDepth: MaxDepth - 1))
16373 return false;
16374 }
16375
16376 return true;
16377 }
16378 case ISD::SELECT: {
16379 return isCanonicalized(DAG, Op: Op.getOperand(i: 1), QueryVT, UserFlags,
16380 MaxDepth: MaxDepth - 1) &&
16381 isCanonicalized(DAG, Op: Op.getOperand(i: 2), QueryVT, UserFlags,
16382 MaxDepth: MaxDepth - 1);
16383 }
16384 case ISD::BUILD_VECTOR: {
16385 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
16386 SDValue SrcOp = Op.getOperand(i);
16387 if (!isCanonicalized(DAG, Op: SrcOp, QueryVT, UserFlags, MaxDepth: MaxDepth - 1))
16388 return false;
16389 }
16390
16391 return true;
16392 }
16393 case ISD::EXTRACT_VECTOR_ELT:
16394 case ISD::EXTRACT_SUBVECTOR: {
16395 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16396 MaxDepth: MaxDepth - 1);
16397 }
16398 case ISD::INSERT_VECTOR_ELT: {
16399 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), QueryVT, UserFlags,
16400 MaxDepth: MaxDepth - 1) &&
16401 isCanonicalized(DAG, Op: Op.getOperand(i: 1), QueryVT, UserFlags,
16402 MaxDepth: MaxDepth - 1);
16403 }
16404 case ISD::POISON:
16405 return true;
16406 case ISD::UNDEF:
16407 // Could be anything.
16408 return false;
16409
16410 case ISD::BITCAST: {
16411 // Carry QueryVT through the bitcast unchanged. The top-of-function guard
16412 // rejects a source whose FP format differs from the consumed type, so a
16413 // value canonical in one FP format is not assumed canonical in another.
16414 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
16415 return isCanonicalized(DAG, Op: Src, QueryVT, UserFlags, MaxDepth: MaxDepth - 1);
16416 }
16417 case ISD::TRUNCATE: {
16418 // Hack round the mess we make when legalizing extract_vector_elt
16419 if (Op.getValueType() == MVT::i16) {
16420 SDValue TruncSrc = Op.getOperand(i: 0);
16421 if (TruncSrc.getValueType() == MVT::i32 &&
16422 TruncSrc.getOpcode() == ISD::BITCAST &&
16423 TruncSrc.getOperand(i: 0).getValueType() == MVT::v2f16) {
16424 return isCanonicalized(DAG, Op: TruncSrc.getOperand(i: 0), QueryVT, UserFlags,
16425 MaxDepth: MaxDepth - 1);
16426 }
16427 }
16428 return false;
16429 }
16430 case ISD::INTRINSIC_WO_CHAIN: {
16431 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
16432 // TODO: Handle more intrinsics
16433 switch (IntrinsicID) {
16434 case Intrinsic::amdgcn_update_dpp: {
16435 enum UpdateDPPOperand {
16436 Src = 2,
16437 RowMask = 4,
16438 BankMask = 5,
16439 BoundCtrl = 6,
16440 };
16441
16442 if (QueryVT == MVT::f32 && (VT == MVT::i32 || VT == MVT::f32) &&
16443 Op.getConstantOperandVal(i: RowMask) == 0xf && // Enable all rows.
16444 Op.getConstantOperandVal(i: BankMask) == 0xf && // Enable all banks.
16445 Op.getConstantOperandVal(i: BoundCtrl) != 0) // Zero-fill OOB lanes.
16446 return isCanonicalized(DAG, Op: Op.getOperand(i: 2), QueryVT, UserFlags,
16447 MaxDepth: MaxDepth - 1);
16448 return false;
16449 }
16450 case Intrinsic::amdgcn_cvt_pkrtz:
16451 case Intrinsic::amdgcn_cubeid:
16452 case Intrinsic::amdgcn_frexp_mant:
16453 case Intrinsic::amdgcn_fdot2:
16454 case Intrinsic::amdgcn_rcp:
16455 case Intrinsic::amdgcn_rsq:
16456 case Intrinsic::amdgcn_rsq_clamp:
16457 case Intrinsic::amdgcn_rcp_legacy:
16458 case Intrinsic::amdgcn_rsq_legacy:
16459 case Intrinsic::amdgcn_trig_preop:
16460 case Intrinsic::amdgcn_tanh:
16461 case Intrinsic::amdgcn_log:
16462 case Intrinsic::amdgcn_exp2:
16463 case Intrinsic::amdgcn_sqrt:
16464 return true;
16465 default:
16466 break;
16467 }
16468
16469 break;
16470 }
16471 default:
16472 break;
16473 }
16474
16475 // FIXME: denormalsEnabledForType is broken for dynamic
16476 return denormalsEnabledForType(DAG, VT: Op.getValueType()) &&
16477 (UserFlags.hasNoNaNs() || DAG.isKnownNeverSNaN(Op));
16478}
16479
16480bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
16481 unsigned MaxDepth) const {
16482 const MachineRegisterInfo &MRI = MF.getRegInfo();
16483 MachineInstr *MI = MRI.getVRegDef(Reg);
16484 unsigned Opcode = MI->getOpcode();
16485
16486 if (Opcode == AMDGPU::G_FCANONICALIZE)
16487 return true;
16488
16489 std::optional<FPValueAndVReg> FCR;
16490 // Constant splat (can be padded with undef) or scalar constant.
16491 if (mi_match(R: Reg, MRI, P: MIPatternMatch::m_GFCstOrSplat(FPValReg&: FCR))) {
16492 if (FCR->Value.isSignaling())
16493 return false;
16494 if (!FCR->Value.isDenormal())
16495 return true;
16496
16497 DenormalMode Mode = MF.getDenormalMode(FPType: FCR->Value.getSemantics());
16498 return Mode == DenormalMode::getIEEE();
16499 }
16500
16501 if (MaxDepth == 0)
16502 return false;
16503
16504 switch (Opcode) {
16505 case AMDGPU::G_FADD:
16506 case AMDGPU::G_FSUB:
16507 case AMDGPU::G_FMUL:
16508 case AMDGPU::G_FCEIL:
16509 case AMDGPU::G_FFLOOR:
16510 case AMDGPU::G_FRINT:
16511 case AMDGPU::G_FNEARBYINT:
16512 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
16513 case AMDGPU::G_INTRINSIC_TRUNC:
16514 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
16515 case AMDGPU::G_FMA:
16516 case AMDGPU::G_FMAD:
16517 case AMDGPU::G_FSQRT:
16518 case AMDGPU::G_FDIV:
16519 case AMDGPU::G_FREM:
16520 case AMDGPU::G_FPOW:
16521 case AMDGPU::G_FPEXT:
16522 case AMDGPU::G_FLOG:
16523 case AMDGPU::G_FLOG2:
16524 case AMDGPU::G_FLOG10:
16525 case AMDGPU::G_FPTRUNC:
16526 case AMDGPU::G_AMDGPU_RCP_IFLAG:
16527 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
16528 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
16529 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
16530 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
16531 return true;
16532 case AMDGPU::G_FNEG:
16533 case AMDGPU::G_FABS:
16534 case AMDGPU::G_FCOPYSIGN:
16535 return isCanonicalized(Reg: MI->getOperand(i: 1).getReg(), MF, MaxDepth: MaxDepth - 1);
16536 case AMDGPU::G_FMINNUM:
16537 case AMDGPU::G_FMAXNUM:
16538 case AMDGPU::G_FMINNUM_IEEE:
16539 case AMDGPU::G_FMAXNUM_IEEE:
16540 case AMDGPU::G_FMINIMUM:
16541 case AMDGPU::G_FMAXIMUM:
16542 case AMDGPU::G_FMINIMUMNUM:
16543 case AMDGPU::G_FMAXIMUMNUM: {
16544 if (Subtarget->supportsMinMaxDenormModes() ||
16545 // FIXME: denormalsEnabledForType is broken for dynamic
16546 denormalsEnabledForType(Ty: MRI.getType(Reg), MF))
16547 return true;
16548
16549 [[fallthrough]];
16550 }
16551 case AMDGPU::G_BUILD_VECTOR:
16552 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
16553 if (!isCanonicalized(Reg: MO.getReg(), MF, MaxDepth: MaxDepth - 1))
16554 return false;
16555 return true;
16556 case AMDGPU::G_INTRINSIC:
16557 case AMDGPU::G_INTRINSIC_CONVERGENT:
16558 switch (cast<GIntrinsic>(Val: MI)->getIntrinsicID()) {
16559 case Intrinsic::amdgcn_fmul_legacy:
16560 case Intrinsic::amdgcn_fmad_ftz:
16561 case Intrinsic::amdgcn_sqrt:
16562 case Intrinsic::amdgcn_fmed3:
16563 case Intrinsic::amdgcn_sin:
16564 case Intrinsic::amdgcn_cos:
16565 case Intrinsic::amdgcn_log:
16566 case Intrinsic::amdgcn_exp2:
16567 case Intrinsic::amdgcn_log_clamp:
16568 case Intrinsic::amdgcn_rcp:
16569 case Intrinsic::amdgcn_rcp_legacy:
16570 case Intrinsic::amdgcn_rsq:
16571 case Intrinsic::amdgcn_rsq_clamp:
16572 case Intrinsic::amdgcn_rsq_legacy:
16573 case Intrinsic::amdgcn_div_scale:
16574 case Intrinsic::amdgcn_div_fmas:
16575 case Intrinsic::amdgcn_div_fixup:
16576 case Intrinsic::amdgcn_fract:
16577 case Intrinsic::amdgcn_cvt_pkrtz:
16578 case Intrinsic::amdgcn_cubeid:
16579 case Intrinsic::amdgcn_cubema:
16580 case Intrinsic::amdgcn_cubesc:
16581 case Intrinsic::amdgcn_cubetc:
16582 case Intrinsic::amdgcn_frexp_mant:
16583 case Intrinsic::amdgcn_fdot2:
16584 case Intrinsic::amdgcn_trig_preop:
16585 case Intrinsic::amdgcn_tanh:
16586 return true;
16587 default:
16588 break;
16589 }
16590
16591 [[fallthrough]];
16592 default:
16593 return false;
16594 }
16595
16596 llvm_unreachable("invalid operation");
16597}
16598
16599// Constant fold canonicalize.
16600SDValue SITargetLowering::getCanonicalConstantFP(SelectionDAG &DAG,
16601 const SDLoc &SL, EVT VT,
16602 const APFloat &C) const {
16603 // Flush denormals to 0 if not enabled.
16604 if (C.isDenormal()) {
16605 DenormalMode Mode =
16606 DAG.getMachineFunction().getDenormalMode(FPType: C.getSemantics());
16607 if (Mode == DenormalMode::getPreserveSign()) {
16608 return DAG.getConstantFP(
16609 Val: APFloat::getZero(Sem: C.getSemantics(), Negative: C.isNegative()), DL: SL, VT);
16610 }
16611
16612 if (Mode != DenormalMode::getIEEE())
16613 return SDValue();
16614 }
16615
16616 if (C.isNaN()) {
16617 if (C.isSignaling()) {
16618 // Quiet a signaling NaN.
16619 return DAG.getConstantFP(Val: C.makeQuiet(), DL: SL, VT);
16620 }
16621 }
16622
16623 // Already canonical.
16624 return DAG.getConstantFP(Val: C, DL: SL, VT);
16625}
16626
16627static bool vectorEltWillFoldAway(SDValue Op) {
16628 return Op.isUndef() || isa<ConstantFPSDNode>(Val: Op);
16629}
16630
16631SDValue
16632SITargetLowering::performFCanonicalizeCombine(SDNode *N,
16633 DAGCombinerInfo &DCI) const {
16634 SelectionDAG &DAG = DCI.DAG;
16635 SDValue N0 = N->getOperand(Num: 0);
16636 EVT VT = N->getValueType(ResNo: 0);
16637
16638 // fcanonicalize undef -> qnan
16639 if (N0.isUndef()) {
16640 APFloat QNaN = APFloat::getQNaN(Sem: VT.getFltSemantics());
16641 return DAG.getConstantFP(Val: QNaN, DL: SDLoc(N), VT);
16642 }
16643
16644 if (ConstantFPSDNode *CFP = isConstOrConstSplatFP(N: N0))
16645 return getCanonicalConstantFP(DAG, SL: SDLoc(N), VT, C: CFP->getValueAPF());
16646
16647 // fcanonicalize (build_vector x, k) -> build_vector (fcanonicalize x),
16648 // (fcanonicalize k)
16649 //
16650 // fcanonicalize (build_vector x, undef) -> build_vector (fcanonicalize x), 0
16651
16652 // TODO: This could be better with wider vectors that will be split to v2f16,
16653 // and to consider uses since there aren't that many packed operations.
16654 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNumOperands() == 2 &&
16655 isTypeLegal(VT)) {
16656 SDLoc SL(N);
16657 SDValue NewElts[2];
16658 SDValue Lo = N0.getOperand(i: 0);
16659 SDValue Hi = N0.getOperand(i: 1);
16660 EVT EltVT = Lo.getValueType();
16661
16662 // Only apply this optimization if scalar canonicalize is legal for the
16663 // element type. Otherwise, scalarizing may require widening the scalar back
16664 // to a vector, adding overhead (e.g., bf16 has no scalar instructions).
16665 if (getOperationAction(Op: ISD::FCANONICALIZE, VT: EltVT) != Legal)
16666 return SDValue();
16667
16668 if (vectorEltWillFoldAway(Op: Lo) || vectorEltWillFoldAway(Op: Hi)) {
16669 for (unsigned I = 0; I != 2; ++I) {
16670 SDValue Op = N0.getOperand(i: I);
16671 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16672 NewElts[I] =
16673 getCanonicalConstantFP(DAG, SL, VT: EltVT, C: CFP->getValueAPF());
16674 } else if (Op.isUndef()) {
16675 // Handled below based on what the other operand is.
16676 NewElts[I] = Op;
16677 } else {
16678 NewElts[I] = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: SL, VT: EltVT, Operand: Op);
16679 }
16680 }
16681
16682 // If one half is undef, and one is constant, prefer a splat vector.
16683 // Otherwise, convert the undef to 0.0 since that's cheaper to use and may
16684 // be free with a packed operation.
16685 if (NewElts[0].isUndef()) {
16686 NewElts[0] = isa<ConstantFPSDNode>(Val: NewElts[1])
16687 ? NewElts[1]
16688 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16689 }
16690
16691 if (NewElts[1].isUndef()) {
16692 NewElts[1] = isa<ConstantFPSDNode>(Val: NewElts[0])
16693 ? NewElts[0]
16694 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16695 }
16696
16697 return DAG.getBuildVector(VT, DL: SL, Ops: NewElts);
16698 }
16699 }
16700
16701 return SDValue();
16702}
16703
16704static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
16705 switch (Opc) {
16706 case ISD::FMAXNUM:
16707 case ISD::FMAXNUM_IEEE:
16708 case ISD::FMAXIMUMNUM:
16709 return AMDGPUISD::FMAX3;
16710 case ISD::FMAXIMUM:
16711 return AMDGPUISD::FMAXIMUM3;
16712 case ISD::SMAX:
16713 return AMDGPUISD::SMAX3;
16714 case ISD::UMAX:
16715 return AMDGPUISD::UMAX3;
16716 case ISD::FMINNUM:
16717 case ISD::FMINNUM_IEEE:
16718 case ISD::FMINIMUMNUM:
16719 return AMDGPUISD::FMIN3;
16720 case ISD::FMINIMUM:
16721 return AMDGPUISD::FMINIMUM3;
16722 case ISD::SMIN:
16723 return AMDGPUISD::SMIN3;
16724 case ISD::UMIN:
16725 return AMDGPUISD::UMIN3;
16726 default:
16727 llvm_unreachable("Not a min/max opcode");
16728 }
16729}
16730
16731SDValue SITargetLowering::performIntMed3ImmCombine(SelectionDAG &DAG,
16732 const SDLoc &SL, SDValue Src,
16733 SDValue MinVal,
16734 SDValue MaxVal,
16735 bool Signed) const {
16736
16737 // med3 comes from
16738 // min(max(x, K0), K1), K0 < K1
16739 // max(min(x, K0), K1), K1 < K0
16740 //
16741 // "MinVal" and "MaxVal" respectively refer to the rhs of the
16742 // min/max op.
16743 ConstantSDNode *MinK = dyn_cast<ConstantSDNode>(Val&: MinVal);
16744 ConstantSDNode *MaxK = dyn_cast<ConstantSDNode>(Val&: MaxVal);
16745
16746 if (!MinK || !MaxK)
16747 return SDValue();
16748
16749 if (Signed) {
16750 if (MaxK->getAPIntValue().sge(RHS: MinK->getAPIntValue()))
16751 return SDValue();
16752 } else {
16753 if (MaxK->getAPIntValue().uge(RHS: MinK->getAPIntValue()))
16754 return SDValue();
16755 }
16756
16757 EVT VT = MinK->getValueType(ResNo: 0);
16758 unsigned Med3Opc = Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3;
16759 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16()))
16760 return DAG.getNode(Opcode: Med3Opc, DL: SL, VT, N1: Src, N2: MaxVal, N3: MinVal);
16761
16762 // Note: we could also extend to i32 and use i32 med3 if i16 med3 is
16763 // not available, but this is unlikely to be profitable as constants
16764 // will often need to be materialized & extended, especially on
16765 // pre-GFX10 where VOP3 instructions couldn't take literal operands.
16766 return SDValue();
16767}
16768
16769static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
16770 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op))
16771 return C;
16772
16773 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
16774 if (ConstantFPSDNode *C = BV->getConstantFPSplatNode())
16775 return C;
16776 }
16777
16778 return nullptr;
16779}
16780
16781SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
16782 const SDLoc &SL, SDValue Op0,
16783 SDValue Op1,
16784 bool IsKnownNoNaNs) const {
16785 ConstantFPSDNode *K1 = getSplatConstantFP(Op: Op1);
16786 if (!K1)
16787 return SDValue();
16788
16789 ConstantFPSDNode *K0 = getSplatConstantFP(Op: Op0.getOperand(i: 1));
16790 if (!K0)
16791 return SDValue();
16792
16793 // Ordered >= (although NaN inputs should have folded away by now).
16794 if (K0->getValueAPF() > K1->getValueAPF())
16795 return SDValue();
16796
16797 // med3 with a nan input acts like
16798 // v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)
16799 //
16800 // So the result depends on whether the IEEE mode bit is enabled or not with a
16801 // signaling nan input.
16802 // ieee=1
16803 // s0 snan: yields s2
16804 // s1 snan: yields s2
16805 // s2 snan: qnan
16806
16807 // s0 qnan: min(s1, s2)
16808 // s1 qnan: min(s0, s2)
16809 // s2 qnan: min(s0, s1)
16810
16811 // ieee=0
16812 // s0 snan: min(s1, s2)
16813 // s1 snan: min(s0, s2)
16814 // s2 snan: qnan
16815
16816 // s0 qnan: min(s1, s2)
16817 // s1 qnan: min(s0, s2)
16818 // s2 qnan: min(s0, s1)
16819 const MachineFunction &MF = DAG.getMachineFunction();
16820 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16821
16822 // TODO: Check IEEE bit enabled. We can form fmed3 with IEEE=0 regardless of
16823 // whether the input is a signaling nan if op0 is fmaximum or fmaximumnum. We
16824 // can only form if op0 is fmaxnum_ieee if IEEE=1.
16825 EVT VT = Op0.getValueType();
16826 if (Info->getMode().DX10Clamp) {
16827 // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the
16828 // hardware fmed3 behavior converting to a min.
16829 // FIXME: Should this be allowing -0.0?
16830 if (K1->isOne() && K0->isPosZero())
16831 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Op0.getOperand(i: 0));
16832 }
16833
16834 // med3 for f16 is only available on gfx9+, and not available for v2f16.
16835 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) {
16836 // This isn't safe with signaling NaNs because in IEEE mode, min/max on a
16837 // signaling NaN gives a quiet NaN. The quiet NaN input to the min would
16838 // then give the other result, which is different from med3 with a NaN
16839 // input.
16840 SDValue Var = Op0.getOperand(i: 0);
16841 if (!IsKnownNoNaNs && !DAG.isKnownNeverSNaN(Op: Var))
16842 return SDValue();
16843
16844 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
16845
16846 if ((!K0->hasOneUse() || TII->isInlineConstant(Imm: K0->getValueAPF())) &&
16847 (!K1->hasOneUse() || TII->isInlineConstant(Imm: K1->getValueAPF()))) {
16848 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT: K0->getValueType(ResNo: 0), N1: Var,
16849 N2: SDValue(K0, 0), N3: SDValue(K1, 0));
16850 }
16851 }
16852
16853 return SDValue();
16854}
16855
16856/// \return true if the subtarget supports minimum3 and maximum3 with the given
16857/// base min/max opcode \p Opc for type \p VT.
16858static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc,
16859 EVT VT) {
16860 switch (Opc) {
16861 case ISD::FMINNUM:
16862 case ISD::FMAXNUM:
16863 case ISD::FMINNUM_IEEE:
16864 case ISD::FMAXNUM_IEEE:
16865 case ISD::FMINIMUMNUM:
16866 case ISD::FMAXIMUMNUM:
16867 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()) ||
16868 (VT == MVT::v2f16 && Subtarget.hasMin3Max3PKF16());
16869 case ISD::FMINIMUM:
16870 case ISD::FMAXIMUM:
16871 return (VT == MVT::f32 && Subtarget.hasMinimum3Maximum3F32()) ||
16872 (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()) ||
16873 (VT == MVT::v2f16 && Subtarget.hasMinimum3Maximum3PKF16());
16874 case ISD::SMAX:
16875 case ISD::SMIN:
16876 case ISD::UMAX:
16877 case ISD::UMIN:
16878 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.hasMin3Max3_16());
16879 default:
16880 return false;
16881 }
16882
16883 llvm_unreachable("not a min/max opcode");
16884}
16885
16886SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
16887 DAGCombinerInfo &DCI) const {
16888 SelectionDAG &DAG = DCI.DAG;
16889
16890 EVT VT = N->getValueType(ResNo: 0);
16891 unsigned Opc = N->getOpcode();
16892 SDValue Op0 = N->getOperand(Num: 0);
16893 SDValue Op1 = N->getOperand(Num: 1);
16894
16895 // Only do this if the inner op has one use since this will just increases
16896 // register pressure for no benefit.
16897
16898 if (supportsMin3Max3(Subtarget: *Subtarget, Opc, VT)) {
16899 auto IsTreeWithCombinableChildren = [Opc](SDValue Op) {
16900 return (Op.getOperand(i: 0).getOpcode() == Opc &&
16901 Op.getOperand(i: 0).hasOneUse()) ||
16902 (Op.getOperand(i: 1).getOpcode() == Opc &&
16903 Op.getOperand(i: 1).hasOneUse());
16904 };
16905
16906 bool CanTreeCombineApply = Op0.getOpcode() == Opc && Op0.hasOneUse() &&
16907 Op1.getOpcode() == Opc && Op1.hasOneUse();
16908 bool HasCombinableTreeChild =
16909 CanTreeCombineApply && (IsTreeWithCombinableChildren(Op0) ||
16910 IsTreeWithCombinableChildren(Op1));
16911
16912 // Tree reduction: when both operands are the same min/max op, restructure
16913 // to keep a 2-op node on top so higher tree levels can still combine.
16914 //
16915 // max(max(a, b), max(c, d)) -> max(max3(a, b, c), d)
16916 // min(min(a, b), min(c, d)) -> min(min3(a, b, c), d)
16917 //
16918 // Defer when either inner op is a tree node with combinable children.
16919 if (CanTreeCombineApply && !HasCombinableTreeChild) {
16920 SDLoc DL(N);
16921 SDValue Inner =
16922 DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT, N1: Op0.getOperand(i: 0),
16923 N2: Op0.getOperand(i: 1), N3: Op1.getOperand(i: 0));
16924 return DAG.getNode(Opcode: Opc, DL, VT, N1: Inner, N2: Op1.getOperand(i: 1));
16925 }
16926
16927 // max(max(a, b), c) -> max3(a, b, c)
16928 // min(min(a, b), c) -> min3(a, b, c)
16929 // Deferred when Op0 is a tree node with combinable children.
16930 if (Op0.getOpcode() == Opc && Op0.hasOneUse() && !HasCombinableTreeChild) {
16931 SDLoc DL(N);
16932 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16933 N1: Op0.getOperand(i: 0), N2: Op0.getOperand(i: 1), N3: Op1);
16934 }
16935
16936 // Try commuted.
16937 // max(a, max(b, c)) -> max3(a, b, c)
16938 // min(a, min(b, c)) -> min3(a, b, c)
16939 // Deferred when Op1 is a tree node with combinable children.
16940 if (Op1.getOpcode() == Opc && Op1.hasOneUse() && !HasCombinableTreeChild) {
16941 SDLoc DL(N);
16942 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16943 N1: Op0, N2: Op1.getOperand(i: 0), N3: Op1.getOperand(i: 1));
16944 }
16945 }
16946
16947 // umin(sffbh(x), bitwidth) -> sffbh(x) if x is known to be not 0 or -1.
16948 SDValue FfbhSrc;
16949 uint64_t Clamp = 0;
16950 if (Opc == ISD::UMIN &&
16951 sd_match(N: Op0,
16952 P: m_IntrinsicWOChain<Intrinsic::amdgcn_sffbh>(Opnds: m_Value(N&: FfbhSrc))) &&
16953 sd_match(N: Op1, P: m_ConstInt(V&: Clamp))) {
16954 unsigned BitWidth = FfbhSrc.getValueType().getScalarSizeInBits();
16955 if (Clamp >= BitWidth) {
16956 KnownBits Known = DAG.computeKnownBits(Op: FfbhSrc);
16957 if (Known.isNonZero() && Known.Zero.getBoolValue())
16958 return Op0;
16959 }
16960 }
16961
16962 // min(max(x, K0), K1), K0 < K1 -> med3(x, K0, K1)
16963 // max(min(x, K0), K1), K1 < K0 -> med3(x, K1, K0)
16964 if (Opc == ISD::SMIN && Op0.getOpcode() == ISD::SMAX && Op0.hasOneUse()) {
16965 if (SDValue Med3 = performIntMed3ImmCombine(
16966 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: true))
16967 return Med3;
16968 }
16969 if (Opc == ISD::SMAX && Op0.getOpcode() == ISD::SMIN && Op0.hasOneUse()) {
16970 if (SDValue Med3 = performIntMed3ImmCombine(
16971 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: true))
16972 return Med3;
16973 }
16974
16975 if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
16976 if (SDValue Med3 = performIntMed3ImmCombine(
16977 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: false))
16978 return Med3;
16979 }
16980 if (Opc == ISD::UMAX && Op0.getOpcode() == ISD::UMIN && Op0.hasOneUse()) {
16981 if (SDValue Med3 = performIntMed3ImmCombine(
16982 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: false))
16983 return Med3;
16984 }
16985
16986 // if !is_snan(x):
16987 // fminnum(fmaxnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16988 // fminnum_ieee(fmaxnum_ieee(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16989 // fminnumnum(fmaxnumnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16990 // fmin_legacy(fmax_legacy(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16991 if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) ||
16992 (Opc == ISD::FMINNUM_IEEE && Op0.getOpcode() == ISD::FMAXNUM_IEEE) ||
16993 (Opc == ISD::FMINIMUMNUM && Op0.getOpcode() == ISD::FMAXIMUMNUM) ||
16994 (Opc == AMDGPUISD::FMIN_LEGACY &&
16995 Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
16996 (VT == MVT::f32 || VT == MVT::f64 ||
16997 (VT == MVT::f16 && Subtarget->has16BitInsts()) ||
16998 (VT == MVT::bf16 && Subtarget->hasBF16PackedInsts()) ||
16999 (VT == MVT::v2bf16 && Subtarget->hasBF16PackedInsts()) ||
17000 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) &&
17001 Op0.hasOneUse()) {
17002 if (SDValue Res = performFPMed3ImmCombine(DAG, SL: SDLoc(N), Op0, Op1,
17003 IsKnownNoNaNs: N->getFlags().hasNoNaNs()))
17004 return Res;
17005 }
17006
17007 // Prefer fminnum_ieee over fminimum. For gfx950, minimum/maximum are legal
17008 // for some types, but at a higher cost since it's implemented with a 3
17009 // operand form.
17010 const SDNodeFlags Flags = N->getFlags();
17011 if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) && Flags.hasNoNaNs() &&
17012 !Subtarget->hasIEEEMinimumMaximumInsts() &&
17013 isOperationLegal(Op: ISD::FMINNUM_IEEE, VT: VT.getScalarType())) {
17014 unsigned NewOpc =
17015 Opc == ISD::FMINIMUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
17016 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(N), VT, N1: Op0, N2: Op1, Flags);
17017 }
17018
17019 return SDValue();
17020}
17021
17022static bool isClampZeroToOne(SDValue A, SDValue B) {
17023 if (ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A)) {
17024 if (ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B)) {
17025 // FIXME: Should this be allowing -0.0?
17026 return (CA->isPosZero() && CB->isOne()) ||
17027 (CA->isOne() && CB->isPosZero());
17028 }
17029 }
17030
17031 return false;
17032}
17033
17034// FIXME: Should only worry about snans for version with chain.
17035SDValue SITargetLowering::performFMed3Combine(SDNode *N,
17036 DAGCombinerInfo &DCI) const {
17037 EVT VT = N->getValueType(ResNo: 0);
17038 // v_med3_f32 and v_max_f32 behave identically wrt denorms, exceptions and
17039 // NaNs. With a NaN input, the order of the operands may change the result.
17040
17041 SelectionDAG &DAG = DCI.DAG;
17042 SDLoc SL(N);
17043
17044 SDValue Src0 = N->getOperand(Num: 0);
17045 SDValue Src1 = N->getOperand(Num: 1);
17046 SDValue Src2 = N->getOperand(Num: 2);
17047
17048 if (isClampZeroToOne(A: Src0, B: Src1)) {
17049 // const_a, const_b, x -> clamp is safe in all cases including signaling
17050 // nans.
17051 // FIXME: Should this be allowing -0.0?
17052 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src2);
17053 }
17054
17055 const MachineFunction &MF = DAG.getMachineFunction();
17056 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
17057
17058 // FIXME: dx10_clamp behavior assumed in instcombine. Should we really bother
17059 // handling no dx10-clamp?
17060 if (Info->getMode().DX10Clamp) {
17061 // If NaNs is clamped to 0, we are free to reorder the inputs.
17062
17063 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
17064 std::swap(a&: Src0, b&: Src1);
17065
17066 if (isa<ConstantFPSDNode>(Val: Src1) && !isa<ConstantFPSDNode>(Val: Src2))
17067 std::swap(a&: Src1, b&: Src2);
17068
17069 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
17070 std::swap(a&: Src0, b&: Src1);
17071
17072 if (isClampZeroToOne(A: Src1, B: Src2))
17073 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src0);
17074 }
17075
17076 return SDValue();
17077}
17078
17079SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
17080 DAGCombinerInfo &DCI) const {
17081 SDValue Src0 = N->getOperand(Num: 0);
17082 SDValue Src1 = N->getOperand(Num: 1);
17083 if (Src0.isUndef() && Src1.isUndef())
17084 return DCI.DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
17085 return SDValue();
17086}
17087
17088/// V_CVT_SCALEF32_* conversions to a narrow format only read the exponent
17089/// field of their f32 scale operand.
17090SDValue
17091SITargetLowering::performExponentOnlyScaleCombine(SDNode *N,
17092 DAGCombinerInfo &DCI) const {
17093 std::optional<unsigned> ScaleArgIdx =
17094 AMDGPU::getExponentOnlyScaleArgIdx(IntrID: N->getConstantOperandVal(Num: 0));
17095 if (!ScaleArgIdx)
17096 return SDValue();
17097
17098 // Operand 0 is the intrinsic ID.
17099 unsigned OpIdx = *ScaleArgIdx + 1;
17100 SelectionDAG &DAG = DCI.DAG;
17101 SDValue Scale = N->getOperand(Num: OpIdx);
17102 APInt Demanded = AMDGPU::getExponentOnlyScaleDemandedBits();
17103
17104 if (SDValue DemandedScale =
17105 SimplifyMultipleUseDemandedBits(Op: Scale, DemandedBits: Demanded, DAG)) {
17106 SmallVector<SDValue> Ops(N->ops());
17107 Ops[OpIdx] = DemandedScale;
17108 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SDLoc(N), VTList: N->getVTList(), Ops);
17109 }
17110
17111 if (SimplifyDemandedBits(Op: Scale, DemandedBits: Demanded, DCI))
17112 return SDValue(N, 0);
17113
17114 return SDValue();
17115}
17116
17117// Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be
17118// expanded into a set of cmp/select instructions.
17119bool SITargetLowering::shouldExpandVectorDynExt(unsigned EltSize,
17120 unsigned NumElem,
17121 bool IsDivergentIdx,
17122 const GCNSubtarget *Subtarget) {
17123 if (Subtarget->getCLOpts().use_divergent_register_indexing)
17124 return false;
17125
17126 unsigned VecSize = EltSize * NumElem;
17127
17128 // Sub-dword vectors of size 2 dword or less have better implementation.
17129 if (VecSize <= 64 && EltSize < 32)
17130 return false;
17131
17132 // Always expand the rest of sub-dword instructions, otherwise it will be
17133 // lowered via memory.
17134 if (EltSize < 32)
17135 return true;
17136
17137 // Always do this if var-idx is divergent, otherwise it will become a loop.
17138 if (IsDivergentIdx)
17139 return true;
17140
17141 // Large vectors would yield too many compares and v_cndmask_b32 instructions.
17142 unsigned NumInsts = NumElem /* Number of compares */ +
17143 ((EltSize + 31) / 32) * NumElem /* Number of cndmasks */;
17144
17145 // On some architectures (GFX9) movrel is not available and it's better
17146 // to expand.
17147 if (Subtarget->useVGPRIndexMode())
17148 return NumInsts <= 16;
17149
17150 // If movrel is available, use it instead of expanding for vector of 8
17151 // elements.
17152 if (Subtarget->hasMovrel())
17153 return NumInsts <= 15;
17154
17155 return true;
17156}
17157
17158bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
17159 SDValue Idx = N->getOperand(Num: N->getNumOperands() - 1);
17160 if (isa<ConstantSDNode>(Val: Idx))
17161 return false;
17162
17163 SDValue Vec = N->getOperand(Num: 0);
17164 EVT VecVT = Vec.getValueType();
17165 EVT EltVT = VecVT.getVectorElementType();
17166 unsigned EltSize = EltVT.getSizeInBits();
17167 unsigned NumElem = VecVT.getVectorNumElements();
17168
17169 return SITargetLowering::shouldExpandVectorDynExt(
17170 EltSize, NumElem, IsDivergentIdx: Idx->isDivergent(), Subtarget: getSubtarget());
17171}
17172
17173SDValue
17174SITargetLowering::performExtractVectorEltCombine(SDNode *N,
17175 DAGCombinerInfo &DCI) const {
17176 SDValue Vec = N->getOperand(Num: 0);
17177 SelectionDAG &DAG = DCI.DAG;
17178
17179 EVT VecVT = Vec.getValueType();
17180 EVT VecEltVT = VecVT.getVectorElementType();
17181 EVT ResVT = N->getValueType(ResNo: 0);
17182
17183 unsigned VecSize = VecVT.getSizeInBits();
17184 unsigned VecEltSize = VecEltVT.getSizeInBits();
17185
17186 if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
17187 allUsesHaveSourceMods(N)) {
17188 SDLoc SL(N);
17189 SDValue Idx = N->getOperand(Num: 1);
17190 SDValue Elt =
17191 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec.getOperand(i: 0), N2: Idx);
17192 return DAG.getNode(Opcode: Vec.getOpcode(), DL: SL, VT: ResVT, Operand: Elt);
17193 }
17194
17195 // (extract_vector_element (and {y0, y1}, (build_vector 0x1f, 0x1f)), index)
17196 // -> (and (extract_vector_element {y0, y1}, index), 0x1f)
17197 // There are optimisations to transform 64-bit shifts into 32-bit shifts
17198 // depending on the shift operand. See e.g. performSraCombine().
17199 // This combine ensures that the optimisation is compatible with v2i32
17200 // legalised AND.
17201 if (VecVT == MVT::v2i32 && Vec->getOpcode() == ISD::AND &&
17202 Vec->getOperand(Num: 1)->getOpcode() == ISD::BUILD_VECTOR) {
17203
17204 const ConstantSDNode *C = isConstOrConstSplat(N: Vec.getOperand(i: 1));
17205 if (!C || C->getZExtValue() != 0x1f)
17206 return SDValue();
17207
17208 SDLoc SL(N);
17209 SDValue AndMask = DAG.getConstant(Val: 0x1f, DL: SL, VT: MVT::i32);
17210 SDValue EVE = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32,
17211 N1: Vec->getOperand(Num: 0), N2: N->getOperand(Num: 1));
17212 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: EVE, N2: AndMask);
17213 DAG.ReplaceAllUsesWith(From: N, To: A.getNode());
17214 }
17215
17216 // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
17217 // =>
17218 // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
17219 // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
17220 // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
17221 if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
17222 SDLoc SL(N);
17223 SDValue Idx = N->getOperand(Num: 1);
17224 unsigned Opc = Vec.getOpcode();
17225
17226 switch (Opc) {
17227 default:
17228 break;
17229 // TODO: Support other binary operations.
17230 case ISD::FADD:
17231 case ISD::FSUB:
17232 case ISD::FMUL:
17233 case ISD::ADD:
17234 case ISD::UMIN:
17235 case ISD::UMAX:
17236 case ISD::SMIN:
17237 case ISD::SMAX:
17238 case ISD::FMAXNUM:
17239 case ISD::FMINNUM:
17240 case ISD::FMAXNUM_IEEE:
17241 case ISD::FMINNUM_IEEE:
17242 case ISD::FMAXIMUM:
17243 case ISD::FMINIMUM: {
17244 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
17245 N1: Vec.getOperand(i: 0), N2: Idx);
17246 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
17247 N1: Vec.getOperand(i: 1), N2: Idx);
17248
17249 DCI.AddToWorklist(N: Elt0.getNode());
17250 DCI.AddToWorklist(N: Elt1.getNode());
17251 return DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT, N1: Elt0, N2: Elt1, Flags: Vec->getFlags());
17252 }
17253 }
17254 }
17255
17256 // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
17257 if (shouldExpandVectorDynExt(N)) {
17258 SDLoc SL(N);
17259 SDValue Idx = N->getOperand(Num: 1);
17260 SDValue V;
17261 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17262 SDValue IC = DAG.getVectorIdxConstant(Val: I, DL: SL);
17263 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec, N2: IC);
17264 if (I == 0)
17265 V = Elt;
17266 else
17267 V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Elt, False: V, Cond: ISD::SETEQ);
17268 }
17269 return V;
17270 }
17271
17272 // EXTRACT_VECTOR_ELT (v2i32 bitcast (i64/f64:k), Idx)
17273 // =>
17274 // i32:Lo(k) if Idx == 0, or
17275 // i32:Hi(k) if Idx == 1
17276 auto *Idx = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17277 if (Vec.getOpcode() == ISD::BITCAST && VecVT == MVT::v2i32 && Idx) {
17278 SDLoc SL(N);
17279 SDValue PeekThrough = Vec.getOperand(i: 0);
17280 auto *KImm = dyn_cast<ConstantSDNode>(Val&: PeekThrough);
17281 if (KImm && KImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
17282 uint64_t KImmValue = KImm->getZExtValue();
17283 return DAG.getConstant(
17284 Val: (KImmValue >> (32 * Idx->getZExtValue())) & 0xffffffff, DL: SL, VT: MVT::i32);
17285 }
17286 auto *KFPImm = dyn_cast<ConstantFPSDNode>(Val&: PeekThrough);
17287 if (KFPImm && KFPImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
17288 uint64_t KFPImmValue =
17289 KFPImm->getValueAPF().bitcastToAPInt().getZExtValue();
17290 return DAG.getConstant(Val: (KFPImmValue >> (32 * Idx->getZExtValue())) &
17291 0xffffffff,
17292 DL: SL, VT: MVT::i32);
17293 }
17294 }
17295
17296 if (!DCI.isBeforeLegalize())
17297 return SDValue();
17298
17299 // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
17300 // elements. This exposes more load reduction opportunities by replacing
17301 // multiple small extract_vector_elements with a single 32-bit extract.
17302 if (isa<MemSDNode>(Val: Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
17303 VecSize > 32 && VecSize % 32 == 0 && Idx) {
17304 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: VecVT);
17305
17306 unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
17307 unsigned EltIdx = BitIndex / 32;
17308 unsigned LeftoverBitIdx = BitIndex % 32;
17309 SDLoc SL(N);
17310
17311 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: Vec);
17312 DCI.AddToWorklist(N: Cast.getNode());
17313
17314 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
17315 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
17316 DCI.AddToWorklist(N: Elt.getNode());
17317 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Elt,
17318 N2: DAG.getConstant(Val: LeftoverBitIdx, DL: SL, VT: MVT::i32));
17319 DCI.AddToWorklist(N: Srl.getNode());
17320
17321 EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
17322 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: VecEltAsIntVT, Operand: Srl);
17323 DCI.AddToWorklist(N: Trunc.getNode());
17324
17325 if (VecEltVT == ResVT) {
17326 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecEltVT, Operand: Trunc);
17327 }
17328
17329 assert(ResVT.isScalarInteger());
17330 return DAG.getAnyExtOrTrunc(Op: Trunc, DL: SL, VT: ResVT);
17331 }
17332
17333 return SDValue();
17334}
17335
17336SDValue
17337SITargetLowering::performInsertVectorEltCombine(SDNode *N,
17338 DAGCombinerInfo &DCI) const {
17339 SDValue Vec = N->getOperand(Num: 0);
17340 SDValue Idx = N->getOperand(Num: 2);
17341 EVT VecVT = Vec.getValueType();
17342 EVT EltVT = VecVT.getVectorElementType();
17343
17344 // INSERT_VECTOR_ELT (<n x e>, var-idx)
17345 // => BUILD_VECTOR n x select (e, const-idx)
17346 // 16-bit elements are left to lowerINSERT_VECTOR_ELT, which is cheaper.
17347 if (!shouldExpandVectorDynExt(N) ||
17348 (EltVT.getSizeInBits() == 16 &&
17349 isOperationCustom(Op: ISD::INSERT_VECTOR_ELT, VT: VecVT)))
17350 return SDValue();
17351
17352 SelectionDAG &DAG = DCI.DAG;
17353 SDLoc SL(N);
17354 SDValue Ins = N->getOperand(Num: 1);
17355 EVT IdxVT = Idx.getValueType();
17356
17357 SmallVector<SDValue, 16> Ops;
17358 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
17359 SDValue IC = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
17360 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec, N2: IC);
17361 SDValue V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Ins, False: Elt, Cond: ISD::SETEQ);
17362 Ops.push_back(Elt: V);
17363 }
17364
17365 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops);
17366}
17367
17368/// Return the source of an fp_extend from f16 to f32, or a converted FP
17369/// constant.
17370static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src) {
17371 if (Src.getOpcode() == ISD::FP_EXTEND &&
17372 Src.getOperand(i: 0).getValueType() == MVT::f16) {
17373 return Src.getOperand(i: 0);
17374 }
17375
17376 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
17377 APFloat Val = CFP->getValueAPF();
17378 bool LosesInfo = true;
17379 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
17380 if (!LosesInfo)
17381 return DAG.getConstantFP(Val, DL: SDLoc(Src), VT: MVT::f16);
17382 }
17383
17384 return SDValue();
17385}
17386
17387SDValue SITargetLowering::performFPRoundCombine(SDNode *N,
17388 DAGCombinerInfo &DCI) const {
17389 assert(Subtarget->has16BitInsts() && !Subtarget->hasMed3_16() &&
17390 "combine only useful on gfx8");
17391
17392 SDValue TruncSrc = N->getOperand(Num: 0);
17393 EVT VT = N->getValueType(ResNo: 0);
17394 if (VT != MVT::f16)
17395 return SDValue();
17396
17397 if (TruncSrc.getOpcode() != AMDGPUISD::FMED3 ||
17398 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse())
17399 return SDValue();
17400
17401 SelectionDAG &DAG = DCI.DAG;
17402 SDLoc SL(N);
17403
17404 // Optimize f16 fmed3 pattern performed on f32. On gfx8 there is no f16 fmed3,
17405 // and expanding it with min/max saves 1 instruction vs. casting to f32 and
17406 // casting back.
17407
17408 // fptrunc (f32 (fmed3 (fpext f16:a, fpext f16:b, fpext f16:c))) =>
17409 // fmin(fmax(a, b), fmax(fmin(a, b), c))
17410 SDValue A = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 0));
17411 if (!A)
17412 return SDValue();
17413
17414 SDValue B = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 1));
17415 if (!B)
17416 return SDValue();
17417
17418 SDValue C = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 2));
17419 if (!C)
17420 return SDValue();
17421
17422 // This changes signaling nan behavior. If an input is a signaling nan, it
17423 // would have been quieted by the fpext originally. We don't care because
17424 // these are unconstrained ops. If we needed to insert quieting canonicalizes
17425 // we would be worse off than just doing the promotion.
17426 SDValue A1 = DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: A, N2: B);
17427 SDValue B1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A, N2: B);
17428 SDValue C1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A1, N2: C);
17429 return DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: B1, N2: C1);
17430}
17431
17432unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG,
17433 const SDNode *N0,
17434 const SDNode *N1) const {
17435 EVT VT = N0->getValueType(ResNo: 0);
17436
17437 // Only do this if we are not trying to support denormals. v_mad_f32 does not
17438 // support denormals ever.
17439 if (((VT == MVT::f32 &&
17440 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction())) ||
17441 (VT == MVT::f16 && Subtarget->hasMadF16() &&
17442 denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction()))) &&
17443 isOperationLegal(Op: ISD::FMAD, VT))
17444 return ISD::FMAD;
17445
17446 if (N0->getFlags().hasAllowContract() && N1->getFlags().hasAllowContract() &&
17447 isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
17448 return ISD::FMA;
17449 }
17450
17451 return 0;
17452}
17453
17454// For a reassociatable opcode perform:
17455// op x, (op y, z) -> op (op x, z), y, if x and z are uniform
17456SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
17457 SelectionDAG &DAG) const {
17458 EVT VT = N->getValueType(ResNo: 0);
17459 if (VT != MVT::i32 && VT != MVT::i64)
17460 return SDValue();
17461
17462 if (DAG.isBaseWithConstantOffset(Op: SDValue(N, 0)))
17463 return SDValue();
17464
17465 unsigned Opc = N->getOpcode();
17466 SDValue Op0 = N->getOperand(Num: 0);
17467 SDValue Op1 = N->getOperand(Num: 1);
17468
17469 if (!(Op0->isDivergent() ^ Op1->isDivergent()))
17470 return SDValue();
17471
17472 if (Op0->isDivergent())
17473 std::swap(a&: Op0, b&: Op1);
17474
17475 if (Op1.getOpcode() != Opc || !Op1.hasOneUse())
17476 return SDValue();
17477
17478 SDValue Op2 = Op1.getOperand(i: 1);
17479 Op1 = Op1.getOperand(i: 0);
17480 if (!(Op1->isDivergent() ^ Op2->isDivergent()))
17481 return SDValue();
17482
17483 if (Op1->isDivergent())
17484 std::swap(a&: Op1, b&: Op2);
17485
17486 SDLoc SL(N);
17487 SDValue Add1 = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Op0, N2: Op1);
17488 return DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Add1, N2: Op2);
17489}
17490
17491static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
17492 SDValue N0, SDValue N1, SDValue N2, bool Signed) {
17493 unsigned MadOpc = Signed ? AMDGPUISD::MAD_I64_I32 : AMDGPUISD::MAD_U64_U32;
17494 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i1);
17495 SDValue Mad = DAG.getNode(Opcode: MadOpc, DL: SL, VTList: VTs, N1: N0, N2: N1, N3: N2);
17496 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Mad);
17497}
17498
17499// Fold
17500// y = lshr i64 x, 32
17501// res = add (mul i64 y, Const), x where "Const" is a 64-bit constant
17502// with Const.hi == -1
17503// To
17504// res = mad_u64_u32 y.lo ,Const.lo, x.lo
17505static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
17506 SDValue MulLHS, SDValue MulRHS,
17507 SDValue AddRHS) {
17508 if (MulRHS.getOpcode() == ISD::SRL)
17509 std::swap(a&: MulLHS, b&: MulRHS);
17510
17511 if (MulLHS.getValueType() != MVT::i64 || MulLHS.getOpcode() != ISD::SRL)
17512 return SDValue();
17513
17514 ConstantSDNode *ShiftVal = dyn_cast<ConstantSDNode>(Val: MulLHS.getOperand(i: 1));
17515 if (!ShiftVal || ShiftVal->getAsZExtVal() != 32 ||
17516 MulLHS.getOperand(i: 0) != AddRHS)
17517 return SDValue();
17518
17519 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: MulRHS.getNode());
17520 if (!Const || Hi_32(Value: Const->getZExtValue()) != uint32_t(-1))
17521 return SDValue();
17522
17523 SDValue ConstMul =
17524 DAG.getConstant(Val: Lo_32(Value: Const->getZExtValue()), DL: SL, VT: MVT::i32);
17525 return getMad64_32(DAG, SL, VT: MVT::i64,
17526 N0: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS), N1: ConstMul,
17527 N2: DAG.getZeroExtendInReg(Op: AddRHS, DL: SL, VT: MVT::i32), Signed: false);
17528}
17529
17530// Fold (add (mul x, y), z) --> (mad_[iu]64_[iu]32 x, y, z) plus high
17531// multiplies, if any.
17532//
17533// Full 64-bit multiplies that feed into an addition are lowered here instead
17534// of using the generic expansion. The generic expansion ends up with
17535// a tree of ADD nodes that prevents us from using the "add" part of the
17536// MAD instruction. The expansion produced here results in a chain of ADDs
17537// instead of a tree.
17538SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
17539 DAGCombinerInfo &DCI) const {
17540 assert(N->isAnyAdd());
17541
17542 SelectionDAG &DAG = DCI.DAG;
17543 EVT VT = N->getValueType(ResNo: 0);
17544 SDLoc SL(N);
17545 SDValue LHS = N->getOperand(Num: 0);
17546 SDValue RHS = N->getOperand(Num: 1);
17547
17548 if (VT.isVector())
17549 return SDValue();
17550
17551 // S_MUL_HI_[IU]32 was added in gfx9, which allows us to keep the overall
17552 // result in scalar registers for uniform values.
17553 if (!N->isDivergent() && Subtarget->hasSMulHi())
17554 return SDValue();
17555
17556 unsigned NumBits = VT.getScalarSizeInBits();
17557 if (NumBits <= 32 || NumBits > 64)
17558 return SDValue();
17559
17560 if (LHS.getOpcode() != ISD::MUL) {
17561 assert(RHS.getOpcode() == ISD::MUL);
17562 std::swap(a&: LHS, b&: RHS);
17563 }
17564
17565 // Avoid the fold if it would unduly increase the number of multiplies due to
17566 // multiple uses, except on hardware with full-rate multiply-add (which is
17567 // part of full-rate 64-bit ops).
17568 if (!Subtarget->hasFullRate64Ops()) {
17569 unsigned NumUsers = 0;
17570 for (SDNode *User : LHS->users()) {
17571 // There is a use that does not feed into addition, so the multiply can't
17572 // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
17573 if (!User->isAnyAdd())
17574 return SDValue();
17575
17576 // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
17577 // MUL + 3xADD + 3xADDC over 3xMAD.
17578 ++NumUsers;
17579 if (NumUsers >= 3)
17580 return SDValue();
17581 }
17582 }
17583
17584 SDValue MulLHS = LHS.getOperand(i: 0);
17585 SDValue MulRHS = LHS.getOperand(i: 1);
17586 SDValue AddRHS = RHS;
17587
17588 if (SDValue FoldedMAD = tryFoldMADwithSRL(DAG, SL, MulLHS, MulRHS, AddRHS))
17589 return FoldedMAD;
17590
17591 // Always check whether operands are small unsigned values, since that
17592 // knowledge is useful in more cases. Check for small signed values only if
17593 // doing so can unlock a shorter code sequence.
17594 bool MulLHSUnsigned32 = numBitsUnsigned(Op: MulLHS, DAG) <= 32;
17595 bool MulRHSUnsigned32 = numBitsUnsigned(Op: MulRHS, DAG) <= 32;
17596
17597 bool MulSignedLo = false;
17598 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
17599 MulSignedLo =
17600 numBitsSigned(Op: MulLHS, DAG) <= 32 && numBitsSigned(Op: MulRHS, DAG) <= 32;
17601 }
17602
17603 // The operands and final result all have the same number of bits. If
17604 // operands need to be extended, they can be extended with garbage. The
17605 // resulting garbage in the high bits of the mad_[iu]64_[iu]32 result is
17606 // truncated away in the end.
17607 if (VT != MVT::i64) {
17608 MulLHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulLHS);
17609 MulRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulRHS);
17610 AddRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: AddRHS);
17611 }
17612
17613 // The basic code generated is conceptually straightforward. Pseudo code:
17614 //
17615 // accum = mad_64_32 lhs.lo, rhs.lo, accum
17616 // accum.hi = add (mul lhs.hi, rhs.lo), accum.hi
17617 // accum.hi = add (mul lhs.lo, rhs.hi), accum.hi
17618 //
17619 // The second and third lines are optional, depending on whether the factors
17620 // are {sign,zero}-extended or not.
17621 //
17622 // The actual DAG is noisier than the pseudo code, but only due to
17623 // instructions that disassemble values into low and high parts, and
17624 // assemble the final result.
17625 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
17626
17627 auto MulLHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS);
17628 auto MulRHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulRHS);
17629 SDValue Accum =
17630 getMad64_32(DAG, SL, VT: MVT::i64, N0: MulLHSLo, N1: MulRHSLo, N2: AddRHS, Signed: MulSignedLo);
17631
17632 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
17633 auto [AccumLo, AccumHi] = DAG.SplitScalar(N: Accum, DL: SL, LoVT: MVT::i32, HiVT: MVT::i32);
17634
17635 if (!MulLHSUnsigned32) {
17636 auto MulLHSHi =
17637 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulLHS, N2: One);
17638 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSHi, N2: MulRHSLo);
17639 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17640 }
17641
17642 if (!MulRHSUnsigned32) {
17643 auto MulRHSHi =
17644 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulRHS, N2: One);
17645 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSLo, N2: MulRHSHi);
17646 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17647 }
17648
17649 Accum = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {AccumLo, AccumHi});
17650 Accum = DAG.getBitcast(VT: MVT::i64, V: Accum);
17651 }
17652
17653 if (VT != MVT::i64)
17654 Accum = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Accum);
17655 return Accum;
17656}
17657
17658SDValue
17659SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
17660 DAGCombinerInfo &DCI) const {
17661 SDValue RHS = N->getOperand(Num: 1);
17662 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
17663 if (!CRHS)
17664 return SDValue();
17665
17666 // TODO: Worth using computeKnownBits? Maybe expensive since it's so
17667 // common.
17668 uint64_t Val = CRHS->getZExtValue();
17669 if (countr_zero(Val) >= 32) {
17670 SelectionDAG &DAG = DCI.DAG;
17671 SDLoc SL(N);
17672 SDValue LHS = N->getOperand(Num: 0);
17673
17674 // Avoid carry machinery if we know the low half of the add does not
17675 // contribute to the final result.
17676 //
17677 // add i64:x, K if computeTrailingZeros(K) >= 32
17678 // => build_pair (add x.hi, K.hi), x.lo
17679
17680 // Breaking the 64-bit add here with this strange constant is unlikely
17681 // to interfere with addressing mode patterns.
17682
17683 SDValue Hi = getHiHalf64(Op: LHS, DAG);
17684 SDValue ConstHi32 = DAG.getConstant(Val: Hi_32(Value: Val), DL: SL, VT: MVT::i32);
17685 unsigned Opcode = N->getOpcode();
17686 if (Opcode == ISD::PTRADD)
17687 Opcode = ISD::ADD;
17688 SDValue AddHi =
17689 DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Hi, N2: ConstHi32, Flags: N->getFlags());
17690
17691 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: LHS);
17692 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: Lo, N2: AddHi);
17693 }
17694
17695 return SDValue();
17696}
17697
17698// Collect the ultimate src of each of the mul node's operands, and confirm
17699// each operand is 8 bytes.
17700static std::optional<ByteProvider<SDValue>>
17701handleMulOperand(const SDValue &MulOperand) {
17702 auto Byte0 = calculateByteProvider(Op: MulOperand, Index: 0, Depth: 0);
17703 if (!Byte0 || Byte0->isConstantZero()) {
17704 return std::nullopt;
17705 }
17706 auto Byte1 = calculateByteProvider(Op: MulOperand, Index: 1, Depth: 0);
17707 if (Byte1 && !Byte1->isConstantZero()) {
17708 return std::nullopt;
17709 }
17710 return Byte0;
17711}
17712
17713static unsigned addPermMasks(unsigned First, unsigned Second) {
17714 unsigned FirstCs = First & 0x0c0c0c0c;
17715 unsigned SecondCs = Second & 0x0c0c0c0c;
17716 unsigned FirstNoCs = First & ~0x0c0c0c0c;
17717 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
17718
17719 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
17720 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
17721 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
17722 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
17723
17724 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
17725}
17726
17727struct DotSrc {
17728 SDValue SrcOp;
17729 int64_t PermMask;
17730 int64_t DWordOffset;
17731};
17732
17733static void placeSources(ByteProvider<SDValue> &Src0,
17734 ByteProvider<SDValue> &Src1,
17735 SmallVectorImpl<DotSrc> &Src0s,
17736 SmallVectorImpl<DotSrc> &Src1s, int Step) {
17737
17738 assert(Src0.Src.has_value() && Src1.Src.has_value());
17739 // Src0s and Src1s are empty, just place arbitrarily.
17740 if (Step == 0) {
17741 Src0s.push_back(Elt: {.SrcOp: *Src0.Src, .PermMask: ((Src0.SrcOffset % 4) << 24) + 0x0c0c0c,
17742 .DWordOffset: Src0.SrcOffset / 4});
17743 Src1s.push_back(Elt: {.SrcOp: *Src1.Src, .PermMask: ((Src1.SrcOffset % 4) << 24) + 0x0c0c0c,
17744 .DWordOffset: Src1.SrcOffset / 4});
17745 return;
17746 }
17747
17748 for (int BPI = 0; BPI < 2; BPI++) {
17749 std::pair<ByteProvider<SDValue>, ByteProvider<SDValue>> BPP = {Src0, Src1};
17750 if (BPI == 1) {
17751 BPP = {Src1, Src0};
17752 }
17753 unsigned ZeroMask = 0x0c0c0c0c;
17754 unsigned FMask = 0xFF << (8 * (3 - Step));
17755
17756 unsigned FirstMask =
17757 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17758 unsigned SecondMask =
17759 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17760 // Attempt to find Src vector which contains our SDValue, if so, add our
17761 // perm mask to the existing one. If we are unable to find a match for the
17762 // first SDValue, attempt to find match for the second.
17763 int FirstGroup = -1;
17764 for (int I = 0; I < 2; I++) {
17765 SmallVectorImpl<DotSrc> &Srcs = I == 0 ? Src0s : Src1s;
17766 auto MatchesFirst = [&BPP](DotSrc &IterElt) {
17767 return IterElt.SrcOp == *BPP.first.Src &&
17768 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
17769 };
17770
17771 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesFirst);
17772 if (Match != Srcs.end()) {
17773 Match->PermMask = addPermMasks(First: FirstMask, Second: Match->PermMask);
17774 FirstGroup = I;
17775 break;
17776 }
17777 }
17778 if (FirstGroup != -1) {
17779 SmallVectorImpl<DotSrc> &Srcs = FirstGroup == 1 ? Src0s : Src1s;
17780 auto MatchesSecond = [&BPP](DotSrc &IterElt) {
17781 return IterElt.SrcOp == *BPP.second.Src &&
17782 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
17783 };
17784 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesSecond);
17785 if (Match != Srcs.end()) {
17786 Match->PermMask = addPermMasks(First: SecondMask, Second: Match->PermMask);
17787 } else
17788 Srcs.push_back(Elt: {.SrcOp: *BPP.second.Src, .PermMask: SecondMask, .DWordOffset: BPP.second.SrcOffset / 4});
17789 return;
17790 }
17791 }
17792
17793 // If we have made it here, then we could not find a match in Src0s or Src1s
17794 // for either Src0 or Src1, so just place them arbitrarily.
17795
17796 unsigned ZeroMask = 0x0c0c0c0c;
17797 unsigned FMask = 0xFF << (8 * (3 - Step));
17798
17799 Src0s.push_back(
17800 Elt: {.SrcOp: *Src0.Src,
17801 .PermMask: ((Src0.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17802 .DWordOffset: Src0.SrcOffset / 4});
17803 Src1s.push_back(
17804 Elt: {.SrcOp: *Src1.Src,
17805 .PermMask: ((Src1.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17806 .DWordOffset: Src1.SrcOffset / 4});
17807}
17808
17809static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
17810 SmallVectorImpl<DotSrc> &Srcs, bool IsSigned,
17811 bool IsAny) {
17812
17813 // If we just have one source, just permute it accordingly.
17814 if (Srcs.size() == 1) {
17815 auto *Elt = Srcs.begin();
17816 auto EltOp = getDWordFromOffset(DAG, SL, Src: Elt->SrcOp, DWordOffset: Elt->DWordOffset);
17817
17818 // v_perm will produce the original value
17819 if (Elt->PermMask == 0x3020100)
17820 return EltOp;
17821
17822 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17823 N3: DAG.getConstant(Val: Elt->PermMask, DL: SL, VT: MVT::i32));
17824 }
17825
17826 auto *FirstElt = Srcs.begin();
17827 auto *SecondElt = std::next(x: FirstElt);
17828
17829 SmallVector<SDValue, 2> Perms;
17830
17831 // If we have multiple sources in the chain, combine them via perms (using
17832 // calculated perm mask) and Ors.
17833 while (true) {
17834 auto FirstMask = FirstElt->PermMask;
17835 auto SecondMask = SecondElt->PermMask;
17836
17837 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
17838 unsigned FirstPlusFour = FirstMask | 0x04040404;
17839 // 0x0c + 0x04 = 0x10, so anding with 0x0F will produced 0x00 for any
17840 // original 0x0C.
17841 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
17842
17843 auto PermMask = addPermMasks(First: FirstMask, Second: SecondMask);
17844 auto FirstVal =
17845 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17846 auto SecondVal =
17847 getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp, DWordOffset: SecondElt->DWordOffset);
17848
17849 Perms.push_back(Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: FirstVal,
17850 N2: SecondVal,
17851 N3: DAG.getConstant(Val: PermMask, DL: SL, VT: MVT::i32)));
17852
17853 FirstElt = std::next(x: SecondElt);
17854 if (FirstElt == Srcs.end())
17855 break;
17856
17857 SecondElt = std::next(x: FirstElt);
17858 // If we only have a FirstElt, then just combine that into the cumulative
17859 // source node.
17860 if (SecondElt == Srcs.end()) {
17861 auto EltOp =
17862 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17863
17864 Perms.push_back(
17865 Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17866 N3: DAG.getConstant(Val: FirstElt->PermMask, DL: SL, VT: MVT::i32)));
17867 break;
17868 }
17869 }
17870
17871 assert(Perms.size() == 1 || Perms.size() == 2);
17872 return Perms.size() == 2
17873 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Perms[0], N2: Perms[1])
17874 : Perms[0];
17875}
17876
17877static void fixMasks(SmallVectorImpl<DotSrc> &Srcs, unsigned ChainLength) {
17878 for (auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
17879 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
17880 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
17881 EntryMask += ZeroMask;
17882 }
17883}
17884
17885static bool isMul(const SDValue Op) {
17886 auto Opcode = Op.getOpcode();
17887
17888 return (Opcode == ISD::MUL || Opcode == AMDGPUISD::MUL_U24 ||
17889 Opcode == AMDGPUISD::MUL_I24);
17890}
17891
17892static std::optional<bool>
17893checkDot4MulSignedness(const SDValue &N, ByteProvider<SDValue> &Src0,
17894 ByteProvider<SDValue> &Src1, const SDValue &S0Op,
17895 const SDValue &S1Op, const SelectionDAG &DAG) {
17896 // If we both ops are i8s (pre legalize-dag), then the signedness semantics
17897 // of the dot4 is irrelevant.
17898 if (S0Op.getValueSizeInBits() == 8 && S1Op.getValueSizeInBits() == 8)
17899 return false;
17900
17901 auto Known0 = DAG.computeKnownBits(Op: S0Op, Depth: 0);
17902 bool S0IsUnsigned = Known0.countMinLeadingZeros() > 0;
17903 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
17904 auto Known1 = DAG.computeKnownBits(Op: S1Op, Depth: 0);
17905 bool S1IsUnsigned = Known1.countMinLeadingZeros() > 0;
17906 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
17907
17908 assert(!(S0IsUnsigned && S0IsSigned));
17909 assert(!(S1IsUnsigned && S1IsSigned));
17910
17911 // There are 9 possible permutations of
17912 // {S0IsUnsigned, S0IsSigned, S1IsUnsigned, S1IsSigned}
17913
17914 // In two permutations, the sign bits are known to be the same for both Ops,
17915 // so simply return Signed / Unsigned corresponding to the MSB
17916
17917 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
17918 return S0IsSigned;
17919
17920 // In another two permutations, the sign bits are known to be opposite. In
17921 // this case return std::nullopt to indicate a bad match.
17922
17923 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
17924 return std::nullopt;
17925
17926 // In the remaining five permutations, we don't know the value of the sign
17927 // bit for at least one Op. Since we have a valid ByteProvider, we know that
17928 // the upper bits must be extension bits. Thus, the only ways for the sign
17929 // bit to be unknown is if it was sign extended from unknown value, or if it
17930 // was any extended. In either case, it is correct to use the signed
17931 // version of the signedness semantics of dot4
17932
17933 // In two of such permutations, we known the sign bit is set for
17934 // one op, and the other is unknown. It is okay to used signed version of
17935 // dot4.
17936 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
17937 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
17938 return true;
17939
17940 // In one such permutation, we don't know either of the sign bits. It is okay
17941 // to used the signed version of dot4.
17942 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
17943 return true;
17944
17945 // In two of such permutations, we known the sign bit is unset for
17946 // one op, and the other is unknown. Return std::nullopt to indicate a
17947 // bad match.
17948 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
17949 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
17950 return std::nullopt;
17951
17952 llvm_unreachable("Fully covered condition");
17953}
17954
17955SDValue SITargetLowering::performAddCombine(SDNode *N,
17956 DAGCombinerInfo &DCI) const {
17957 SelectionDAG &DAG = DCI.DAG;
17958 EVT VT = N->getValueType(ResNo: 0);
17959 SDLoc SL(N);
17960 SDValue LHS = N->getOperand(Num: 0);
17961 SDValue RHS = N->getOperand(Num: 1);
17962
17963 if (LHS.getOpcode() == ISD::MUL || RHS.getOpcode() == ISD::MUL) {
17964 if (Subtarget->hasMad64_32()) {
17965 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
17966 return Folded;
17967 }
17968 }
17969
17970 if (SDValue V = reassociateScalarOps(N, DAG)) {
17971 return V;
17972 }
17973
17974 if (VT == MVT::i64) {
17975 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17976 return Folded;
17977 }
17978
17979 // dot4 produces a 32-bit result, so a wider VT can't be folded.
17980 if (!VT.isVector() && VT.getSizeInBits() <= 32 &&
17981 (isMul(Op: LHS) || isMul(Op: RHS)) && Subtarget->hasDot7Insts() &&
17982 (Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
17983 SDValue TempNode(N, 0);
17984 std::optional<bool> IsSigned;
17985 SmallVector<DotSrc, 4> Src0s;
17986 SmallVector<DotSrc, 4> Src1s;
17987 SmallVector<SDValue, 4> Src2s;
17988
17989 // Match the v_dot4 tree, while collecting src nodes.
17990 int ChainLength = 0;
17991 for (int I = 0; I < 4; I++) {
17992 auto MulIdx = isMul(Op: LHS) ? 0 : isMul(Op: RHS) ? 1 : -1;
17993 if (MulIdx == -1)
17994 break;
17995 auto Src0 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0));
17996 if (!Src0)
17997 break;
17998 auto Src1 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1));
17999 if (!Src1)
18000 break;
18001
18002 auto IterIsSigned = checkDot4MulSignedness(
18003 N: TempNode->getOperand(Num: MulIdx), Src0&: *Src0, Src1&: *Src1,
18004 S0Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0),
18005 S1Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1), DAG);
18006 if (!IterIsSigned)
18007 break;
18008 if (!IsSigned)
18009 IsSigned = *IterIsSigned;
18010 if (*IterIsSigned != *IsSigned)
18011 break;
18012 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I);
18013 auto AddIdx = 1 - MulIdx;
18014 // Allow the special case where add (add (mul24, 0), mul24) became ->
18015 // add (mul24, mul24).
18016 if (I == 2 && isMul(Op: TempNode->getOperand(Num: AddIdx))) {
18017 Src2s.push_back(Elt: TempNode->getOperand(Num: AddIdx));
18018 auto Src0 =
18019 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0));
18020 if (!Src0)
18021 break;
18022 auto Src1 =
18023 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1));
18024 if (!Src1)
18025 break;
18026 auto IterIsSigned = checkDot4MulSignedness(
18027 N: TempNode->getOperand(Num: AddIdx), Src0&: *Src0, Src1&: *Src1,
18028 S0Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0),
18029 S1Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1), DAG);
18030 if (!IterIsSigned)
18031 break;
18032 assert(IsSigned);
18033 if (*IterIsSigned != *IsSigned)
18034 break;
18035 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I + 1);
18036 Src2s.push_back(Elt: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
18037 ChainLength = I + 2;
18038 break;
18039 }
18040
18041 TempNode = TempNode->getOperand(Num: AddIdx);
18042 Src2s.push_back(Elt: TempNode);
18043 ChainLength = I + 1;
18044 // The loop body treats TempNode's operands as addends.
18045 if (TempNode.getOpcode() != ISD::ADD)
18046 break;
18047 LHS = TempNode->getOperand(Num: 0);
18048 RHS = TempNode->getOperand(Num: 1);
18049 }
18050
18051 if (ChainLength < 2)
18052 return SDValue();
18053
18054 // Masks were constructed with assumption that we would find a chain of
18055 // length 4. If not, then we need to 0 out the MSB bits (via perm mask of
18056 // 0x0c) so they do not affect dot calculation.
18057 if (ChainLength < 4) {
18058 fixMasks(Srcs&: Src0s, ChainLength);
18059 fixMasks(Srcs&: Src1s, ChainLength);
18060 }
18061
18062 SDValue Src0, Src1;
18063
18064 // If we are just using a single source for both, and have permuted the
18065 // bytes consistently, we can just use the sources without permuting
18066 // (commutation).
18067 bool UseOriginalSrc = false;
18068 if (ChainLength == 4 && Src0s.size() == 1 && Src1s.size() == 1 &&
18069 Src0s.begin()->PermMask == Src1s.begin()->PermMask &&
18070 Src0s.begin()->SrcOp.getValueSizeInBits() >= 32 &&
18071 Src1s.begin()->SrcOp.getValueSizeInBits() >= 32) {
18072 SmallVector<unsigned, 4> SrcBytes;
18073 auto Src0Mask = Src0s.begin()->PermMask;
18074 SrcBytes.push_back(Elt: (Src0Mask >> 24) & 0xFF);
18075 bool UniqueEntries = true;
18076 for (auto I = 1; I < 4; I++) {
18077 auto NextByte = (Src0Mask >> ((3 - I) * 8)) & 0xFF;
18078
18079 if (is_contained(Range&: SrcBytes, Element: NextByte)) {
18080 UniqueEntries = false;
18081 break;
18082 }
18083 SrcBytes.push_back(Elt: NextByte);
18084 }
18085
18086 if (UniqueEntries) {
18087 UseOriginalSrc = true;
18088
18089 auto *FirstElt = Src0s.begin();
18090 auto FirstEltOp =
18091 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
18092
18093 auto *SecondElt = Src1s.begin();
18094 auto SecondEltOp = getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp,
18095 DWordOffset: SecondElt->DWordOffset);
18096
18097 Src0 = DAG.getBitcastedAnyExtOrTrunc(Op: FirstEltOp, DL: SL,
18098 VT: MVT::getIntegerVT(BitWidth: 32));
18099 Src1 = DAG.getBitcastedAnyExtOrTrunc(Op: SecondEltOp, DL: SL,
18100 VT: MVT::getIntegerVT(BitWidth: 32));
18101 }
18102 }
18103
18104 if (!UseOriginalSrc) {
18105 Src0 = resolveSources(DAG, SL, Srcs&: Src0s, IsSigned: false, IsAny: true);
18106 Src1 = resolveSources(DAG, SL, Srcs&: Src1s, IsSigned: false, IsAny: true);
18107 }
18108
18109 assert(IsSigned);
18110 SDValue Src2 =
18111 DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Src2s[ChainLength - 1], DL: SL, VT: MVT::i32);
18112
18113 SDValue IID = DAG.getTargetConstant(Val: *IsSigned ? Intrinsic::amdgcn_sdot4
18114 : Intrinsic::amdgcn_udot4,
18115 DL: SL, VT: MVT::i64);
18116
18117 assert(!VT.isVector());
18118 auto Dot = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32, N1: IID, N2: Src0,
18119 N3: Src1, N4: Src2, N5: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
18120
18121 return DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Dot, DL: SL, VT);
18122 }
18123
18124 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
18125 return SDValue();
18126
18127 // add x, zext (setcc) => uaddo_carry x, 0, setcc
18128 // add x, sext (setcc) => usubo_carry x, 0, setcc
18129 unsigned Opc = LHS.getOpcode();
18130 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND ||
18131 Opc == ISD::ANY_EXTEND || Opc == ISD::UADDO_CARRY)
18132 std::swap(a&: RHS, b&: LHS);
18133
18134 Opc = RHS.getOpcode();
18135 switch (Opc) {
18136 default:
18137 break;
18138 case ISD::ZERO_EXTEND:
18139 case ISD::SIGN_EXTEND:
18140 case ISD::ANY_EXTEND: {
18141 auto Cond = RHS.getOperand(i: 0);
18142 // If this won't be a real VOPC output, we would still need to insert an
18143 // extra instruction anyway.
18144 if (!isBoolSGPR(V: Cond))
18145 break;
18146 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
18147 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
18148 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::USUBO_CARRY : ISD::UADDO_CARRY;
18149 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
18150 }
18151 case ISD::UADDO_CARRY: {
18152 // add x, (uaddo_carry y, 0, cc) => uaddo_carry x, y, cc
18153 if (!isNullConstant(V: RHS.getOperand(i: 1)))
18154 break;
18155 SDValue Args[] = {LHS, RHS.getOperand(i: 0), RHS.getOperand(i: 2)};
18156 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: RHS->getVTList(), Ops: Args);
18157 }
18158 }
18159 return SDValue();
18160}
18161
18162SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
18163 DAGCombinerInfo &DCI) const {
18164 SelectionDAG &DAG = DCI.DAG;
18165 SDLoc DL(N);
18166 EVT VT = N->getValueType(ResNo: 0);
18167 SDValue N0 = N->getOperand(Num: 0);
18168 SDValue N1 = N->getOperand(Num: 1);
18169
18170 // The following folds transform PTRADDs into regular arithmetic in cases
18171 // where the PTRADD wouldn't be folded as an immediate offset into memory
18172 // instructions anyway. They are target-specific in that other targets might
18173 // prefer to not lose information about the pointer arithmetic.
18174
18175 // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
18176 // Adapted from DAGCombiner::visitADDLikeCommutative.
18177 SDValue V, K;
18178 if (sd_match(N: N1, P: m_Shl(L: m_Neg(V: m_Value(N&: V)), R: m_Value(N&: K)))) {
18179 SDNodeFlags ShlFlags = N1->getFlags();
18180 // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
18181 // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
18182 // preserved.
18183 SDNodeFlags NewShlFlags =
18184 ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
18185 ? SDNodeFlags::NoSignedWrap
18186 : SDNodeFlags();
18187 SDValue Inner = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: V, N2: K, Flags: NewShlFlags);
18188 DCI.AddToWorklist(N: Inner.getNode());
18189 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Inner);
18190 }
18191
18192 // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
18193 // performAddCombine.
18194 if (N1.getOpcode() == ISD::MUL) {
18195 if (Subtarget->hasMad64_32()) {
18196 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
18197 return Folded;
18198 }
18199 }
18200
18201 // If the 32 low bits of the constant are all zero, there is nothing to fold
18202 // into an immediate offset, so it's better to eliminate the unnecessary
18203 // addition for the lower 32 bits than to preserve the PTRADD.
18204 // Analogous to a fold in performAddCombine.
18205 if (VT == MVT::i64) {
18206 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18207 return Folded;
18208 }
18209
18210 if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
18211 return SDValue();
18212
18213 SDValue X = N0;
18214 SDValue Y = N1.getOperand(i: 0);
18215 SDValue Z = N1.getOperand(i: 1);
18216 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Y);
18217 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Z);
18218
18219 if (!YIsConstant && !ZIsConstant && !X->isDivergent() &&
18220 Y->isDivergent() != Z->isDivergent()) {
18221 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
18222 // y are uniform and z isn't.
18223 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
18224 // z are uniform and y isn't.
18225 // The goal is to push uniform operands up in the computation, so that they
18226 // can be handled with scalar operations. We can't use reassociateScalarOps
18227 // for this since it requires two identical commutative operations to
18228 // reassociate.
18229 if (Y->isDivergent())
18230 std::swap(a&: Y, b&: Z);
18231 // If both additions in the original were NUW, reassociation preserves that.
18232 SDNodeFlags ReassocFlags =
18233 (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
18234 SDValue UniformInner = DAG.getMemBasePlusOffset(Base: X, Offset: Y, DL, Flags: ReassocFlags);
18235 DCI.AddToWorklist(N: UniformInner.getNode());
18236 return DAG.getMemBasePlusOffset(Base: UniformInner, Offset: Z, DL, Flags: ReassocFlags);
18237 }
18238
18239 return SDValue();
18240}
18241
18242static bool isCtlzOpc(unsigned Opc) {
18243 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
18244}
18245
18246SDValue SITargetLowering::performSubCombine(SDNode *N,
18247 DAGCombinerInfo &DCI) const {
18248 SelectionDAG &DAG = DCI.DAG;
18249 EVT VT = N->getValueType(ResNo: 0);
18250
18251 if (VT == MVT::i64) {
18252 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
18253 return Folded;
18254 }
18255
18256 if (VT != MVT::i32)
18257 return SDValue();
18258
18259 SDLoc SL(N);
18260 SDValue LHS = N->getOperand(Num: 0);
18261 SDValue RHS = N->getOperand(Num: 1);
18262
18263 // sub x, zext (setcc) => usubo_carry x, 0, setcc
18264 // sub x, sext (setcc) => uaddo_carry x, 0, setcc
18265 unsigned Opc = RHS.getOpcode();
18266 switch (Opc) {
18267 default:
18268 break;
18269 case ISD::ZERO_EXTEND:
18270 case ISD::SIGN_EXTEND:
18271 case ISD::ANY_EXTEND: {
18272 auto Cond = RHS.getOperand(i: 0);
18273 // If this won't be a real VOPC output, we would still need to insert an
18274 // extra instruction anyway.
18275 if (!isBoolSGPR(V: Cond))
18276 break;
18277 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
18278 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
18279 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
18280 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
18281 }
18282 }
18283
18284 if (LHS.getOpcode() == ISD::USUBO_CARRY) {
18285 // sub (usubo_carry x, 0, cc), y => usubo_carry x, y, cc
18286 if (!isNullConstant(V: LHS.getOperand(i: 1)))
18287 return SDValue();
18288 SDValue Args[] = {LHS.getOperand(i: 0), RHS, LHS.getOperand(i: 2)};
18289 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: SDLoc(N), VTList: LHS->getVTList(), Ops: Args);
18290 }
18291
18292 // sub (ctlz (xor x, (sra x, 31))), 1 -> ctls x.
18293 if (isOneConstant(V: RHS) && isCtlzOpc(Opc: LHS.getOpcode())) {
18294 SDValue CtlzSrc = LHS.getOperand(i: 0);
18295 // Check for xor x, (sra x, 31) pattern.
18296 if (CtlzSrc.getOpcode() == ISD::XOR) {
18297 SDValue X = CtlzSrc.getOperand(i: 0);
18298 SDValue SignExt = CtlzSrc.getOperand(i: 1);
18299 // Try both ordering of XOR operands.
18300 if (SignExt.getOpcode() != ISD::SRA)
18301 std::swap(a&: X, b&: SignExt);
18302 if (SignExt.getOpcode() == ISD::SRA && SignExt.getOperand(i: 0) == X) {
18303 ConstantSDNode *ShiftAmt =
18304 dyn_cast<ConstantSDNode>(Val: SignExt.getOperand(i: 1));
18305 unsigned BitWidth = X.getValueType().getScalarSizeInBits();
18306 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1)
18307 return DAG.getNode(Opcode: ISD::CTLS, DL: SL, VT, Operand: X);
18308 }
18309 }
18310 }
18311
18312 return SDValue();
18313}
18314
18315SDValue SITargetLowering::performFAddCombine(SDNode *N,
18316 DAGCombinerInfo &DCI) const {
18317 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18318 return SDValue();
18319
18320 SelectionDAG &DAG = DCI.DAG;
18321 EVT VT = N->getValueType(ResNo: 0);
18322
18323 SDLoc SL(N);
18324 SDValue LHS = N->getOperand(Num: 0);
18325 SDValue RHS = N->getOperand(Num: 1);
18326
18327 // These should really be instruction patterns, but writing patterns with
18328 // source modifiers is a pain.
18329
18330 // fadd (fadd (a, a), b) -> mad 2.0, a, b
18331 if (LHS.getOpcode() == ISD::FADD) {
18332 SDValue A = LHS.getOperand(i: 0);
18333 if (A == LHS.getOperand(i: 1)) {
18334 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
18335 if (FusedOp != 0) {
18336 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18337 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: RHS);
18338 }
18339 }
18340 }
18341
18342 // fadd (b, fadd (a, a)) -> mad 2.0, a, b
18343 if (RHS.getOpcode() == ISD::FADD) {
18344 SDValue A = RHS.getOperand(i: 0);
18345 if (A == RHS.getOperand(i: 1)) {
18346 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
18347 if (FusedOp != 0) {
18348 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18349 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: LHS);
18350 }
18351 }
18352 }
18353
18354 return SDValue();
18355}
18356
18357SDValue SITargetLowering::performFSubCombine(SDNode *N,
18358 DAGCombinerInfo &DCI) const {
18359 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
18360 return SDValue();
18361
18362 SelectionDAG &DAG = DCI.DAG;
18363 SDLoc SL(N);
18364 EVT VT = N->getValueType(ResNo: 0);
18365 assert(!VT.isVector());
18366
18367 // Try to get the fneg to fold into the source modifier. This undoes generic
18368 // DAG combines and folds them into the mad.
18369 //
18370 // Only do this if we are not trying to support denormals. v_mad_f32 does
18371 // not support denormals ever.
18372 SDValue LHS = N->getOperand(Num: 0);
18373 SDValue RHS = N->getOperand(Num: 1);
18374 if (LHS.getOpcode() == ISD::FADD) {
18375 // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c)
18376 SDValue A = LHS.getOperand(i: 0);
18377 if (A == LHS.getOperand(i: 1)) {
18378 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
18379 if (FusedOp != 0) {
18380 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
18381 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
18382
18383 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: NegRHS);
18384 }
18385 }
18386 }
18387
18388 if (RHS.getOpcode() == ISD::FADD) {
18389 // (fsub c, (fadd a, a)) -> mad -2.0, a, c
18390
18391 SDValue A = RHS.getOperand(i: 0);
18392 if (A == RHS.getOperand(i: 1)) {
18393 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
18394 if (FusedOp != 0) {
18395 const SDValue NegTwo = DAG.getConstantFP(Val: -2.0, DL: SL, VT);
18396 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: NegTwo, N3: LHS);
18397 }
18398 }
18399 }
18400
18401 return SDValue();
18402}
18403
18404SDValue SITargetLowering::performFDivCombine(SDNode *N,
18405 DAGCombinerInfo &DCI) const {
18406 SelectionDAG &DAG = DCI.DAG;
18407 SDLoc SL(N);
18408 EVT VT = N->getValueType(ResNo: 0);
18409
18410 if (VT != MVT::f16 && VT != MVT::bf16)
18411 return SDValue();
18412
18413 SDValue LHS = N->getOperand(Num: 0);
18414 SDValue RHS = N->getOperand(Num: 1);
18415
18416 SDNodeFlags Flags = N->getFlags();
18417 SDNodeFlags RHSFlags = RHS->getFlags();
18418 if (!Flags.hasAllowContract() || !RHSFlags.hasAllowContract() ||
18419 !RHS->hasOneUse())
18420 return SDValue();
18421
18422 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
18423 bool IsNegative = false;
18424 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
18425 // fdiv contract 1.0, (sqrt contract x) -> rsq
18426 // fdiv contract -1.0, (sqrt contract x) -> fneg(rsq)
18427 if (RHS.getOpcode() == ISD::FSQRT) {
18428 // TODO: Or in RHS flags, somehow missing from SDNodeFlags
18429 SDValue SqrtOp = RHS.getOperand(i: 0);
18430 SDValue Rsq;
18431 if (isOperationLegal(Op: ISD::FSQRT, VT)) {
18432 // fsqrt legality correlates to rsq availability of the same type.
18433 Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT, Operand: SqrtOp, Flags);
18434 } else if (VT == MVT::f16) {
18435 // Targets without 16-bit instructions (gfx6/gfx7) have no f16 rsq,
18436 // but v_rsq_f32 is more than accurate enough for f16. Unlike bf16,
18437 // every f16 value (including denormals) extends to a normal f32, and
18438 // an f16 rsq result is never denormal, so the f32 reciprocal square
18439 // root needs no denormal handling. Compute it in f32 and round back.
18440 SDValue Ext =
18441 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: SqrtOp, Flags);
18442 SDValue F32Rsq =
18443 DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
18444 Rsq = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: F32Rsq,
18445 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
18446 } else {
18447 // bf16 shares f32's exponent range, so bf16 denormals would extend to
18448 // f32 denormals that v_rsq_f32 does not handle. Leave it expanded.
18449 return SDValue();
18450 }
18451 return IsNegative ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Rsq, Flags) : Rsq;
18452 }
18453 }
18454 }
18455
18456 return SDValue();
18457}
18458
18459SDValue SITargetLowering::performFMulCombine(SDNode *N,
18460 DAGCombinerInfo &DCI) const {
18461 SelectionDAG &DAG = DCI.DAG;
18462 EVT VT = N->getValueType(ResNo: 0);
18463 EVT ScalarVT = VT.getScalarType();
18464 EVT IntVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
18465
18466 if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() &&
18467 (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
18468 // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32.
18469 return SDValue();
18470 }
18471
18472 SDValue LHS = N->getOperand(Num: 0);
18473 SDValue RHS = N->getOperand(Num: 1);
18474
18475 // It is cheaper to realize i32 inline constants as compared against
18476 // materializing f16 or f64 (or even non-inline f32) values,
18477 // possible via ldexp usage, as shown below :
18478 //
18479 // Given : A = 2^a & B = 2^b ; where a and b are integers.
18480 // fmul x, (select y, A, B) -> ldexp( x, (select i32 y, a, b) )
18481 // fmul x, (select y, -A, -B) -> ldexp( (fneg x), (select i32 y, a, b) )
18482 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
18483 (RHS.hasOneUse() && RHS.getOpcode() == ISD::SELECT)) {
18484 const ConstantFPSDNode *TrueNode = isConstOrConstSplatFP(N: RHS.getOperand(i: 1));
18485 if (!TrueNode)
18486 return SDValue();
18487 const ConstantFPSDNode *FalseNode =
18488 isConstOrConstSplatFP(N: RHS.getOperand(i: 2));
18489 if (!FalseNode)
18490 return SDValue();
18491
18492 if (TrueNode->isNegative() != FalseNode->isNegative())
18493 return SDValue();
18494
18495 // For f32, only non-inline constants should be transformed.
18496 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18497 if (ScalarVT == MVT::f32 &&
18498 TII->isInlineConstant(Imm: TrueNode->getValueAPF()) &&
18499 TII->isInlineConstant(Imm: FalseNode->getValueAPF()))
18500 return SDValue();
18501
18502 int TrueNodeExpVal = TrueNode->getValueAPF().getExactLog2Abs();
18503 if (TrueNodeExpVal == INT_MIN)
18504 return SDValue();
18505 int FalseNodeExpVal = FalseNode->getValueAPF().getExactLog2Abs();
18506 if (FalseNodeExpVal == INT_MIN)
18507 return SDValue();
18508
18509 SDLoc SL(N);
18510 SDValue SelectNode =
18511 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: IntVT, N1: RHS.getOperand(i: 0),
18512 N2: DAG.getSignedConstant(Val: TrueNodeExpVal, DL: SL, VT: IntVT),
18513 N3: DAG.getSignedConstant(Val: FalseNodeExpVal, DL: SL, VT: IntVT));
18514
18515 LHS = TrueNode->isNegative()
18516 ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS, Flags: LHS->getFlags())
18517 : LHS;
18518
18519 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: LHS, N2: SelectNode, Flags: N->getFlags());
18520 }
18521
18522 return SDValue();
18523}
18524
18525SDValue SITargetLowering::performFMACombine(SDNode *N,
18526 DAGCombinerInfo &DCI) const {
18527 SelectionDAG &DAG = DCI.DAG;
18528 EVT VT = N->getValueType(ResNo: 0);
18529 SDLoc SL(N);
18530
18531 if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
18532 return SDValue();
18533
18534 // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
18535 // FDOT2((V2F16)S0, (V2F16)S1, (F32)z))
18536 SDValue Op1 = N->getOperand(Num: 0);
18537 SDValue Op2 = N->getOperand(Num: 1);
18538 SDValue FMA = N->getOperand(Num: 2);
18539
18540 if (FMA.getOpcode() != ISD::FMA || Op1.getOpcode() != ISD::FP_EXTEND ||
18541 Op2.getOpcode() != ISD::FP_EXTEND)
18542 return SDValue();
18543
18544 // The fdot2 fold (fma_mix -> dot2) is only safe when both instructions agree
18545 // on how f16 subnormal inputs are handled. However, if both FMAs carry afn
18546 // the caller accepts approximate results, so any subnormal flushing
18547 // introduced by dot2 is acceptable regardless of mode.
18548 //
18549 // gfx90a (CDNA2) is the sole exception (dot2UnconditionalFlush): v_dot2c
18550 // unconditionally flushes f16 subnormal inputs to zero regardless of MODE,
18551 // while v_fma_mix_f32 preserves them when ieee=1 (the default compute kernel
18552 // mode). The fold is safe only when f32 denorm = PreserveSign, which implies
18553 // ieee=0 so both flush.
18554 //
18555 // All other GPUs: v_dot2 does NOT flush f16 subnormal inputs. v_fma_mix_f32
18556 // flushes them only when f32 denorm = PreserveSign. The fold is safe only
18557 // when f32 denorm is IEEE (both preserve the subnormal). Dynamic mode is
18558 // also rejected since the runtime value is unknown.
18559 bool AllowInaccuracy = N->getFlags().hasApproximateFuncs() &&
18560 FMA->getFlags().hasApproximateFuncs();
18561 if (!AllowInaccuracy) {
18562 const MachineFunction &MF = DAG.getMachineFunction();
18563 DenormalMode Mode = MF.getDenormalMode(FPType: APFloat::IEEEsingle());
18564 if (Subtarget->dot2UnconditionalFlush()) {
18565 // gfx90a: fold safe only when f32 denorm flushes.
18566 if (Mode != DenormalMode::getPreserveSign())
18567 return SDValue();
18568 } else {
18569 // All other GPUs: fold safe only when f32 denorm is IEEE.
18570 if (Mode != DenormalMode::getIEEE())
18571 return SDValue();
18572 }
18573 }
18574
18575 // fp-contract allows reassociating the fma tree into a dot product.
18576 if (N->getFlags().hasAllowContract() && FMA->getFlags().hasAllowContract()) {
18577 Op1 = Op1.getOperand(i: 0);
18578 Op2 = Op2.getOperand(i: 0);
18579 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18580 Op2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18581 return SDValue();
18582
18583 SDValue Vec1 = Op1.getOperand(i: 0);
18584 SDValue Idx1 = Op1.getOperand(i: 1);
18585 SDValue Vec2 = Op2.getOperand(i: 0);
18586
18587 SDValue FMAOp1 = FMA.getOperand(i: 0);
18588 SDValue FMAOp2 = FMA.getOperand(i: 1);
18589 SDValue FMAAcc = FMA.getOperand(i: 2);
18590
18591 if (FMAOp1.getOpcode() != ISD::FP_EXTEND ||
18592 FMAOp2.getOpcode() != ISD::FP_EXTEND)
18593 return SDValue();
18594
18595 FMAOp1 = FMAOp1.getOperand(i: 0);
18596 FMAOp2 = FMAOp2.getOperand(i: 0);
18597 if (FMAOp1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18598 FMAOp2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18599 return SDValue();
18600
18601 SDValue Vec3 = FMAOp1.getOperand(i: 0);
18602 SDValue Vec4 = FMAOp2.getOperand(i: 0);
18603 SDValue Idx2 = FMAOp1.getOperand(i: 1);
18604
18605 if (Idx1 != Op2.getOperand(i: 1) || Idx2 != FMAOp2.getOperand(i: 1))
18606 return SDValue();
18607
18608 if (!isa<ConstantSDNode>(Val: Idx1) || !isa<ConstantSDNode>(Val: Idx2) ||
18609 Idx1 == Idx2)
18610 return SDValue();
18611
18612 if (Vec1 == Vec2 || Vec3 == Vec4)
18613 return SDValue();
18614
18615 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16)
18616 return SDValue();
18617
18618 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
18619 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL: SL, VT: MVT::f32, N1: Vec1, N2: Vec2, N3: FMAAcc,
18620 N4: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
18621 }
18622 }
18623 return SDValue();
18624}
18625
18626// Given a double-precision ordered or unordered comparison, return the
18627// condition code for an equivalent integral comparison of the operands' upper
18628// 32 bits, or `SETCC_INVALID` if not possible.
18629// For simplicity, no simplification occurs if the operands are not both known
18630// to have sign bit zero.
18631//
18632// EQ/NE:
18633// If LHS.lo32 == RHS.lo32:
18634// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18635// If LHS.lo32 != RHS.lo32:
18636// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18637// The reduction is not possible if operands may be +0 and -0.
18638// For ordered eq / unordered ne, at most one operand may be NaN.
18639// For unordered eq / ordered ne, neither operand can be NaN.
18640//
18641// LT/GE:
18642// If LHS.lo32 >= RHS.lo32 (unsigned):
18643// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18644// If LHS.lo32 < RHS.lo32 (unsigned):
18645// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18646// The reduction is only supported if both operands are nonnegative.
18647// For ordered lt / unordered ge, the RHS cannot be NaN.
18648// For unordered lt / ordered ge, neither operand can be NaN.
18649//
18650// LE/GT:
18651// If LHS.lo32 > RHS.lo32 (unsigned):
18652// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18653// If LHS.lo32 <= RHS.lo32 (unsigned):
18654// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18655// The reduction is only supported if both operands are nonnegative.
18656// For unordered le / ordered gt, the LHS cannot be NaN.
18657// For ordered le / unordered gt, neither operand can be NaN.
18658static ISD::CondCode tryReduceF64CompareToHiHalf(const ISD::CondCode CC,
18659 const SDValue LHS,
18660 const SDValue RHS,
18661 const SelectionDAG &DAG) {
18662 EVT VT = LHS.getValueType();
18663 assert(VT == MVT::f64 && "Incorrect operand type!");
18664
18665 const KnownBits RHSBits = DAG.computeKnownBits(Op: RHS);
18666 // Bail if RHS sign bit is not known to be zero.
18667 if (!RHSBits.Zero.isSignBitSet())
18668 return ISD::SETCC_INVALID;
18669
18670 const KnownBits RHSKnownLo32 = RHSBits.trunc(BitWidth: 32);
18671 const KnownFPClass RHSFPClass =
18672 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: RHSBits);
18673 const bool RHSMaybeNaN = !RHSFPClass.isKnownNeverNaN();
18674
18675 const KnownBits LHSBits = DAG.computeKnownBits(Op: LHS);
18676 const KnownBits LHSKnownLo32 = LHSBits.trunc(BitWidth: 32);
18677 const KnownFPClass LHSFPClass =
18678 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: LHSBits);
18679 const bool LHSMaybeNaN = !LHSFPClass.isKnownNeverNaN();
18680
18681 // Bail if LHS sign bit is not known to be zero.
18682 if (!LHSBits.Zero.isSignBitSet())
18683 return ISD::SETCC_INVALID;
18684
18685 switch (CC) {
18686 default:
18687 break;
18688 case ISD::SETEQ:
18689 case ISD::SETOEQ:
18690 case ISD::SETUEQ:
18691 case ISD::SETONE:
18692 case ISD::SETUNE: {
18693 // OEQ should be false if either operand is NaN, so it suffices that at
18694 // least one operand is not NaN.
18695 if (CC == ISD::SETOEQ && LHSMaybeNaN && RHSMaybeNaN)
18696 break;
18697 // UEQ should be true if either operand is NaN, but this cannot be checked
18698 // on underlying bits.
18699 if (CC == ISD::SETUEQ && (LHSMaybeNaN || RHSMaybeNaN))
18700 break;
18701 // ONE should be false if either operand is NaN, but this cannot be
18702 // checked on underlying bits.
18703 if (CC == ISD::SETONE && (LHSMaybeNaN || RHSMaybeNaN))
18704 break;
18705 // UNE should be true if either operand is NaN, so it suffices that they
18706 // are not both NaN.
18707 if (CC == ISD::SETUNE && LHSMaybeNaN && RHSMaybeNaN)
18708 break;
18709
18710 const std::optional<bool> KnownEq =
18711 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18712
18713 if (!KnownEq)
18714 break;
18715
18716 if (*KnownEq)
18717 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18718 ? ISD::SETEQ
18719 : ISD::SETNE;
18720
18721 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18722 ? ISD::SETFALSE
18723 : ISD::SETTRUE;
18724 }
18725 case ISD::SETLT:
18726 case ISD::SETOLT:
18727 case ISD::SETULT:
18728 case ISD::SETGE:
18729 case ISD::SETOGE:
18730 case ISD::SETUGE: {
18731 // OLT should be false if either operand is NaN.
18732 // Since NaNs have maximum exponent and nonzero mantissa, false positives
18733 // are only possible if the RHS is NaN. (No issue with RHS == +inf since
18734 // the inequality is strict)
18735 if (CC == ISD::SETOLT && RHSMaybeNaN)
18736 break;
18737 // ULT should be true if either operand is NaN, but this cannot be ensured
18738 // with a truncated comparison.
18739 if (CC == ISD::SETULT && (LHSMaybeNaN || RHSMaybeNaN))
18740 break;
18741 // OGE should be false if either operand is NaN, but this cannot be
18742 // ensured with a truncated comparison.
18743 if (CC == ISD::SETOGE && (LHSMaybeNaN || RHSMaybeNaN))
18744 break;
18745 // UGE should be true if either operand is NaN.
18746 // False negatives are only possible if the RHS is NaN.
18747 // (No issue with RHS == +inf since the inequality is inclusive)
18748 if (CC == ISD::SETUGE && RHSMaybeNaN)
18749 break;
18750
18751 const std::optional<bool> KnownUge =
18752 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18753
18754 if (!KnownUge)
18755 break;
18756
18757 if (*KnownUge) {
18758 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18759 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18760 ? ISD::SETLT
18761 : ISD::SETGE;
18762 }
18763 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18764 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18765 ? ISD::SETLE
18766 : ISD::SETGT;
18767 }
18768 case ISD::SETLE:
18769 case ISD::SETOLE:
18770 case ISD::SETULE:
18771 case ISD::SETGT:
18772 case ISD::SETOGT:
18773 case ISD::SETUGT: {
18774 // OLE should be false if either operand is NaN, but this cannot be
18775 // ensured with a truncated comparison.
18776 if (CC == ISD::SETOLE && (LHSMaybeNaN || RHSMaybeNaN))
18777 break;
18778 // ULE should be true if either operand is NaN.
18779 // False negatives are only possible if the LHS is NaN.
18780 // (No issue with LHS == +inf since the inequality is inclusive)
18781 if (CC == ISD::SETULE && LHSMaybeNaN)
18782 break;
18783 // OGT should be false if either operand is NaN.
18784 // False positives are only possible if the LHS is NaN.
18785 // (No issue with LHS == +inf since the inequality is strict)
18786 if (CC == ISD::SETOGT && LHSMaybeNaN)
18787 break;
18788 // UGT should be true if either operand is NaN, but this cannot be ensured
18789 // with a truncated comparison.
18790 if (CC == ISD::SETUGT && (LHSMaybeNaN || RHSMaybeNaN))
18791 break;
18792
18793 const std::optional<bool> KnownUle =
18794 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18795
18796 if (!KnownUle)
18797 break;
18798
18799 if (*KnownUle) {
18800 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18801 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18802 ? ISD::SETLE
18803 : ISD::SETGT;
18804 }
18805 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18806 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18807 ? ISD::SETLT
18808 : ISD::SETGE;
18809 }
18810 }
18811
18812 return ISD::SETCC_INVALID;
18813}
18814
18815SDValue SITargetLowering::performSetCCCombine(SDNode *N,
18816 DAGCombinerInfo &DCI) const {
18817 SelectionDAG &DAG = DCI.DAG;
18818 SDLoc SL(N);
18819
18820 SDValue LHS = N->getOperand(Num: 0);
18821 SDValue RHS = N->getOperand(Num: 1);
18822 EVT VT = LHS.getValueType();
18823 ISD::CondCode CC = N->getOperandCondCode(Num: 2);
18824
18825 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
18826 if (!CRHS) {
18827 CRHS = dyn_cast<ConstantSDNode>(Val&: LHS);
18828 if (CRHS) {
18829 std::swap(a&: LHS, b&: RHS);
18830 CC = getSetCCSwappedOperands(Operation: CC);
18831 }
18832 }
18833
18834 if (CRHS) {
18835 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND &&
18836 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18837 // setcc (sext from i1 cc), -1, ne|sgt|ult) => not cc => xor cc, -1
18838 // setcc (sext from i1 cc), -1, eq|sle|uge) => cc
18839 // setcc (sext from i1 cc), 0, eq|sge|ule) => not cc => xor cc, -1
18840 // setcc (sext from i1 cc), 0, ne|ugt|slt) => cc
18841 if ((CRHS->isAllOnes() &&
18842 (CC == ISD::SETNE || CC == ISD::SETGT || CC == ISD::SETULT)) ||
18843 (CRHS->isZero() &&
18844 (CC == ISD::SETEQ || CC == ISD::SETGE || CC == ISD::SETULE)))
18845 return DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
18846 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i1));
18847 if ((CRHS->isAllOnes() &&
18848 (CC == ISD::SETEQ || CC == ISD::SETLE || CC == ISD::SETUGE)) ||
18849 (CRHS->isZero() &&
18850 (CC == ISD::SETNE || CC == ISD::SETUGT || CC == ISD::SETLT)))
18851 return LHS.getOperand(i: 0);
18852 }
18853
18854 const APInt &CRHSVal = CRHS->getAPIntValue();
18855 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
18856 LHS.getOpcode() == ISD::SELECT &&
18857 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
18858 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2)) &&
18859 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18860 // Given CT != FT:
18861 // setcc (select cc, CT, CF), CF, eq => xor cc, -1
18862 // setcc (select cc, CT, CF), CF, ne => cc
18863 // setcc (select cc, CT, CF), CT, ne => xor cc, -1
18864 // setcc (select cc, CT, CF), CT, eq => cc
18865 const APInt &CT = LHS.getConstantOperandAPInt(i: 1);
18866 const APInt &CF = LHS.getConstantOperandAPInt(i: 2);
18867
18868 if (CT != CF) {
18869 if ((CF == CRHSVal && CC == ISD::SETEQ) ||
18870 (CT == CRHSVal && CC == ISD::SETNE))
18871 return DAG.getNOT(DL: SL, Val: LHS.getOperand(i: 0), VT: MVT::i1);
18872 if ((CF == CRHSVal && CC == ISD::SETNE) ||
18873 (CT == CRHSVal && CC == ISD::SETEQ))
18874 return LHS.getOperand(i: 0);
18875 }
18876 }
18877 }
18878
18879 // Truncate 64-bit setcc to only test upper or lower 32-bits.
18880 if (VT == MVT::i64) {
18881 const KnownBits LHSKnown = DAG.computeKnownBits(Op: LHS);
18882 const KnownBits RHSKnown = DAG.computeKnownBits(Op: RHS);
18883 const KnownBits LHSKnownLo32 = LHSKnown.trunc(BitWidth: 32);
18884 const KnownBits RHSKnownLo32 = RHSKnown.trunc(BitWidth: 32);
18885 // Truncate 64-bit setcc to test only upper 32-bits of its operands in the
18886 // following cases where information about the lower 32-bits of its operands
18887 // is known:
18888 //
18889 // If LHS.lo32 == RHS.lo32:
18890 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18891 // If LHS.lo32 != RHS.lo32:
18892 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18893 // If LHS.lo32 >= RHS.lo32 (unsigned):
18894 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18895 // If LHS.lo32 > RHS.lo32 (unsigned):
18896 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18897 // If LHS.lo32 <= RHS.lo32 (unsigned):
18898 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18899 // If LHS.lo32 < RHS.lo32 (unsigned):
18900 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18901
18902 // NewCC is valid iff we can truncate the setcc to only test the upper 32
18903 // bits
18904 ISD::CondCode NewCC = ISD::SETCC_INVALID;
18905
18906 switch (CC) {
18907 default:
18908 llvm_unreachable("Unexpected condition code for integer setcc");
18909 break;
18910 case ISD::SETEQ: {
18911 const std::optional<bool> KnownEq =
18912 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18913 if (KnownEq)
18914 NewCC = *KnownEq ? ISD::SETEQ : ISD::SETFALSE;
18915
18916 break;
18917 }
18918 case ISD::SETNE: {
18919 const std::optional<bool> KnownEq =
18920 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18921 if (KnownEq)
18922 NewCC = *KnownEq ? ISD::SETNE : ISD::SETTRUE;
18923
18924 break;
18925 }
18926 case ISD::SETULT:
18927 case ISD::SETUGE:
18928 case ISD::SETLT:
18929 case ISD::SETGE: {
18930 const std::optional<bool> KnownUge =
18931 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18932 if (KnownUge) {
18933 if (*KnownUge) {
18934 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18935 NewCC = CC;
18936 } else {
18937 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18938 NewCC = CC == ISD::SETULT ? ISD::SETULE
18939 : CC == ISD::SETUGE ? ISD::SETUGT
18940 : CC == ISD::SETLT ? ISD::SETLE
18941 : ISD::SETGT;
18942 }
18943 }
18944 break;
18945 }
18946 case ISD::SETULE:
18947 case ISD::SETUGT:
18948 case ISD::SETLE:
18949 case ISD::SETGT: {
18950 const std::optional<bool> KnownUle =
18951 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18952 if (KnownUle) {
18953 if (*KnownUle) {
18954 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18955 NewCC = CC;
18956 } else {
18957 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18958 NewCC = CC == ISD::SETULE ? ISD::SETULT
18959 : CC == ISD::SETUGT ? ISD::SETUGE
18960 : CC == ISD::SETLE ? ISD::SETLT
18961 : ISD::SETGE;
18962 }
18963 }
18964 break;
18965 }
18966 }
18967
18968 if (NewCC != ISD::SETCC_INVALID)
18969 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
18970 RHS: getHiHalf64(Op: RHS, DAG), Cond: NewCC);
18971
18972 // Truncate 64-bit setcc to test only lower 32-bits when upper 32-bits are
18973 // known equal.
18974 const KnownBits LHSKnownHi32 = LHSKnown.extractBits(NumBits: 32, BitPosition: 32);
18975 const KnownBits RHSKnownHi32 = RHSKnown.extractBits(NumBits: 32, BitPosition: 32);
18976 const std::optional<bool> KnownEq =
18977 KnownBits::eq(LHS: LHSKnownHi32, RHS: RHSKnownHi32);
18978
18979 if (KnownEq == true) {
18980 // For signed comparisons, convert to unsigned since upper halves match
18981 ISD::CondCode NewCC = ISD::getUnsignedSetCC(Code: CC);
18982
18983 SDValue LHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: LHS);
18984 SDValue RHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: RHS);
18985 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: LHSLo, RHS: RHSLo, Cond: NewCC);
18986 }
18987 }
18988
18989 // Eliminate setcc by using carryout from add/sub instruction
18990
18991 // LHS = ADD i64 RHS, Z LHSlo = UADDO i32 RHSlo, Zlo
18992 // setcc LHS ult RHS -> LHSHi = UADDO_CARRY i32 RHShi, Zhi
18993 // similarly for subtraction
18994
18995 // LHS = ADD i64 Y, 1 LHSlo = UADDO i32 Ylo, 1
18996 // setcc LHS eq 0 -> LHSHi = UADDO_CARRY i32 Yhi, 0
18997
18998 if (VT == MVT::i64 && ((CC == ISD::SETULT &&
18999 sd_match(N: LHS, P: m_Add(L: m_Specific(N: RHS), R: m_Value()))) ||
19000 (CC == ISD::SETUGT &&
19001 sd_match(N: LHS, P: m_Sub(L: m_Specific(N: RHS), R: m_Value()))) ||
19002 (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
19003 sd_match(N: LHS, P: m_Add(L: m_Value(), R: m_One()))))) {
19004 bool IsAdd = LHS.getOpcode() == ISD::ADD;
19005
19006 SDValue Op0 = LHS.getOperand(i: 0);
19007 SDValue Op1 = LHS.getOperand(i: 1);
19008
19009 SDValue Op0Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op0);
19010 SDValue Op1Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op1);
19011
19012 SDValue Op0Hi = getHiHalf64(Op: Op0, DAG);
19013 SDValue Op1Hi = getHiHalf64(Op: Op1, DAG);
19014
19015 SDValue NodeLo =
19016 DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: SL,
19017 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1), Ops: {Op0Lo, Op1Lo});
19018
19019 SDValue CarryInHi = NodeLo.getValue(R: 1);
19020 SDValue NodeHi = DAG.getNode(Opcode: IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
19021 DL: SL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1),
19022 Ops: {Op0Hi, Op1Hi, CarryInHi});
19023
19024 SDValue ResultLo = NodeLo.getValue(R: 0);
19025 SDValue ResultHi = NodeHi.getValue(R: 0);
19026
19027 SDValue JoinedResult =
19028 DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {ResultLo, ResultHi});
19029
19030 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: JoinedResult);
19031 SDValue Overflow = NodeHi.getValue(R: 1);
19032 DCI.CombineTo(N: LHS.getNode(), Res: Result);
19033 return Overflow;
19034 }
19035
19036 if (VT != MVT::f32 && VT != MVT::f64 &&
19037 (!Subtarget->has16BitInsts() || VT != MVT::f16))
19038 return SDValue();
19039
19040 // Match isinf/isfinite pattern
19041 // (fcmp oeq (fabs x), inf) -> (fp_class x, (p_infinity | n_infinity))
19042 // (fcmp one (fabs x), inf) -> (fp_class x,
19043 // (p_normal | n_normal | p_subnormal | n_subnormal | p_zero | n_zero)
19044 if ((CC == ISD::SETOEQ || CC == ISD::SETONE) &&
19045 LHS.getOpcode() == ISD::FABS) {
19046 const ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
19047 if (!CRHS)
19048 return SDValue();
19049
19050 const APFloat &APF = CRHS->getValueAPF();
19051 if (APF.isInfinity() && !APF.isNegative()) {
19052 const unsigned IsInfMask =
19053 SIInstrFlags::P_INFINITY | SIInstrFlags::N_INFINITY;
19054 const unsigned IsFiniteMask =
19055 SIInstrFlags::N_ZERO | SIInstrFlags::P_ZERO | SIInstrFlags::N_NORMAL |
19056 SIInstrFlags::P_NORMAL | SIInstrFlags::N_SUBNORMAL |
19057 SIInstrFlags::P_SUBNORMAL;
19058 unsigned Mask = CC == ISD::SETOEQ ? IsInfMask : IsFiniteMask;
19059 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
19060 N2: DAG.getConstant(Val: Mask, DL: SL, VT: MVT::i32));
19061 }
19062 }
19063
19064 if (VT == MVT::f64) {
19065 ISD::CondCode HiHalfCC = tryReduceF64CompareToHiHalf(CC, LHS, RHS, DAG);
19066 if (HiHalfCC != ISD::SETCC_INVALID)
19067 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
19068 RHS: getHiHalf64(Op: RHS, DAG), Cond: HiHalfCC);
19069 }
19070
19071 return SDValue();
19072}
19073
19074SDValue
19075SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
19076 DAGCombinerInfo &DCI) const {
19077 SelectionDAG &DAG = DCI.DAG;
19078 SDLoc SL(N);
19079 unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
19080
19081 SDValue Src = N->getOperand(Num: 0);
19082 SDValue Shift = N->getOperand(Num: 0);
19083
19084 // TODO: Extend type shouldn't matter (assuming legal types).
19085 if (Shift.getOpcode() == ISD::ZERO_EXTEND)
19086 Shift = Shift.getOperand(i: 0);
19087
19088 if (Shift.getOpcode() == ISD::SRL || Shift.getOpcode() == ISD::SHL) {
19089 // cvt_f32_ubyte1 (shl x, 8) -> cvt_f32_ubyte0 x
19090 // cvt_f32_ubyte3 (shl x, 16) -> cvt_f32_ubyte1 x
19091 // cvt_f32_ubyte0 (srl x, 16) -> cvt_f32_ubyte2 x
19092 // cvt_f32_ubyte1 (srl x, 16) -> cvt_f32_ubyte3 x
19093 // cvt_f32_ubyte0 (srl x, 8) -> cvt_f32_ubyte1 x
19094 if (auto *C = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1))) {
19095 SDValue Shifted = DAG.getZExtOrTrunc(
19096 Op: Shift.getOperand(i: 0), DL: SDLoc(Shift.getOperand(i: 0)), VT: MVT::i32);
19097
19098 unsigned ShiftOffset = 8 * Offset;
19099 if (Shift.getOpcode() == ISD::SHL)
19100 ShiftOffset -= C->getZExtValue();
19101 else
19102 ShiftOffset += C->getZExtValue();
19103
19104 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
19105 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, DL: SL,
19106 VT: MVT::f32, Operand: Shifted);
19107 }
19108 }
19109 }
19110
19111 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19112 APInt DemandedBits = APInt::getBitsSet(numBits: 32, loBit: 8 * Offset, hiBit: 8 * Offset + 8);
19113 if (TLI.SimplifyDemandedBits(Op: Src, DemandedBits, DCI)) {
19114 // We simplified Src. If this node is not dead, visit it again so it is
19115 // folded properly.
19116 if (N->getOpcode() != ISD::DELETED_NODE)
19117 DCI.AddToWorklist(N);
19118 return SDValue(N, 0);
19119 }
19120
19121 // Handle (or x, (srl y, 8)) pattern when known bits are zero.
19122 if (SDValue DemandedSrc =
19123 TLI.SimplifyMultipleUseDemandedBits(Op: Src, DemandedBits, DAG))
19124 return DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: MVT::f32, Operand: DemandedSrc);
19125
19126 return SDValue();
19127}
19128
19129SDValue SITargetLowering::performClampCombine(SDNode *N,
19130 DAGCombinerInfo &DCI) const {
19131 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
19132 if (!CSrc)
19133 return SDValue();
19134
19135 const MachineFunction &MF = DCI.DAG.getMachineFunction();
19136 const APFloat &F = CSrc->getValueAPF();
19137 APFloat Zero = APFloat::getZero(Sem: F.getSemantics());
19138 if (F < Zero ||
19139 (F.isNaN() && MF.getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) {
19140 return DCI.DAG.getConstantFP(Val: Zero, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
19141 }
19142
19143 APFloat One = APFloat::getOne(Sem: F.getSemantics());
19144 if (F > One)
19145 return DCI.DAG.getConstantFP(Val: One, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
19146
19147 return getCanonicalConstantFP(DAG&: DCI.DAG, SL: SDLoc(N), VT: N->getValueType(ResNo: 0), C: F);
19148}
19149
19150// Check if V is the exponent result of a frexp operation. Returns the frexp
19151// input via FrexpInput if matched. We only match the exponent (not mantissa)
19152// because V_FREXP_MANT returns its input for Inf/NaN, not zero.
19153static bool isFrexpExp(SDValue V, SDValue &FrexpInput) {
19154 // ISD::FFREXP returns {mant, exp} - only match if using the exp result
19155 // (result number 1).
19156 if (V.getOpcode() == ISD::FFREXP && V.getResNo() == 1) {
19157 FrexpInput = V.getOperand(i: 0);
19158 return true;
19159 }
19160 if (sd_match(N: V, P: m_IntrinsicWOChain<Intrinsic::amdgcn_frexp_exp>(
19161 Opnds: m_Value(N&: FrexpInput))))
19162 return true;
19163 return false;
19164}
19165
19166SDValue
19167SITargetLowering::performFrexpSelectCombine(SDNode *N,
19168 DAGCombinerInfo &DCI) const {
19169 // This optimization only applies when the hardware handles inf/nan correctly.
19170 if (Subtarget->hasFractBug())
19171 return SDValue();
19172
19173 SDValue Cond = N->getOperand(Num: 0);
19174 SDValue TrueVal = N->getOperand(Num: 1);
19175 SDValue FalseVal = N->getOperand(Num: 2);
19176
19177 // Identify which operand is the frexp result and which is the zero constant.
19178 // Pattern 1: select cond, 0, frexp_result (cond true -> return 0)
19179 // Pattern 2: select cond, frexp_result, 0 (cond false -> return 0)
19180 SDValue FrexpVal;
19181 SDValue ZeroVal;
19182 bool CondSelectsZero; // If true, condition=true selects zero
19183
19184 // Check if FrexpVal comes from ISD::FFREXP (exponent result only) or
19185 // amdgcn_frexp_exp intrinsic.
19186 SDValue FrexpInput;
19187 if (isFrexpExp(V: FalseVal, FrexpInput)) {
19188 FrexpVal = FalseVal;
19189 ZeroVal = TrueVal;
19190 CondSelectsZero = true;
19191 } else if (isFrexpExp(V: TrueVal, FrexpInput)) {
19192 FrexpVal = TrueVal;
19193 ZeroVal = FalseVal;
19194 CondSelectsZero = false;
19195 } else {
19196 return SDValue();
19197 }
19198
19199 // frexp_exp returns integer, so check for integer zero.
19200 if (!isNullConstant(V: ZeroVal))
19201 return SDValue();
19202
19203 // The frexp intrinsics ignore sign, so we can strip sign ops when comparing.
19204 SDValue FrexpInputStripped = peekFPSignOps(Val: FrexpInput);
19205
19206 bool IsNonFiniteTest = false;
19207
19208 // Handle SETCC conditions for inf/nan tests.
19209 // The canonical form of these checks is fcmp + fabs.
19210 if (Cond.getOpcode() == ISD::SETCC) {
19211 ISD::CondCode CC = Cond.getOperandCondCode(i: 2);
19212 SDValue CondLHS = Cond.getOperand(i: 0);
19213 SDValue CondRHS = Cond.getOperand(i: 1);
19214
19215 // Check if LHS is fabs(FrexpInput) - required for infinity comparisons.
19216 SDValue FAbsInput;
19217 bool LHSIsFabs = sd_match(N: CondLHS, P: m_FAbs(Op: m_Value(N&: FAbsInput)));
19218 bool LHSMatchesFrexp =
19219 (CondLHS == FrexpInput) ||
19220 (LHSIsFabs && peekFPSignOps(Val: FAbsInput) == FrexpInputStripped) ||
19221 (peekFPSignOps(Val: CondLHS) == FrexpInputStripped);
19222 bool RHSMatchesFrexp = (CondRHS == FrexpInput) ||
19223 (peekFPSignOps(Val: CondRHS) == FrexpInputStripped);
19224
19225 if (CC == ISD::SETUO) {
19226 // fcmp uno x, y - true if either x or y is NaN
19227 // We can only fold if the non-frexp operand is known to never be NaN,
19228 // otherwise the comparison could be true due to the other operand.
19229 // Special case: fcmp uno x, x (same operand) is a valid NaN test.
19230 SelectionDAG &DAG = DCI.DAG;
19231 if (LHSMatchesFrexp &&
19232 (CondLHS == CondRHS || DAG.isKnownNeverNaN(Op: CondRHS)))
19233 IsNonFiniteTest = CondSelectsZero;
19234 else if (RHSMatchesFrexp && DAG.isKnownNeverNaN(Op: CondLHS))
19235 IsNonFiniteTest = CondSelectsZero;
19236 } else if ((CC == ISD::SETOEQ || CC == ISD::SETUEQ) && LHSMatchesFrexp &&
19237 LHSIsFabs &&
19238 sd_match(N: CondRHS,
19239 P: m_SpecificFP(V: APFloat::getInf(
19240 Sem: CondRHS.getValueType().getFltSemantics())))) {
19241 // fcmp oeq/ueq fabs(x), +inf - true if x is inf (or inf/nan for ueq)
19242 IsNonFiniteTest = CondSelectsZero;
19243 } else if ((CC == ISD::SETONE || CC == ISD::SETUNE) && LHSMatchesFrexp &&
19244 LHSIsFabs &&
19245 sd_match(N: CondRHS,
19246 P: m_SpecificFP(V: APFloat::getInf(
19247 Sem: CondRHS.getValueType().getFltSemantics())))) {
19248 // fcmp one/une fabs(x), +inf - true if x is NOT inf
19249 IsNonFiniteTest = !CondSelectsZero;
19250 } else if (CC == ISD::SETO) {
19251 // fcmp ord x, y - true if both are NOT NaN
19252 // We can only fold if the non-frexp operand is known to never be NaN,
19253 // otherwise the comparison could be false due to the other operand.
19254 // Special case: fcmp ord x, x (same operand) is a valid not-NaN test.
19255 SelectionDAG &DAG = DCI.DAG;
19256 if (LHSMatchesFrexp &&
19257 (CondLHS == CondRHS || DAG.isKnownNeverNaN(Op: CondRHS)))
19258 IsNonFiniteTest = !CondSelectsZero;
19259 else if (RHSMatchesFrexp && DAG.isKnownNeverNaN(Op: CondLHS))
19260 IsNonFiniteTest = !CondSelectsZero;
19261 }
19262 }
19263
19264 if (!IsNonFiniteTest)
19265 return SDValue();
19266
19267 // The select can be eliminated - just return the frexp result directly.
19268 return FrexpVal;
19269}
19270
19271SDValue SITargetLowering::performSelectCombine(SDNode *N,
19272 DAGCombinerInfo &DCI) const {
19273
19274 // Try to fold CMP + SELECT patterns with shared constants (both FP and
19275 // integer).
19276 // Detect when CMP and SELECT use the same constant and fold them to avoid
19277 // loading the constant twice. Specifically handles patterns like:
19278 // %cmp = icmp eq i32 %val, 4242
19279 // %sel = select i1 %cmp, i32 4242, i32 %other
19280 // It can be optimized to reuse %val instead of 4242 in select.
19281 SDValue Cond = N->getOperand(Num: 0);
19282 SDValue TrueVal = N->getOperand(Num: 1);
19283 SDValue FalseVal = N->getOperand(Num: 2);
19284
19285 // Check if condition is a comparison.
19286 if (Cond.getOpcode() != ISD::SETCC)
19287 return SDValue();
19288
19289 SDValue LHS = Cond.getOperand(i: 0);
19290 SDValue RHS = Cond.getOperand(i: 1);
19291 ISD::CondCode CC = Cond.getOperandCondCode(i: 2);
19292
19293 bool isFloatingPoint = LHS.getValueType().isFloatingPoint();
19294 bool isInteger = LHS.getValueType().isInteger();
19295
19296 // Handle simple floating-point and integer types only.
19297 if (!isFloatingPoint && !isInteger)
19298 return SDValue();
19299
19300 // Bare SETEQ/SETNE is the builder's NaN-impossible downgrade.
19301 bool isEquality = CC == ISD::SETEQ || (isFloatingPoint && CC == ISD::SETOEQ);
19302 bool isNonEquality =
19303 CC == ISD::SETNE || (isFloatingPoint && CC == ISD::SETONE);
19304 if (!isEquality && !isNonEquality)
19305 return SDValue();
19306
19307 SDValue ArgVal, ConstVal;
19308 if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: RHS)) ||
19309 (isInteger && isa<ConstantSDNode>(Val: RHS))) {
19310 ConstVal = RHS;
19311 ArgVal = LHS;
19312 } else if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: LHS)) ||
19313 (isInteger && isa<ConstantSDNode>(Val: LHS))) {
19314 ConstVal = LHS;
19315 ArgVal = RHS;
19316 } else {
19317 return SDValue();
19318 }
19319
19320 // Skip optimization for inlinable immediates.
19321 if (isFloatingPoint) {
19322 const APFloat &Val = cast<ConstantFPSDNode>(Val&: ConstVal)->getValueAPF();
19323 if (!Val.isNormal() || Subtarget->getInstrInfo()->isInlineConstant(Imm: Val))
19324 return SDValue();
19325 } else {
19326 const std::optional<int64_t> Val =
19327 cast<ConstantSDNode>(Val&: ConstVal)->getAPIntValue().trySExtValue();
19328 if (Val && AMDGPU::isInlinableIntLiteral(Literal: *Val))
19329 return SDValue();
19330 }
19331
19332 // For equality and non-equality comparisons, patterns:
19333 // select (setcc x, const), const, y -> select (setcc x, const), x, y
19334 // select (setccinv x, const), y, const -> select (setccinv x, const), y, x
19335 if (!(isEquality && TrueVal == ConstVal) &&
19336 !(isNonEquality && FalseVal == ConstVal))
19337 return SDValue();
19338
19339 // SETONE's false arm is also taken for NaN ArgVal, so require NaN excluded.
19340 if (isFloatingPoint && isNonEquality && FalseVal == ConstVal &&
19341 !Cond->getFlags().hasNoNaNs() && !DCI.DAG.isKnownNeverNaN(Op: ArgVal))
19342 return SDValue();
19343
19344 SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
19345 SDValue SelectRHS =
19346 (isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
19347 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Cond,
19348 N2: SelectLHS, N3: SelectRHS);
19349}
19350
19351SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
19352 DAGCombinerInfo &DCI) const {
19353 switch (N->getOpcode()) {
19354 case ISD::ABS:
19355 if (SDValue Res = promoteUniformUnaryOpToI32(Op: SDValue(N, 0), DCI))
19356 return Res;
19357 break;
19358 case ISD::ADD:
19359 case ISD::SUB:
19360 case ISD::SHL:
19361 case ISD::SRL:
19362 case ISD::SRA:
19363 case ISD::AND:
19364 case ISD::OR:
19365 case ISD::XOR:
19366 case ISD::MUL:
19367 case ISD::SETCC:
19368 case ISD::SELECT:
19369 case ISD::SMIN:
19370 case ISD::SMAX:
19371 case ISD::UMIN:
19372 case ISD::UMAX:
19373 case ISD::USUBSAT:
19374 case ISD::UADDSAT:
19375 if (auto Res = promoteUniformOpToI32(Op: SDValue(N, 0), DCI))
19376 return Res;
19377 break;
19378 case ISD::LOAD:
19379 case ISD::STORE:
19380 // TODO: Move this to LowerLOAD and LowerSTORE once the action of a
19381 // non-extending load or non-truncating store can depend on the address
19382 // space. Until then one whose action is Legal, such as i32, never reaches
19383 // them.
19384 if (cast<MemSDNode>(Val: N)->getAddressSpace() == AMDGPUAS::VGPR) {
19385 if (SDValue V = lowerLoadStoreVGPR(Op: SDValue(N, 0), DAG&: DCI.DAG))
19386 return V;
19387 }
19388 break;
19389 default:
19390 break;
19391 }
19392
19393 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
19394 return SDValue();
19395
19396 switch (N->getOpcode()) {
19397 case ISD::ADD:
19398 return performAddCombine(N, DCI);
19399 case ISD::PTRADD:
19400 return performPtrAddCombine(N, DCI);
19401 case ISD::SUB:
19402 return performSubCombine(N, DCI);
19403 case ISD::FADD:
19404 return performFAddCombine(N, DCI);
19405 case ISD::FSUB:
19406 return performFSubCombine(N, DCI);
19407 case ISD::FDIV:
19408 return performFDivCombine(N, DCI);
19409 case ISD::FMUL:
19410 return performFMulCombine(N, DCI);
19411 case ISD::SETCC:
19412 return performSetCCCombine(N, DCI);
19413 case ISD::SELECT:
19414 if (auto Res = performFrexpSelectCombine(N, DCI))
19415 return Res;
19416 if (auto Res = performSelectCombine(N, DCI))
19417 return Res;
19418 break;
19419 case ISD::FMAXNUM:
19420 case ISD::FMINNUM:
19421 case ISD::FMAXNUM_IEEE:
19422 case ISD::FMINNUM_IEEE:
19423 case ISD::FMAXIMUM:
19424 case ISD::FMINIMUM:
19425 case ISD::FMAXIMUMNUM:
19426 case ISD::FMINIMUMNUM:
19427 case ISD::SMAX:
19428 case ISD::SMIN:
19429 case ISD::UMAX:
19430 case ISD::UMIN:
19431 case AMDGPUISD::FMIN_LEGACY:
19432 case AMDGPUISD::FMAX_LEGACY:
19433 return performMinMaxCombine(N, DCI);
19434 case ISD::FMA:
19435 return performFMACombine(N, DCI);
19436 case ISD::AND:
19437 return performAndCombine(N, DCI);
19438 case ISD::OR:
19439 return performOrCombine(N, DCI);
19440 case ISD::FSHR: {
19441 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19442 if (N->getValueType(ResNo: 0) == MVT::i32 && N->isDivergent() &&
19443 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
19444 return matchPERM(N, DCI);
19445 }
19446 break;
19447 }
19448 case ISD::XOR:
19449 return performXorCombine(N, DCI);
19450 case ISD::ANY_EXTEND:
19451 case ISD::ZERO_EXTEND:
19452 return performZeroOrAnyExtendCombine(N, DCI);
19453 case ISD::SIGN_EXTEND:
19454 return performExtendTruncShiftCombine(N, DCI);
19455 case ISD::SIGN_EXTEND_INREG:
19456 return performSignExtendInRegCombine(N, DCI);
19457 case AMDGPUISD::FP_CLASS:
19458 return performClassCombine(N, DCI);
19459 case ISD::FCANONICALIZE:
19460 return performFCanonicalizeCombine(N, DCI);
19461 case AMDGPUISD::RCP:
19462 return performRcpCombine(N, DCI);
19463 case ISD::FLDEXP:
19464 case AMDGPUISD::FRACT:
19465 case AMDGPUISD::RSQ:
19466 case AMDGPUISD::RCP_LEGACY:
19467 case AMDGPUISD::RCP_IFLAG:
19468 case AMDGPUISD::RSQ_CLAMP: {
19469 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
19470 SDValue Src = N->getOperand(Num: 0);
19471 if (Src.isUndef())
19472 return Src;
19473 break;
19474 }
19475 case ISD::SINT_TO_FP:
19476 case ISD::UINT_TO_FP:
19477 return performUCharToFloatCombine(N, DCI);
19478 case ISD::FCOPYSIGN:
19479 return performFCopySignCombine(N, DCI);
19480 case AMDGPUISD::CVT_F32_UBYTE0:
19481 case AMDGPUISD::CVT_F32_UBYTE1:
19482 case AMDGPUISD::CVT_F32_UBYTE2:
19483 case AMDGPUISD::CVT_F32_UBYTE3:
19484 return performCvtF32UByteNCombine(N, DCI);
19485 case AMDGPUISD::FMED3:
19486 return performFMed3Combine(N, DCI);
19487 case AMDGPUISD::CVT_PKRTZ_F16_F32:
19488 return performCvtPkRTZCombine(N, DCI);
19489 case ISD::INTRINSIC_WO_CHAIN:
19490 if (SDValue Res = performExponentOnlyScaleCombine(N, DCI))
19491 return Res;
19492 break;
19493 case AMDGPUISD::CLAMP:
19494 return performClampCombine(N, DCI);
19495 case ISD::SCALAR_TO_VECTOR: {
19496 SelectionDAG &DAG = DCI.DAG;
19497 EVT VT = N->getValueType(ResNo: 0);
19498
19499 // v2i16 (scalar_to_vector i16:x) -> v2i16 (bitcast (any_extend i16:x))
19500 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
19501 SDLoc SL(N);
19502 SDValue Src = N->getOperand(Num: 0);
19503 EVT EltVT = Src.getValueType();
19504 if (EltVT != MVT::i16)
19505 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
19506
19507 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Src);
19508 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Ext);
19509 }
19510
19511 break;
19512 }
19513 case ISD::EXTRACT_VECTOR_ELT:
19514 return performExtractVectorEltCombine(N, DCI);
19515 case ISD::INSERT_VECTOR_ELT:
19516 return performInsertVectorEltCombine(N, DCI);
19517 case ISD::FP_ROUND:
19518 return performFPRoundCombine(N, DCI);
19519 case ISD::LOAD: {
19520 if (SDValue Widened = widenLoad(Ld: cast<LoadSDNode>(Val: N), DCI))
19521 return Widened;
19522 [[fallthrough]];
19523 }
19524 default: {
19525 if (!DCI.isBeforeLegalize()) {
19526 if (MemSDNode *MemNode = dyn_cast<MemSDNode>(Val: N))
19527 return performMemSDNodeCombine(N: MemNode, DCI);
19528 }
19529
19530 break;
19531 }
19532 }
19533
19534 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
19535}
19536
19537/// Helper function for adjustWritemask
19538static unsigned SubIdx2Lane(unsigned Idx) {
19539 switch (Idx) {
19540 default:
19541 return ~0u;
19542 case AMDGPU::sub0:
19543 return 0;
19544 case AMDGPU::sub1:
19545 return 1;
19546 case AMDGPU::sub2:
19547 return 2;
19548 case AMDGPU::sub3:
19549 return 3;
19550 case AMDGPU::sub4:
19551 return 4; // Possible with TFE/LWE
19552 }
19553}
19554
19555/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
19556SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
19557 SelectionDAG &DAG) const {
19558 unsigned Opcode = Node->getMachineOpcode();
19559
19560 // Subtract 1 because the vdata output is not a MachineSDNode operand.
19561 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::d16) - 1;
19562 if (D16Idx >= 0 && Node->getConstantOperandVal(Num: D16Idx))
19563 return Node; // not implemented for D16
19564
19565 SDNode *Users[5] = {nullptr};
19566 unsigned Lane = 0;
19567 unsigned DmaskIdx =
19568 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::dmask) - 1;
19569 unsigned OldDmask = Node->getConstantOperandVal(Num: DmaskIdx);
19570 unsigned NewDmask = 0;
19571 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::tfe) - 1;
19572 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::lwe) - 1;
19573 bool UsesTFC = (int(TFEIdx) >= 0 && Node->getConstantOperandVal(Num: TFEIdx)) ||
19574 (int(LWEIdx) >= 0 && Node->getConstantOperandVal(Num: LWEIdx));
19575 unsigned TFCLane = 0;
19576 bool HasChain = Node->getNumValues() > 1;
19577
19578 if (OldDmask == 0) {
19579 // These are folded out, but on the chance it happens don't assert.
19580 return Node;
19581 }
19582
19583 unsigned OldBitsSet = llvm::popcount(Value: OldDmask);
19584 // Work out which is the TFE/LWE lane if that is enabled.
19585 if (UsesTFC) {
19586 TFCLane = OldBitsSet;
19587 }
19588
19589 // Try to figure out the used register components
19590 for (SDUse &Use : Node->uses()) {
19591
19592 // Don't look at users of the chain.
19593 if (Use.getResNo() != 0)
19594 continue;
19595
19596 SDNode *User = Use.getUser();
19597
19598 // Abort if we can't understand the usage
19599 if (!User->isMachineOpcode() ||
19600 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
19601 return Node;
19602
19603 // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
19604 // Note that subregs are packed, i.e. Lane==0 is the first bit set
19605 // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
19606 // set, etc.
19607 Lane = SubIdx2Lane(Idx: User->getConstantOperandVal(Num: 1));
19608 if (Lane == ~0u)
19609 return Node;
19610
19611 // Check if the use is for the TFE/LWE generated result at VGPRn+1.
19612 if (UsesTFC && Lane == TFCLane) {
19613 Users[Lane] = User;
19614 } else {
19615 // Set which texture component corresponds to the lane.
19616 unsigned Comp;
19617 for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
19618 Comp = llvm::countr_zero(Val: Dmask);
19619 Dmask &= ~(1 << Comp);
19620 }
19621
19622 // Abort if we have more than one user per component.
19623 if (Users[Lane])
19624 return Node;
19625
19626 Users[Lane] = User;
19627 NewDmask |= 1 << Comp;
19628 }
19629 }
19630
19631 // Don't allow 0 dmask, as hardware assumes one channel enabled.
19632 bool NoChannels = !NewDmask;
19633 if (NoChannels) {
19634 if (!UsesTFC) {
19635 // No uses of the result and not using TFC. Then do nothing.
19636 return Node;
19637 }
19638 // If the original dmask has one channel - then nothing to do
19639 if (OldBitsSet == 1)
19640 return Node;
19641 // Use an arbitrary dmask - required for the instruction to work
19642 NewDmask = 1;
19643 }
19644 // Abort if there's no change
19645 if (NewDmask == OldDmask)
19646 return Node;
19647
19648 unsigned BitsSet = llvm::popcount(Value: NewDmask);
19649
19650 // Check for TFE or LWE - increase the number of channels by one to account
19651 // for the extra return value
19652 // This will need adjustment for D16 if this is also included in
19653 // adjustWriteMask (this function) but at present D16 are excluded.
19654 unsigned NewChannels = BitsSet + UsesTFC;
19655
19656 int NewOpcode =
19657 AMDGPU::getMaskedMIMGOp(Opc: Node->getMachineOpcode(), NewChannels);
19658 assert(NewOpcode != -1 &&
19659 NewOpcode != static_cast<int>(Node->getMachineOpcode()) &&
19660 "failed to find equivalent MIMG op");
19661
19662 // Adjust the writemask in the node
19663 SmallVector<SDValue, 12> Ops;
19664 llvm::append_range(C&: Ops, R: Node->ops().take_front(N: DmaskIdx));
19665 Ops.push_back(Elt: DAG.getTargetConstant(Val: NewDmask, DL: SDLoc(Node), VT: MVT::i32));
19666 llvm::append_range(C&: Ops, R: Node->ops().drop_front(N: DmaskIdx + 1));
19667
19668 MVT SVT = Node->getValueType(ResNo: 0).getVectorElementType().getSimpleVT();
19669
19670 MVT ResultVT = NewChannels == 1
19671 ? SVT
19672 : MVT::getVectorVT(VT: SVT, NumElements: NewChannels == 3 ? 4
19673 : NewChannels == 5 ? 8
19674 : NewChannels);
19675 SDVTList NewVTList =
19676 HasChain ? DAG.getVTList(VT1: ResultVT, VT2: MVT::Other) : DAG.getVTList(VT: ResultVT);
19677
19678 MachineSDNode *NewNode =
19679 DAG.getMachineNode(Opcode: NewOpcode, dl: SDLoc(Node), VTs: NewVTList, Ops);
19680
19681 if (HasChain) {
19682 // Update chain.
19683 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: Node->memoperands());
19684 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: SDValue(NewNode, 1));
19685 }
19686
19687 if (NewChannels == 1) {
19688 assert(Node->hasNUsesOfValue(1, 0));
19689 SDNode *Copy =
19690 DAG.getMachineNode(Opcode: TargetOpcode::COPY, dl: SDLoc(Node),
19691 VT: Users[Lane]->getValueType(ResNo: 0), Op1: SDValue(NewNode, 0));
19692 DAG.ReplaceAllUsesWith(From: Users[Lane], To: Copy);
19693 return nullptr;
19694 }
19695
19696 // Update the users of the node with the new indices
19697 for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
19698 SDNode *User = Users[i];
19699 if (!User) {
19700 // Handle the special case of NoChannels. We set NewDmask to 1 above, but
19701 // Users[0] is still nullptr because channel 0 doesn't really have a use.
19702 if (i || !NoChannels)
19703 continue;
19704 } else {
19705 SDValue Op = DAG.getTargetConstant(Val: Idx, DL: SDLoc(User), VT: MVT::i32);
19706 SDNode *NewUser = DAG.UpdateNodeOperands(N: User, Op1: SDValue(NewNode, 0), Op2: Op);
19707 if (NewUser != User) {
19708 DAG.ReplaceAllUsesWith(From: SDValue(User, 0), To: SDValue(NewUser, 0));
19709 DAG.RemoveDeadNode(N: User);
19710 }
19711 }
19712
19713 switch (Idx) {
19714 default:
19715 break;
19716 case AMDGPU::sub0:
19717 Idx = AMDGPU::sub1;
19718 break;
19719 case AMDGPU::sub1:
19720 Idx = AMDGPU::sub2;
19721 break;
19722 case AMDGPU::sub2:
19723 Idx = AMDGPU::sub3;
19724 break;
19725 case AMDGPU::sub3:
19726 Idx = AMDGPU::sub4;
19727 break;
19728 }
19729 }
19730
19731 DAG.RemoveDeadNode(N: Node);
19732 return nullptr;
19733}
19734
19735static bool isFrameIndexOp(SDValue Op) {
19736 if (Op.getOpcode() == ISD::AssertZext)
19737 Op = Op.getOperand(i: 0);
19738
19739 return isa<FrameIndexSDNode>(Val: Op);
19740}
19741
19742/// Legalize target independent instructions (e.g. INSERT_SUBREG)
19743/// with frame index operands.
19744/// LLVM assumes that inputs are to these instructions are registers.
19745SDNode *
19746SITargetLowering::legalizeTargetIndependentNode(SDNode *Node,
19747 SelectionDAG &DAG) const {
19748 if (Node->getOpcode() == ISD::CopyToReg) {
19749 RegisterSDNode *DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1));
19750 SDValue SrcVal = Node->getOperand(Num: 2);
19751
19752 // Insert a copy to a VReg_1 virtual register so LowerI1Copies doesn't have
19753 // to try understanding copies to physical registers.
19754 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) {
19755 SDLoc SL(Node);
19756 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19757 SDValue VReg = DAG.getRegister(
19758 Reg: MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_1RegClass), VT: MVT::i1);
19759
19760 SDNode *Glued = Node->getGluedNode();
19761 SDValue ToVReg = DAG.getCopyToReg(
19762 Chain: Node->getOperand(Num: 0), dl: SL, Reg: VReg, N: SrcVal,
19763 Glue: SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0));
19764 SDValue ToResultReg = DAG.getCopyToReg(Chain: ToVReg, dl: SL, Reg: SDValue(DestReg, 0),
19765 N: VReg, Glue: ToVReg.getValue(R: 1));
19766 DAG.ReplaceAllUsesWith(From: Node, To: ToResultReg.getNode());
19767 DAG.RemoveDeadNode(N: Node);
19768 return ToResultReg.getNode();
19769 }
19770 }
19771
19772 SmallVector<SDValue, 8> Ops;
19773 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
19774 if (!isFrameIndexOp(Op: Node->getOperand(Num: i))) {
19775 Ops.push_back(Elt: Node->getOperand(Num: i));
19776 continue;
19777 }
19778
19779 SDLoc DL(Node);
19780 Ops.push_back(Elt: SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL,
19781 VT: Node->getOperand(Num: i).getValueType(),
19782 Op1: Node->getOperand(Num: i)),
19783 0));
19784 }
19785
19786 return DAG.UpdateNodeOperands(N: Node, Ops);
19787}
19788
19789/// Fold the instructions after selecting them.
19790/// Returns null if users were already updated.
19791SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
19792 SelectionDAG &DAG) const {
19793 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19794 unsigned Opcode = Node->getMachineOpcode();
19795
19796 if (TII->isImage(Opcode) && !TII->get(Opcode).mayStore() &&
19797 !TII->isGather4(Opcode) &&
19798 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::dmask)) {
19799 return adjustWritemask(Node, DAG);
19800 }
19801
19802 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
19803 legalizeTargetIndependentNode(Node, DAG);
19804 return Node;
19805 }
19806
19807 switch (Opcode) {
19808 case AMDGPU::V_DIV_SCALE_F32_e64:
19809 case AMDGPU::V_DIV_SCALE_F64_e64: {
19810 // Satisfy the operand register constraint when one of the inputs is
19811 // undefined. Ordinarily each undef value will have its own implicit_def of
19812 // a vreg, so force these to use a single register.
19813 SDValue Src0 = Node->getOperand(Num: 1);
19814 SDValue Src1 = Node->getOperand(Num: 3);
19815 SDValue Src2 = Node->getOperand(Num: 5);
19816
19817 if ((Src0.isMachineOpcode() &&
19818 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) &&
19819 (Src0 == Src1 || Src0 == Src2))
19820 break;
19821
19822 MVT VT = Src0.getValueType().getSimpleVT();
19823 const TargetRegisterClass *RC =
19824 getRegClassFor(VT, isDivergent: Src0.getNode()->isDivergent());
19825
19826 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19827 SDValue UndefReg = DAG.getRegister(Reg: MRI.createVirtualRegister(RegClass: RC), VT);
19828
19829 SDValue ImpDef = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: SDLoc(Node), Reg: UndefReg,
19830 N: Src0, Glue: SDValue());
19831
19832 // src0 must be the same register as src1 or src2, even if the value is
19833 // undefined, so make sure we don't violate this constraint.
19834 if (Src0.isMachineOpcode() &&
19835 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) {
19836 if (Src1.isMachineOpcode() &&
19837 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19838 Src0 = Src1;
19839 else if (Src2.isMachineOpcode() &&
19840 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19841 Src0 = Src2;
19842 else {
19843 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF);
19844 Src0 = UndefReg;
19845 Src1 = UndefReg;
19846 }
19847 } else
19848 break;
19849
19850 SmallVector<SDValue, 9> Ops(Node->ops());
19851 Ops[1] = Src0;
19852 Ops[3] = Src1;
19853 Ops[5] = Src2;
19854 Ops.push_back(Elt: ImpDef.getValue(R: 1));
19855 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
19856 }
19857 default:
19858 break;
19859 }
19860
19861 return Node;
19862}
19863
19864// Any MIMG instructions that use tfe or lwe require an initialization of the
19865// result register that will be written in the case of a memory access failure.
19866// The required code is also added to tie this init code to the result of the
19867// img instruction.
19868void SITargetLowering::AddMemOpInit(MachineInstr &MI) const {
19869 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19870 const SIRegisterInfo &TRI = TII->getRegisterInfo();
19871 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
19872 MachineBasicBlock &MBB = *MI.getParent();
19873
19874 int DstIdx =
19875 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
19876 unsigned InitIdx = 0;
19877
19878 if (TII->isImage(MI)) {
19879 MachineOperand *TFE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
19880 MachineOperand *LWE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
19881 MachineOperand *D16 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
19882
19883 if (!TFE && !LWE) // intersect_ray
19884 return;
19885
19886 unsigned TFEVal = TFE ? TFE->getImm() : 0;
19887 unsigned LWEVal = LWE ? LWE->getImm() : 0;
19888 unsigned D16Val = D16 ? D16->getImm() : 0;
19889
19890 if (!TFEVal && !LWEVal)
19891 return;
19892
19893 // At least one of TFE or LWE are non-zero
19894 // We have to insert a suitable initialization of the result value and
19895 // tie this to the dest of the image instruction.
19896
19897 // Calculate which dword we have to initialize to 0.
19898 MachineOperand *MO_Dmask = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
19899
19900 // check that dmask operand is found.
19901 assert(MO_Dmask && "Expected dmask operand in instruction");
19902
19903 unsigned dmask = MO_Dmask->getImm();
19904 // Determine the number of active lanes taking into account the
19905 // Gather4 special case
19906 unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(Value: dmask);
19907
19908 bool Packed = !Subtarget->hasUnpackedD16VMem();
19909
19910 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
19911
19912 // Abandon attempt if the dst size isn't large enough
19913 // - this is in fact an error but this is picked up elsewhere and
19914 // reported correctly.
19915 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19916
19917 uint32_t DstSize = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19918 if (DstSize < InitIdx)
19919 return;
19920 } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(Opc: MI.getOpcode())) {
19921 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19922 InitIdx = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19923 } else {
19924 return;
19925 }
19926
19927 const DebugLoc &DL = MI.getDebugLoc();
19928
19929 // Create a register for the initialization value.
19930 Register PrevDst = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: DstIdx).getReg());
19931 unsigned NewDst = 0; // Final initialized value will be in here
19932
19933 // If PRTStrictNull feature is enabled (the default) then initialize
19934 // all the result registers to 0, otherwise just the error indication
19935 // register (VGPRn+1)
19936 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1;
19937 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1);
19938
19939 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: PrevDst);
19940 for (; SizeLeft; SizeLeft--, CurrIdx++) {
19941 NewDst = MRI.createVirtualRegister(RegClass: TII->getOpRegClass(MI, OpNo: DstIdx));
19942 // Initialize dword
19943 Register SubReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
19944 // clang-format off
19945 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: SubReg)
19946 .addImm(Val: 0);
19947 // clang-format on
19948 // Insert into the super-reg
19949 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewDst)
19950 .addReg(RegNo: PrevDst)
19951 .addReg(RegNo: SubReg)
19952 .addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: CurrIdx));
19953
19954 PrevDst = NewDst;
19955 }
19956
19957 // Add as an implicit operand
19958 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewDst, isDef: false, isImp: true));
19959
19960 // Tie the just added implicit operand to the dst
19961 MI.tieOperands(DefIdx: DstIdx, UseIdx: MI.getNumOperands() - 1);
19962}
19963
19964/// Assign the register class depending on the number of
19965/// bits set in the writemask
19966void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
19967 SDNode *Node) const {
19968 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19969
19970 MachineFunction *MF = MI.getMF();
19971 MachineRegisterInfo &MRI = MF->getRegInfo();
19972
19973 if (TII->isVOP3(Opcode: MI.getOpcode())) {
19974 // Make sure constant bus requirements are respected.
19975 TII->legalizeOperandsVOP3(MRI, MI);
19976
19977 if (TII->isMAI(MI)) {
19978 // The ordinary src0, src1, src2 were legalized above.
19979 //
19980 // We have to also legalize the appended v_mfma_ld_scale_b32 operands,
19981 // as a separate instruction.
19982 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19983 Name: AMDGPU::OpName::scale_src0);
19984 if (Src0Idx != -1) {
19985 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19986 Name: AMDGPU::OpName::scale_src1);
19987 if (TII->usesConstantBus(MRI, MI, OpIdx: Src0Idx) &&
19988 TII->usesConstantBus(MRI, MI, OpIdx: Src1Idx))
19989 TII->legalizeOpWithMove(MI, OpIdx: Src1Idx);
19990 }
19991 }
19992
19993 return;
19994 }
19995
19996 if (TII->isImage(MI))
19997 TII->enforceOperandRCAlignment(MI, OpName: AMDGPU::OpName::vaddr);
19998}
19999
20000static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
20001 uint64_t Val) {
20002 SDValue K = DAG.getTargetConstant(Val, DL, VT: MVT::i32);
20003 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: K), 0);
20004}
20005
20006MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG,
20007 const SDLoc &DL,
20008 SDValue Ptr) const {
20009 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20010
20011 // Build the half of the subregister with the constants before building the
20012 // full 128-bit register. If we are building multiple resource descriptors,
20013 // this will allow CSEing of the 2-component register.
20014 const SDValue Ops0[] = {
20015 DAG.getTargetConstant(Val: AMDGPU::SGPR_64RegClassID, DL, VT: MVT::i32),
20016 buildSMovImm32(DAG, DL, Val: 0),
20017 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
20018 buildSMovImm32(DAG, DL, Val: TII->getDefaultRsrcDataFormat() >> 32),
20019 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32)};
20020
20021 SDValue SubRegHi = SDValue(
20022 DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v2i32, Ops: Ops0), 0);
20023
20024 // Combine the constants and the pointer.
20025 const SDValue Ops1[] = {
20026 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32), Ptr,
20027 DAG.getTargetConstant(Val: AMDGPU::sub0_sub1, DL, VT: MVT::i32), SubRegHi,
20028 DAG.getTargetConstant(Val: AMDGPU::sub2_sub3, DL, VT: MVT::i32)};
20029
20030 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops: Ops1);
20031}
20032
20033/// Return a resource descriptor with the 'Add TID' bit enabled
20034/// The TID (Thread ID) is multiplied by the stride value (bits [61:48]
20035/// of the resource descriptor) to create an offset, which is added to
20036/// the resource pointer.
20037MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL,
20038 SDValue Ptr, uint32_t RsrcDword1,
20039 uint64_t RsrcDword2And3) const {
20040 SDValue PtrLo = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub0, DL, VT: MVT::i32, Operand: Ptr);
20041 SDValue PtrHi = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub1, DL, VT: MVT::i32, Operand: Ptr);
20042 if (RsrcDword1) {
20043 PtrHi = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: PtrHi,
20044 N2: DAG.getConstant(Val: RsrcDword1, DL, VT: MVT::i32));
20045 }
20046
20047 SDValue DataLo =
20048 buildSMovImm32(DAG, DL, Val: RsrcDword2And3 & UINT64_C(0xFFFFFFFF));
20049 SDValue DataHi = buildSMovImm32(DAG, DL, Val: RsrcDword2And3 >> 32);
20050
20051 const SDValue Ops[] = {
20052 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32),
20053 PtrLo,
20054 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
20055 PtrHi,
20056 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32),
20057 DataLo,
20058 DAG.getTargetConstant(Val: AMDGPU::sub2, DL, VT: MVT::i32),
20059 DataHi,
20060 DAG.getTargetConstant(Val: AMDGPU::sub3, DL, VT: MVT::i32)};
20061
20062 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops);
20063}
20064
20065//===----------------------------------------------------------------------===//
20066// SI Inline Assembly Support
20067//===----------------------------------------------------------------------===//
20068
20069std::pair<unsigned, const TargetRegisterClass *>
20070SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
20071 StringRef Constraint,
20072 MVT VT) const {
20073 const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(TRI_);
20074
20075 const TargetRegisterClass *RC = nullptr;
20076 if (Constraint.size() == 1) {
20077 // Check if we cannot determine the bit size of the given value type. This
20078 // can happen, for example, in this situation where we have an empty struct
20079 // (size 0): `call void asm "", "v"({} poison)`-
20080 if (VT == MVT::Other)
20081 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20082 const unsigned BitWidth = VT.getSizeInBits();
20083 switch (Constraint[0]) {
20084 default:
20085 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20086 case 's':
20087 case 'r':
20088 switch (BitWidth) {
20089 case 16:
20090 RC = &AMDGPU::SReg_32RegClass;
20091 break;
20092 case 64:
20093 RC = &AMDGPU::SGPR_64RegClass;
20094 break;
20095 default:
20096 RC = SIRegisterInfo::getSGPRClassForBitWidth(BitWidth);
20097 if (!RC)
20098 return std::pair(0U, nullptr);
20099 break;
20100 }
20101 break;
20102 case 'v':
20103 switch (BitWidth) {
20104 case 1:
20105 return std::pair(0U, nullptr);
20106 case 16:
20107 RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass
20108 : &AMDGPU::VGPR_32_Lo256RegClass;
20109 break;
20110 default:
20111 RC = Subtarget->hasVGPRMSBs()
20112 ? TRI->getAlignedLo256VGPRClassForBitWidth(BitWidth)
20113 : TRI->getVGPRClassForBitWidth(BitWidth);
20114 if (!RC)
20115 return std::pair(0U, nullptr);
20116 break;
20117 }
20118 break;
20119 case 'a':
20120 if (!Subtarget->hasMAIInsts())
20121 break;
20122 switch (BitWidth) {
20123 case 1:
20124 return std::pair(0U, nullptr);
20125 case 16:
20126 RC = &AMDGPU::AGPR_32RegClass;
20127 break;
20128 default:
20129 RC = TRI->getAGPRClassForBitWidth(BitWidth);
20130 if (!RC)
20131 return std::pair(0U, nullptr);
20132 break;
20133 }
20134 break;
20135 }
20136 } else if (Constraint == "VA" && Subtarget->hasGFX90AInsts()) {
20137 const unsigned BitWidth = VT.getSizeInBits();
20138 switch (BitWidth) {
20139 case 16:
20140 RC = &AMDGPU::AV_32RegClass;
20141 break;
20142 default:
20143 RC = TRI->getVectorSuperClassForBitWidth(BitWidth);
20144 if (!RC)
20145 return std::pair(0U, nullptr);
20146 break;
20147 }
20148 }
20149
20150 // We actually support i128, i16 and f16 as inline parameters
20151 // even if they are not reported as legal
20152 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
20153 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
20154 return std::pair(0U, RC);
20155
20156 auto [Kind, Idx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint);
20157 if (Kind != '\0') {
20158 if (Kind == 'v') {
20159 RC = &AMDGPU::VGPR_32_Lo256RegClass;
20160 } else if (Kind == 's') {
20161 RC = &AMDGPU::SGPR_32RegClass;
20162 } else if (Kind == 'a') {
20163 RC = &AMDGPU::AGPR_32RegClass;
20164 }
20165
20166 if (RC) {
20167 if (NumRegs > 1) {
20168 if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 >= RC->getNumRegs())
20169 return std::pair(0U, nullptr);
20170
20171 uint32_t Width = NumRegs * 32;
20172 // Prohibit constraints for register ranges with a width that does not
20173 // match the required type.
20174 if (VT.SimpleTy != MVT::Other && Width != VT.getSizeInBits())
20175 return std::pair(0U, nullptr);
20176
20177 MCRegister Reg = RC->getRegister(i: Idx);
20178 if (SIRegisterInfo::isVGPRClass(RC))
20179 RC = TRI->getVGPRClassForBitWidth(BitWidth: Width);
20180 else if (SIRegisterInfo::isSGPRClass(RC))
20181 RC = TRI->getSGPRClassForBitWidth(BitWidth: Width);
20182 else if (SIRegisterInfo::isAGPRClass(RC))
20183 RC = TRI->getAGPRClassForBitWidth(BitWidth: Width);
20184 if (RC) {
20185 Reg = TRI->getMatchingSuperReg(Reg, SubIdx: AMDGPU::sub0, RC);
20186 if (!Reg) {
20187 // The register class does not contain the requested register,
20188 // e.g., because it is an SGPR pair that would violate alignment
20189 // requirements.
20190 return std::pair(0U, nullptr);
20191 }
20192 return std::pair(Reg, RC);
20193 }
20194 }
20195
20196 // Reject types that do not fit a single 32-bit register: any scalar wider
20197 // than 32 bits, or a vector that is not exactly 32 bits.
20198 if (VT.SimpleTy != MVT::Other &&
20199 (VT.getSizeInBits() > 32 ||
20200 (VT.isVector() && VT.getSizeInBits() != 32)))
20201 return std::pair(0U, nullptr);
20202 if (RC && Idx < RC->getNumRegs())
20203 return std::pair(RC->getRegister(i: Idx), RC);
20204 return std::pair(0U, nullptr);
20205 }
20206 }
20207
20208 auto Ret = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
20209 if (Ret.first) {
20210 Ret.second = TRI->getPhysRegBaseClass(Reg: Ret.first);
20211 if (!Subtarget->useRealTrue16Insts() &&
20212 TRI->getRegSizeInBits(RC: *Ret.second) == 16)
20213 return std::pair(0U, nullptr);
20214 }
20215
20216 return Ret;
20217}
20218
20219static bool isImmConstraint(StringRef Constraint) {
20220 if (Constraint.size() == 1) {
20221 switch (Constraint[0]) {
20222 default:
20223 break;
20224 case 'I':
20225 case 'J':
20226 case 'A':
20227 case 'B':
20228 case 'C':
20229 return true;
20230 }
20231 } else if (Constraint == "DA" || Constraint == "DB") {
20232 return true;
20233 }
20234 return false;
20235}
20236
20237SITargetLowering::ConstraintType
20238SITargetLowering::getConstraintType(StringRef Constraint) const {
20239 if (Constraint.size() == 1) {
20240 switch (Constraint[0]) {
20241 default:
20242 break;
20243 case 's':
20244 case 'v':
20245 case 'a':
20246 return C_RegisterClass;
20247 case 'm':
20248 return C_Memory;
20249 }
20250 } else if (Constraint.size() == 2) {
20251 if (Constraint == "VA")
20252 return C_RegisterClass;
20253 if (Constraint == "RF")
20254 return C_Memory;
20255 }
20256 if (isImmConstraint(Constraint)) {
20257 return C_Other;
20258 }
20259 return TargetLowering::getConstraintType(Constraint);
20260}
20261
20262InlineAsm::ConstraintCode
20263SITargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
20264 if (ConstraintCode == "RF")
20265 return InlineAsm::ConstraintCode::RF;
20266
20267 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
20268}
20269
20270static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) {
20271 if (!AMDGPU::isInlinableIntLiteral(Literal: Val)) {
20272 Val = Val & maskTrailingOnes<uint64_t>(N: Size);
20273 }
20274 return Val;
20275}
20276
20277void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op,
20278 StringRef Constraint,
20279 std::vector<SDValue> &Ops,
20280 SelectionDAG &DAG) const {
20281 if (isImmConstraint(Constraint)) {
20282 uint64_t Val;
20283 if (getAsmOperandConstVal(Op, Val) &&
20284 checkAsmConstraintVal(Op, Constraint, Val)) {
20285 Val = clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits());
20286 Ops.push_back(x: DAG.getTargetConstant(Val, DL: SDLoc(Op), VT: MVT::i64));
20287 }
20288 } else {
20289 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
20290 }
20291}
20292
20293bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const {
20294 unsigned Size = Op.getScalarValueSizeInBits();
20295 if (Size > 64)
20296 return false;
20297
20298 if (Size == 16 && !Subtarget->has16BitInsts())
20299 return false;
20300
20301 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
20302 Val = C->getSExtValue();
20303 return true;
20304 }
20305 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
20306 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
20307 return true;
20308 }
20309 if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
20310 if (Size != 16 || Op.getNumOperands() != 2)
20311 return false;
20312 if (Op.getOperand(i: 0).isUndef() || Op.getOperand(i: 1).isUndef())
20313 return false;
20314 if (ConstantSDNode *C = V->getConstantSplatNode()) {
20315 Val = C->getSExtValue();
20316 return true;
20317 }
20318 if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
20319 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
20320 return true;
20321 }
20322 }
20323
20324 return false;
20325}
20326
20327bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint,
20328 uint64_t Val) const {
20329 if (Constraint.size() == 1) {
20330 switch (Constraint[0]) {
20331 case 'I':
20332 return AMDGPU::isInlinableIntLiteral(Literal: Val);
20333 case 'J':
20334 return isInt<16>(x: Val);
20335 case 'A':
20336 return checkAsmConstraintValA(Op, Val);
20337 case 'B':
20338 return isInt<32>(x: Val);
20339 case 'C':
20340 return isUInt<32>(x: clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits())) ||
20341 AMDGPU::isInlinableIntLiteral(Literal: Val);
20342 default:
20343 break;
20344 }
20345 } else if (Constraint.size() == 2) {
20346 if (Constraint == "DA") {
20347 int64_t HiBits = static_cast<int32_t>(Val >> 32);
20348 int64_t LoBits = static_cast<int32_t>(Val);
20349 return checkAsmConstraintValA(Op, Val: HiBits, MaxSize: 32) &&
20350 checkAsmConstraintValA(Op, Val: LoBits, MaxSize: 32);
20351 }
20352 if (Constraint == "DB") {
20353 return true;
20354 }
20355 }
20356 llvm_unreachable("Invalid asm constraint");
20357}
20358
20359bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val,
20360 unsigned MaxSize) const {
20361 unsigned Size = std::min<unsigned>(a: Op.getScalarValueSizeInBits(), b: MaxSize);
20362 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
20363 if (Size == 16) {
20364 MVT VT = Op.getSimpleValueType();
20365 switch (VT.SimpleTy) {
20366 default:
20367 return false;
20368 case MVT::i16:
20369 return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi);
20370 case MVT::f16:
20371 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi);
20372 case MVT::bf16:
20373 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi);
20374 case MVT::v2i16:
20375 return AMDGPU::getInlineEncodingV2I16(Literal: Val).has_value();
20376 case MVT::v2f16:
20377 return AMDGPU::getInlineEncodingV2F16(Literal: Val).has_value();
20378 case MVT::v2bf16:
20379 return AMDGPU::getInlineEncodingV2BF16(Literal: Val).has_value();
20380 }
20381 }
20382 if ((Size == 32 && AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi)) ||
20383 (Size == 64 && AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi)))
20384 return true;
20385 return false;
20386}
20387
20388static int getAlignedAGPRClassID(unsigned UnalignedClassID) {
20389 switch (UnalignedClassID) {
20390 case AMDGPU::VReg_64RegClassID:
20391 return AMDGPU::VReg_64_Align2RegClassID;
20392 case AMDGPU::VReg_96RegClassID:
20393 return AMDGPU::VReg_96_Align2RegClassID;
20394 case AMDGPU::VReg_128RegClassID:
20395 return AMDGPU::VReg_128_Align2RegClassID;
20396 case AMDGPU::VReg_160RegClassID:
20397 return AMDGPU::VReg_160_Align2RegClassID;
20398 case AMDGPU::VReg_192RegClassID:
20399 return AMDGPU::VReg_192_Align2RegClassID;
20400 case AMDGPU::VReg_224RegClassID:
20401 return AMDGPU::VReg_224_Align2RegClassID;
20402 case AMDGPU::VReg_256RegClassID:
20403 return AMDGPU::VReg_256_Align2RegClassID;
20404 case AMDGPU::VReg_288RegClassID:
20405 return AMDGPU::VReg_288_Align2RegClassID;
20406 case AMDGPU::VReg_320RegClassID:
20407 return AMDGPU::VReg_320_Align2RegClassID;
20408 case AMDGPU::VReg_352RegClassID:
20409 return AMDGPU::VReg_352_Align2RegClassID;
20410 case AMDGPU::VReg_384RegClassID:
20411 return AMDGPU::VReg_384_Align2RegClassID;
20412 case AMDGPU::VReg_512RegClassID:
20413 return AMDGPU::VReg_512_Align2RegClassID;
20414 case AMDGPU::VReg_1024RegClassID:
20415 return AMDGPU::VReg_1024_Align2RegClassID;
20416 case AMDGPU::AReg_64RegClassID:
20417 return AMDGPU::AReg_64_Align2RegClassID;
20418 case AMDGPU::AReg_96RegClassID:
20419 return AMDGPU::AReg_96_Align2RegClassID;
20420 case AMDGPU::AReg_128RegClassID:
20421 return AMDGPU::AReg_128_Align2RegClassID;
20422 case AMDGPU::AReg_160RegClassID:
20423 return AMDGPU::AReg_160_Align2RegClassID;
20424 case AMDGPU::AReg_192RegClassID:
20425 return AMDGPU::AReg_192_Align2RegClassID;
20426 case AMDGPU::AReg_256RegClassID:
20427 return AMDGPU::AReg_256_Align2RegClassID;
20428 case AMDGPU::AReg_512RegClassID:
20429 return AMDGPU::AReg_512_Align2RegClassID;
20430 case AMDGPU::AReg_1024RegClassID:
20431 return AMDGPU::AReg_1024_Align2RegClassID;
20432 default:
20433 return -1;
20434 }
20435}
20436
20437// Figure out which registers should be reserved for stack access. Only after
20438// the function is legalized do we know all of the non-spill stack objects or if
20439// calls are present.
20440void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
20441 MachineRegisterInfo &MRI = MF.getRegInfo();
20442 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20443 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
20444 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20445 const SIInstrInfo *TII = ST.getInstrInfo();
20446
20447 if (Info->isEntryFunction()) {
20448 // Callable functions have fixed registers used for stack access.
20449 reservePrivateMemoryRegs(TM: getTargetMachine(), MF, TRI: *TRI, Info&: *Info);
20450 }
20451
20452 // TODO: Move this logic to getReservedRegs()
20453 // Reserve the SGPR(s) to save/restore EXEC for WWM spill/copy handling.
20454 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
20455 Register SReg = ST.isWave32()
20456 ? AMDGPU::SGPR_32RegClass.getRegister(i: MaxNumSGPRs - 1)
20457 : TRI->getAlignedHighSGPRForRC(MF, /*Align=*/2,
20458 RC: &AMDGPU::SGPR_64RegClass);
20459 Info->setSGPRForEXECCopy(SReg);
20460
20461 assert(!TRI->isSubRegister(Info->getScratchRSrcReg(),
20462 Info->getStackPtrOffsetReg()));
20463 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
20464 MRI.replaceRegWith(FromReg: AMDGPU::SP_REG, ToReg: Info->getStackPtrOffsetReg());
20465
20466 // We need to worry about replacing the default register with itself in case
20467 // of MIR testcases missing the MFI.
20468 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
20469 MRI.replaceRegWith(FromReg: AMDGPU::PRIVATE_RSRC_REG, ToReg: Info->getScratchRSrcReg());
20470
20471 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG)
20472 MRI.replaceRegWith(FromReg: AMDGPU::FP_REG, ToReg: Info->getFrameOffsetReg());
20473
20474 Info->limitOccupancy(MF);
20475
20476 if (ST.isWave32() && !MF.empty()) {
20477 for (auto &MBB : MF) {
20478 for (auto &MI : MBB) {
20479 TII->fixImplicitOperands(MI);
20480 }
20481 }
20482 }
20483
20484 // FIXME: This is a hack to fixup AGPR classes to use the properly aligned
20485 // classes if required. Ideally the register class constraints would differ
20486 // per-subtarget, but there's no easy way to achieve that right now. This is
20487 // not a problem for VGPRs because the correctly aligned VGPR class is implied
20488 // from using them as the register class for legal types.
20489 if (ST.needsAlignedVGPRs()) {
20490 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
20491 const Register Reg = Register::index2VirtReg(Index: I);
20492 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
20493 if (!RC)
20494 continue;
20495 int NewClassID = getAlignedAGPRClassID(UnalignedClassID: RC->getID());
20496 if (NewClassID != -1)
20497 MRI.setRegClass(Reg, RC: TRI->getRegClass(i: NewClassID));
20498 }
20499 }
20500
20501 TargetLoweringBase::finalizeLowering(MF);
20502}
20503
20504void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
20505 KnownBits &Known,
20506 const APInt &DemandedElts,
20507 const SelectionDAG &DAG,
20508 unsigned Depth) const {
20509 Known.resetAll();
20510 unsigned Opc = Op.getOpcode();
20511 switch (Opc) {
20512 case ISD::INTRINSIC_WO_CHAIN: {
20513 unsigned IID = Op.getConstantOperandVal(i: 0);
20514 switch (IID) {
20515 case Intrinsic::amdgcn_mbcnt_lo:
20516 case Intrinsic::amdgcn_mbcnt_hi: {
20517 const GCNSubtarget &ST =
20518 DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
20519 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20520 // most 31 + src1.
20521 Known.Zero.setBitsFrom(
20522 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
20523 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
20524 Known = KnownBits::add(LHS: Known, RHS: Known2);
20525 return;
20526 }
20527 }
20528 break;
20529 }
20530 }
20531 return AMDGPUTargetLowering::computeKnownBitsForTargetNode(
20532 Op, Known, DemandedElts, DAG, Depth);
20533}
20534
20535void SITargetLowering::computeKnownBitsForStackObjectPointer(
20536 KnownBits &Known, const MachineFunction &MF, Align Alignment) const {
20537 TargetLowering::computeKnownBitsForStackObjectPointer(Known, MF, Alignment);
20538
20539 // Set the high bits to zero based on the maximum allowed scratch size per
20540 // wave. We can't use vaddr in MUBUF instructions if we don't know the address
20541 // calculation won't overflow, so assume the sign bit is never set.
20542 Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex());
20543}
20544
20545static void knownBitsForWorkitemID(const GCNSubtarget &ST,
20546 GISelValueTracking &VT, KnownBits &Known,
20547 unsigned Dim) {
20548 unsigned MaxValue =
20549 ST.getMaxWorkitemID(Kernel: VT.getMachineFunction().getFunction(), Dimension: Dim);
20550 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
20551}
20552
20553static void knownBitsForSBFE(const MachineInstr &MI, GISelValueTracking &VT,
20554 KnownBits &Known, const APInt &DemandedElts,
20555 unsigned BFEWidth, bool SExt, unsigned Depth) {
20556 const MachineRegisterInfo &MRI = VT.getMachineFunction().getRegInfo();
20557 const MachineOperand &Src1 = MI.getOperand(i: 2);
20558
20559 unsigned Src1Cst = 0;
20560 if (Src1.isImm()) {
20561 Src1Cst = Src1.getImm();
20562 } else if (Src1.isReg()) {
20563 auto Cst = getIConstantVRegValWithLookThrough(VReg: Src1.getReg(), MRI);
20564 if (!Cst)
20565 return;
20566 Src1Cst = Cst->Value.getZExtValue();
20567 } else {
20568 return;
20569 }
20570
20571 // Offset is at bits [4:0] for 32 bit, [5:0] for 64 bit.
20572 // Width is always [22:16].
20573 const unsigned Offset =
20574 Src1Cst & maskTrailingOnes<unsigned>(N: (BFEWidth == 32) ? 5 : 6);
20575 const unsigned Width = (Src1Cst >> 16) & maskTrailingOnes<unsigned>(N: 6);
20576
20577 if (Width >= BFEWidth) // Ill-formed.
20578 return;
20579
20580 VT.computeKnownBitsImpl(R: MI.getOperand(i: 1).getReg(), Known, DemandedElts,
20581 Depth: Depth + 1);
20582
20583 Known = Known.extractBits(NumBits: Width, BitPosition: Offset);
20584
20585 if (SExt)
20586 Known = Known.sext(BitWidth: BFEWidth);
20587 else
20588 Known = Known.zext(BitWidth: BFEWidth);
20589}
20590
20591void SITargetLowering::computeKnownBitsForTargetInstr(
20592 GISelValueTracking &VT, Register R, KnownBits &Known,
20593 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
20594 unsigned Depth) const {
20595 Known.resetAll();
20596 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
20597 switch (MI->getOpcode()) {
20598 case AMDGPU::S_BFE_I32:
20599 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
20600 /*SExt=*/true, Depth);
20601 case AMDGPU::S_BFE_U32:
20602 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
20603 /*SExt=*/false, Depth);
20604 case AMDGPU::S_BFE_I64:
20605 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
20606 /*SExt=*/true, Depth);
20607 case AMDGPU::S_BFE_U64:
20608 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
20609 /*SExt=*/false, Depth);
20610 case AMDGPU::G_INTRINSIC:
20611 case AMDGPU::G_INTRINSIC_CONVERGENT: {
20612 Intrinsic::ID IID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
20613 switch (IID) {
20614 case Intrinsic::amdgcn_workitem_id_x:
20615 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 0);
20616 break;
20617 case Intrinsic::amdgcn_workitem_id_y:
20618 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 1);
20619 break;
20620 case Intrinsic::amdgcn_workitem_id_z:
20621 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 2);
20622 break;
20623 case Intrinsic::amdgcn_mbcnt_lo:
20624 case Intrinsic::amdgcn_mbcnt_hi: {
20625 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
20626 // most 31 + src1.
20627 Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
20628 ? getSubtarget()->getWavefrontSizeLog2()
20629 : 5);
20630 KnownBits Known2;
20631 VT.computeKnownBitsImpl(R: MI->getOperand(i: 3).getReg(), Known&: Known2, DemandedElts,
20632 Depth: Depth + 1);
20633 Known = KnownBits::add(LHS: Known, RHS: Known2);
20634 break;
20635 }
20636 case Intrinsic::amdgcn_groupstaticsize: {
20637 // We can report everything over the maximum size as 0. We can't report
20638 // based on the actual size because we don't know if it's accurate or not
20639 // at any given point.
20640 Known.Zero.setHighBits(
20641 llvm::countl_zero(Val: getSubtarget()->getAddressableLocalMemorySize()));
20642 break;
20643 }
20644 case Intrinsic::amdgcn_readfirstlane:
20645 case Intrinsic::amdgcn_readlane: {
20646 // Result is the data operand's value from some lane.
20647 VT.computeKnownBitsImpl(R: MI->getOperand(i: 2).getReg(), Known, DemandedElts,
20648 Depth: Depth + 1);
20649 break;
20650 }
20651 case Intrinsic::amdgcn_mul_u24:
20652 case Intrinsic::amdgcn_mul_i24: {
20653 Register LHS = MI->getOperand(i: 2).getReg();
20654 Register RHS = MI->getOperand(i: 3).getReg();
20655
20656 KnownBits RHSKnown;
20657 VT.computeKnownBitsImpl(R: RHS, Known&: RHSKnown, DemandedElts, Depth: Depth + 1);
20658
20659 bool SelfMultiply = LHS == RHS;
20660
20661 // For a self-multiply both operands are the same value, so RHSKnown also
20662 // describes the LHS; avoid the duplicated query.
20663 KnownBits LHSKnown;
20664 if (SelfMultiply)
20665 LHSKnown = RHSKnown;
20666 else
20667 VT.computeKnownBitsImpl(R: LHS, Known&: LHSKnown, DemandedElts, Depth: Depth + 1);
20668
20669 if (SelfMultiply)
20670 SelfMultiply &= isGuaranteedNotToBeUndef(Reg: LHS, MRI, Depth: Depth + 1);
20671
20672 Known = computeKnownBitsForMul24(LHSKnown, RHSKnown, BitWidth: Known.getBitWidth(),
20673 IsSigned: IID == Intrinsic::amdgcn_mul_i24,
20674 SelfMultiply);
20675 break;
20676 }
20677 }
20678 break;
20679 }
20680 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
20681 Known.Zero.setHighBits(24);
20682 break;
20683 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
20684 Known.Zero.setHighBits(16);
20685 break;
20686 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
20687 // G_AMDGPU_COPY_SCC_VCC converts a uniform boolean in VCC to SGPR s32,
20688 // producing exactly 0 or 1.
20689 Known.Zero.setHighBits(Known.getBitWidth() - 1);
20690 break;
20691 case AMDGPU::G_AMDGPU_SMED3:
20692 case AMDGPU::G_AMDGPU_UMED3: {
20693 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
20694
20695 KnownBits Known2;
20696 VT.computeKnownBitsImpl(R: Src2, Known&: Known2, DemandedElts, Depth: Depth + 1);
20697 if (Known2.isUnknown())
20698 break;
20699
20700 KnownBits Known1;
20701 VT.computeKnownBitsImpl(R: Src1, Known&: Known1, DemandedElts, Depth: Depth + 1);
20702 if (Known1.isUnknown())
20703 break;
20704
20705 KnownBits Known0;
20706 VT.computeKnownBitsImpl(R: Src0, Known&: Known0, DemandedElts, Depth: Depth + 1);
20707 if (Known0.isUnknown())
20708 break;
20709
20710 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
20711 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
20712 Known.One = Known0.One & Known1.One & Known2.One;
20713 break;
20714 }
20715 }
20716}
20717
20718Align SITargetLowering::computeKnownAlignForTargetInstr(
20719 GISelValueTracking &VT, Register R, const MachineRegisterInfo &MRI,
20720 unsigned Depth) const {
20721 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
20722 if (auto *GI = dyn_cast<GIntrinsic>(Val: MI)) {
20723 // FIXME: Can this move to generic code? What about the case where the call
20724 // site specifies a lower alignment?
20725 Intrinsic::ID IID = GI->getIntrinsicID();
20726 LLVMContext &Ctx = VT.getMachineFunction().getFunction().getContext();
20727 AttributeList Attrs =
20728 Intrinsic::getAttributes(C&: Ctx, id: IID, FT: Intrinsic::getType(Context&: Ctx, id: IID));
20729 if (MaybeAlign RetAlign = Attrs.getRetAlignment())
20730 return *RetAlign;
20731 }
20732 return Align(1);
20733}
20734
20735Align SITargetLowering::getPrefLoopAlignment(
20736 MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
20737 const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
20738 const Align CacheLineAlign = Align(64);
20739
20740 // GFX950: Prevent an 8-byte instruction at the block being aligned from being
20741 // split by the 32-byte instruction fetch window boundary. This avoids a
20742 // significant fetch delay after a backward branch. We use 32-byte alignment
20743 // with max padding of 4 bytes (one s_nop), see
20744 // getMaxPermittedBytesForAlignment().
20745 if (ML && getSubtarget()->getCLOpts().loop_alignment &&
20746 getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
20747 // Loop rotation can make the backedge destination a block other than the
20748 // LoopInfo header, so prefer the block the caller is actually aligning.
20749 if (!BlockToAlign)
20750 BlockToAlign = ML->getHeader();
20751 // Respect user-specified or previously set alignment.
20752 if (BlockToAlign->getAlignment() != PrefAlign)
20753 return BlockToAlign->getAlignment();
20754 if (needsFetchWindowAlignment(MBB: *BlockToAlign))
20755 return Align(32);
20756 }
20757
20758 // Pre-GFX10 target did not benefit from loop alignment
20759 if (!ML || !getSubtarget()->getCLOpts().loop_alignment ||
20760 !getSubtarget()->hasInstPrefetch() ||
20761 getSubtarget()->hasInstFwdPrefetchBug())
20762 return PrefAlign;
20763
20764 // On GFX10 I$ is 4 x 64 bytes cache lines.
20765 // By default prefetcher keeps one cache line behind and reads two ahead.
20766 // We can modify it with S_INST_PREFETCH for larger loops to have two lines
20767 // behind and one ahead.
20768 // Therefor we can benefit from aligning loop headers if loop fits 192 bytes.
20769 // If loop fits 64 bytes it always spans no more than two cache lines and
20770 // does not need an alignment.
20771 // Else if loop is less or equal 128 bytes we do not need to modify prefetch,
20772 // Else if loop is less or equal 192 bytes we need two lines behind.
20773
20774 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20775 const MachineBasicBlock *Header = ML->getHeader();
20776 if (Header->getAlignment() != PrefAlign)
20777 return Header->getAlignment(); // Already processed.
20778
20779 unsigned LoopSize = 0;
20780 for (const MachineBasicBlock *MBB : ML->blocks()) {
20781 // If inner loop block is aligned assume in average half of the alignment
20782 // size to be added as nops.
20783 if (MBB != Header)
20784 LoopSize += MBB->getAlignment().value() / 2;
20785
20786 for (const MachineInstr &MI : *MBB) {
20787 LoopSize += TII->getInstSizeInBytes(MI);
20788 if (LoopSize > 192)
20789 return PrefAlign;
20790 }
20791 }
20792
20793 if (LoopSize <= 64)
20794 return PrefAlign;
20795
20796 if (LoopSize <= 128)
20797 return CacheLineAlign;
20798
20799 // If any of parent loops is surrounded by prefetch instructions do not
20800 // insert new for inner loop, which would reset parent's settings.
20801 for (MachineLoop *P = ML->getParentLoop(); P; P = P->getParentLoop()) {
20802 if (MachineBasicBlock *Exit = P->getExitBlock()) {
20803 auto I = Exit->getFirstNonDebugInstr();
20804 if (I != Exit->end() && I->getOpcode() == AMDGPU::S_INST_PREFETCH)
20805 return CacheLineAlign;
20806 }
20807 }
20808
20809 MachineBasicBlock *Pre = ML->getLoopPreheader();
20810 MachineBasicBlock *Exit = ML->getExitBlock();
20811
20812 if (Pre && Exit) {
20813 auto PreTerm = Pre->getFirstTerminator();
20814 if (PreTerm == Pre->begin() ||
20815 std::prev(x: PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
20816 BuildMI(BB&: *Pre, I: PreTerm, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20817 .addImm(Val: 1); // prefetch 2 lines behind PC
20818
20819 auto ExitHead = Exit->getFirstNonDebugInstr();
20820 if (ExitHead == Exit->end() ||
20821 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
20822 BuildMI(BB&: *Exit, I: ExitHead, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20823 .addImm(Val: 2); // prefetch 1 line behind PC
20824 }
20825
20826 return CacheLineAlign;
20827}
20828
20829unsigned SITargetLowering::getMaxPermittedBytesForAlignment(
20830 MachineBasicBlock *MBB) const {
20831 // GFX950: Limit padding to 4 bytes (one s_nop) for blocks where an 8-byte
20832 // instruction could be split by the 32-byte fetch window boundary.
20833 // See getPrefLoopAlignment() for context.
20834 if (needsFetchWindowAlignment(MBB: *MBB))
20835 return 4;
20836 return TargetLowering::getMaxPermittedBytesForAlignment(MBB);
20837}
20838
20839bool SITargetLowering::needsFetchWindowAlignment(
20840 const MachineBasicBlock &MBB) const {
20841 if (!getSubtarget()->hasLoopHeadInstSplitSensitivity())
20842 return false;
20843 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20844 for (const MachineInstr &MI : MBB) {
20845 if (MI.isMetaInstruction())
20846 continue;
20847 // Instructions larger than 4 bytes can be split by a 32-byte boundary.
20848 return TII->getInstSizeInBytes(MI) > 4;
20849 }
20850 return false;
20851}
20852
20853[[maybe_unused]]
20854static bool isCopyFromRegOfInlineAsm(const SDNode *N) {
20855 assert(N->getOpcode() == ISD::CopyFromReg);
20856 do {
20857 // Follow the chain until we find an INLINEASM node.
20858 N = N->getOperand(Num: 0).getNode();
20859 if (N->getOpcode() == ISD::INLINEASM || N->getOpcode() == ISD::INLINEASM_BR)
20860 return true;
20861 } while (N->getOpcode() == ISD::CopyFromReg);
20862 return false;
20863}
20864
20865bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
20866 FunctionLoweringInfo *FLI,
20867 UniformityInfo *UA) const {
20868 switch (N->getOpcode()) {
20869 case ISD::CopyFromReg: {
20870 const RegisterSDNode *R = cast<RegisterSDNode>(Val: N->getOperand(Num: 1));
20871 const MachineRegisterInfo &MRI = FLI->MF->getRegInfo();
20872 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20873 Register Reg = R->getReg();
20874
20875 // FIXME: Why does this need to consider isLiveIn?
20876 if (Reg.isPhysical() || MRI.isLiveIn(Reg))
20877 return !TRI->isSGPRReg(MRI, Reg);
20878
20879 if (const Value *V = FLI->getValueFromVirtualReg(Vreg: R->getReg()))
20880 return UA->isDivergentAtDef(V);
20881
20882 assert(Reg == FLI->DemoteRegister || isCopyFromRegOfInlineAsm(N));
20883 return !TRI->isSGPRReg(MRI, Reg);
20884 }
20885 case ISD::LOAD: {
20886 const LoadSDNode *L = cast<LoadSDNode>(Val: N);
20887 unsigned AS = L->getAddressSpace();
20888 // A flat load may access private memory, and a VGPR load reads each lane's
20889 // own registers.
20890 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS ||
20891 AS == AMDGPUAS::VGPR;
20892 }
20893 // As above, after the pre-ISel combine.
20894 case AMDGPUISD::REG_LOAD:
20895 return true;
20896 case ISD::CALLSEQ_END:
20897 return true;
20898 case ISD::INTRINSIC_WO_CHAIN:
20899 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 0));
20900 case ISD::INTRINSIC_W_CHAIN:
20901 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 1));
20902 case AMDGPUISD::ATOMIC_CMP_SWAP:
20903 case AMDGPUISD::BUFFER_ATOMIC_SWAP:
20904 case AMDGPUISD::BUFFER_ATOMIC_ADD:
20905 case AMDGPUISD::BUFFER_ATOMIC_SUB:
20906 case AMDGPUISD::BUFFER_ATOMIC_SMIN:
20907 case AMDGPUISD::BUFFER_ATOMIC_UMIN:
20908 case AMDGPUISD::BUFFER_ATOMIC_SMAX:
20909 case AMDGPUISD::BUFFER_ATOMIC_UMAX:
20910 case AMDGPUISD::BUFFER_ATOMIC_AND:
20911 case AMDGPUISD::BUFFER_ATOMIC_OR:
20912 case AMDGPUISD::BUFFER_ATOMIC_XOR:
20913 case AMDGPUISD::BUFFER_ATOMIC_INC:
20914 case AMDGPUISD::BUFFER_ATOMIC_DEC:
20915 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP:
20916 case AMDGPUISD::BUFFER_ATOMIC_FADD:
20917 case AMDGPUISD::BUFFER_ATOMIC_FMIN:
20918 case AMDGPUISD::BUFFER_ATOMIC_FMAX:
20919 // Target-specific read-modify-write atomics are sources of divergence.
20920 return true;
20921 default:
20922 if (auto *A = dyn_cast<AtomicSDNode>(Val: N)) {
20923 // Generic read-modify-write atomics are sources of divergence.
20924 return A->readMem() && A->writeMem();
20925 }
20926 return false;
20927 }
20928}
20929
20930bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG,
20931 EVT VT) const {
20932 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
20933 case MVT::f32:
20934 return !denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
20935 case MVT::f64:
20936 case MVT::f16:
20937 return !denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction());
20938 default:
20939 return false;
20940 }
20941}
20942
20943bool SITargetLowering::denormalsEnabledForType(
20944 LLT Ty, const MachineFunction &MF) const {
20945 switch (Ty.getScalarSizeInBits()) {
20946 case 32:
20947 return !denormalModeIsFlushAllF32(MF);
20948 case 64:
20949 case 16:
20950 return !denormalModeIsFlushAllF64F16(MF);
20951 default:
20952 return false;
20953 }
20954}
20955
20956bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
20957 const APInt &DemandedElts,
20958 const SelectionDAG &DAG,
20959 bool SNaN,
20960 unsigned Depth) const {
20961 if (Op.getOpcode() == AMDGPUISD::CLAMP) {
20962 const MachineFunction &MF = DAG.getMachineFunction();
20963 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20964
20965 if (Info->getMode().DX10Clamp)
20966 return true; // Clamped to 0.
20967 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
20968 }
20969
20970 return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
20971 DAG, SNaN, Depth);
20972}
20973
20974namespace {
20975
20976/// Why a floating-point atomic instruction which may flush denormals is
20977/// acceptable.
20978enum class AtomicFlushDenormalReason {
20979 Native,
20980 IEEE,
20981 IgnoreDenormalMode,
20982 FunctionFlushesDenormals
20983};
20984
20985/// Why a native floating-point atomic instruction is acceptable for a global
20986/// memory address.
20987enum class GlobalFPAtomicLegality {
20988 Illegal,
20989 AgentScopeFineGrainedRemoteMemory,
20990 EmulatedSystemScope,
20991 NoRemoteMemory,
20992 NoFineGrainedMemory
20993};
20994
20995} // end anonymous namespace
20996
20997// On older subtargets, global FP atomic instructions have a hardcoded FP mode
20998// and do not support FP32 denormals, and only support v2f16/f64 denormals.
20999static AtomicFlushDenormalReason
21000getAtomicFlushDenormalReason(const AtomicRMWInst *RMW) {
21001 if (RMW->hasMetadata(KindID: LLVMContext::MD_atomic_ignore_denormal_mode))
21002 return AtomicFlushDenormalReason::IgnoreDenormalMode;
21003
21004 const fltSemantics &Flt = RMW->getType()->getScalarType()->getFltSemantics();
21005 auto DenormMode = RMW->getFunction()->getDenormalMode(FPType: Flt);
21006 return DenormMode == DenormalMode::getPreserveSign()
21007 ? AtomicFlushDenormalReason::FunctionFlushesDenormals
21008 : AtomicFlushDenormalReason::IEEE;
21009}
21010
21011static OptimizationRemark
21012emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW,
21013 GlobalFPAtomicLegality MemLegality,
21014 AtomicFlushDenormalReason DenormReason) {
21015 LLVMContext &Ctx = RMW->getContext();
21016 StringRef MemScope = Ctx.getSyncScopeName(Id: RMW->getSyncScopeID()).value_or(u: "");
21017 if (MemScope.empty())
21018 MemScope = "system";
21019
21020 OptimizationRemark R(DEBUG_TYPE, "Passed", RMW);
21021 R << "hardware instruction generated for atomic "
21022 << ore::NV("Operation", RMW->getOperationName(Op: RMW->getOperation()))
21023 << " at " << ore::NV("SyncScope", MemScope) << " scope since ";
21024
21025 switch (MemLegality) {
21026 case GlobalFPAtomicLegality::AgentScopeFineGrainedRemoteMemory:
21027 R << "fine-grained remote memory atomics work below system scope";
21028 break;
21029 case GlobalFPAtomicLegality::EmulatedSystemScope:
21030 R << "system scope atomics are emulated in hardware";
21031 break;
21032 case GlobalFPAtomicLegality::NoRemoteMemory:
21033 R << "memory is not remote (!amdgpu.no.remote.memory)";
21034 break;
21035 case GlobalFPAtomicLegality::NoFineGrainedMemory:
21036 R << "memory is not fine-grained (!amdgpu.no.fine.grained.memory)";
21037 break;
21038 case GlobalFPAtomicLegality::Illegal:
21039 llvm_unreachable("remark for illegal atomic");
21040 }
21041
21042 switch (DenormReason) {
21043 case AtomicFlushDenormalReason::Native:
21044 break;
21045 case AtomicFlushDenormalReason::IgnoreDenormalMode:
21046 R << ", and denormals may be flushed (!atomic.ignore.denormal.mode)";
21047 break;
21048 case AtomicFlushDenormalReason::FunctionFlushesDenormals:
21049 R << ", and the floating-point environment flushes denormals";
21050 break;
21051 case AtomicFlushDenormalReason::IEEE:
21052 llvm_unreachable("remark for illegal atomic");
21053 }
21054
21055 return R;
21056}
21057
21058static bool isV2F16OrV2BF16(Type *Ty) {
21059 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
21060 Type *EltTy = VT->getElementType();
21061 return VT->getNumElements() == 2 &&
21062 (EltTy->isHalfTy() || EltTy->isBFloatTy());
21063 }
21064
21065 return false;
21066}
21067
21068static bool isV2F16(Type *Ty) {
21069 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
21070 return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
21071}
21072
21073static bool isV2BF16(Type *Ty) {
21074 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
21075 return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
21076}
21077
21078/// \return true if atomicrmw integer ops work for the type.
21079static bool isAtomicRMWLegalIntTy(Type *Ty) {
21080 if (auto *IT = dyn_cast<IntegerType>(Val: Ty)) {
21081 unsigned BW = IT->getBitWidth();
21082 return BW == 32 || BW == 64;
21083 }
21084
21085 return false;
21086}
21087
21088/// \return true if this atomicrmw xchg type can be selected.
21089static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
21090 Type *Ty = RMW->getType();
21091 if (isAtomicRMWLegalIntTy(Ty))
21092 return true;
21093
21094 if (PointerType *PT = dyn_cast<PointerType>(Val: Ty)) {
21095 const DataLayout &DL = RMW->getFunction()->getDataLayout();
21096 unsigned BW = DL.getPointerSizeInBits(AS: PT->getAddressSpace());
21097 return BW == 32 || BW == 64;
21098 }
21099
21100 if (Ty->isFloatTy() || Ty->isDoubleTy())
21101 return true;
21102
21103 if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
21104 return VT->getNumElements() == 2 &&
21105 VT->getElementType()->getPrimitiveSizeInBits() == 16;
21106 }
21107
21108 return false;
21109}
21110
21111/// \returns whether it's valid to emit a native instruction for \p RMW, and
21112/// why, based on the properties of the target memory.
21113static GlobalFPAtomicLegality
21114getGlobalMemoryFPAtomicLegality(const GCNSubtarget &Subtarget,
21115 const AtomicRMWInst *RMW, bool HasSystemScope) {
21116 // The remote/fine-grained access logic is different from the integer
21117 // atomics. Without AgentScopeFineGrainedRemoteMemoryAtomics support,
21118 // fine-grained access does not work, even for a device local allocation.
21119 //
21120 // With AgentScopeFineGrainedRemoteMemoryAtomics, system scoped device local
21121 // allocations work.
21122 if (HasSystemScope) {
21123 if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics() &&
21124 RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
21125 return GlobalFPAtomicLegality::NoRemoteMemory;
21126 if (Subtarget.hasEmulatedSystemScopeAtomics())
21127 return GlobalFPAtomicLegality::EmulatedSystemScope;
21128 } else if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics())
21129 return GlobalFPAtomicLegality::AgentScopeFineGrainedRemoteMemory;
21130
21131 return RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory")
21132 ? GlobalFPAtomicLegality::NoFineGrainedMemory
21133 : GlobalFPAtomicLegality::Illegal;
21134}
21135
21136/// \return Action to perform on AtomicRMWInsts for integer operations.
21137static TargetLowering::AtomicExpansionKind
21138atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
21139 return isAtomicRMWLegalIntTy(Ty: RMW->getType())
21140 ? TargetLowering::AtomicExpansionKind::None
21141 : TargetLowering::AtomicExpansionKind::CmpXChg;
21142}
21143
21144/// Return if a flat address space atomicrmw can access private memory.
21145static bool flatInstrMayAccessPrivate(const Instruction *I) {
21146 const MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_noalias_addrspace);
21147 return !MD ||
21148 !AMDGPU::hasValueInRangeLikeMetadata(MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
21149}
21150
21151static TargetLowering::AtomicExpansionKind
21152getPrivateAtomicExpansionKind(const GCNSubtarget &STI) {
21153 // For GAS, lower to flat atomic.
21154 return STI.hasGloballyAddressableScratch()
21155 ? TargetLowering::AtomicExpansionKind::CustomExpand
21156 : TargetLowering::AtomicExpansionKind::NotAtomic;
21157}
21158
21159TargetLowering::AtomicExpansionKind
21160SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
21161 unsigned AS = RMW->getPointerAddressSpace();
21162 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
21163 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
21164
21165 // Only the executing lane can access its view of the VGPRs, so as with
21166 // private memory there is nothing to be atomic with respect to.
21167 if (AS == AMDGPUAS::VGPR)
21168 return AtomicExpansionKind::NotAtomic;
21169
21170 // 64-bit flat atomics that dynamically reside in private memory will silently
21171 // be dropped.
21172 //
21173 // Note that we will emit a new copy of the original atomic in the expansion,
21174 // which will be incrementally relegalized.
21175 const DataLayout &DL = RMW->getFunction()->getDataLayout();
21176 if (AS == AMDGPUAS::FLAT_ADDRESS &&
21177 DL.getTypeSizeInBits(Ty: RMW->getType()) == 64 &&
21178 flatInstrMayAccessPrivate(I: RMW))
21179 return AtomicExpansionKind::CustomExpand;
21180
21181 GlobalFPAtomicLegality MemLegality = GlobalFPAtomicLegality::Illegal;
21182 AtomicFlushDenormalReason DenormReason = AtomicFlushDenormalReason::Native;
21183 auto ReportHWInst = [&](TargetLowering::AtomicExpansionKind Kind) {
21184 OptimizationRemarkEmitter ORE(RMW->getFunction());
21185 ORE.emit(RemarkBuilder: [&]() {
21186 return emitAtomicRMWLegalRemark(RMW, MemLegality, DenormReason);
21187 });
21188 return Kind;
21189 };
21190
21191 auto SSID = RMW->getSyncScopeID();
21192 bool HasSystemScope =
21193 SSID == SyncScope::System ||
21194 SSID == RMW->getContext().getOrInsertSyncScopeID(SSN: *getAtomicScopeIRString(
21195 T: getTargetMachine().getTargetTriple(), S: AtomicScope::System,
21196 /*OneAddressSpace=*/IsSingleAddressSpace: true));
21197
21198 auto Op = RMW->getOperation();
21199 switch (Op) {
21200 case AtomicRMWInst::Xchg:
21201 // PCIe supports add and xchg for system atomics.
21202 return isAtomicRMWLegalXChgTy(RMW)
21203 ? TargetLowering::AtomicExpansionKind::None
21204 : TargetLowering::AtomicExpansionKind::CmpXChg;
21205 case AtomicRMWInst::Add:
21206 // PCIe supports add and xchg for system atomics.
21207 return atomicSupportedIfLegalIntType(RMW);
21208 case AtomicRMWInst::Sub:
21209 case AtomicRMWInst::And:
21210 case AtomicRMWInst::Or:
21211 case AtomicRMWInst::Xor:
21212 case AtomicRMWInst::Max:
21213 case AtomicRMWInst::Min:
21214 case AtomicRMWInst::UMax:
21215 case AtomicRMWInst::UMin:
21216 case AtomicRMWInst::UIncWrap:
21217 case AtomicRMWInst::UDecWrap:
21218 case AtomicRMWInst::USubCond:
21219 case AtomicRMWInst::USubSat: {
21220 if (Op == AtomicRMWInst::USubCond && !Subtarget->hasCondSubInsts())
21221 return AtomicExpansionKind::CmpXChg;
21222 if (Op == AtomicRMWInst::USubSat) {
21223 // The global and buffer forms predate the LDS and flat ones.
21224 if (!Subtarget->hasSubClampInsts() ||
21225 (AS == AMDGPUAS::LOCAL_ADDRESS &&
21226 !Subtarget->hasAtomicDsCondSubClampInsts()) ||
21227 (AS == AMDGPUAS::FLAT_ADDRESS &&
21228 !Subtarget->hasAtomicCondSubClampFlatInsts()))
21229 return AtomicExpansionKind::CmpXChg;
21230 }
21231 if (Op == AtomicRMWInst::USubCond || Op == AtomicRMWInst::USubSat) {
21232 auto *IT = dyn_cast<IntegerType>(Val: RMW->getType());
21233 if (!IT || IT->getBitWidth() != 32)
21234 return AtomicExpansionKind::CmpXChg;
21235 }
21236
21237 if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
21238 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21239 if (Subtarget->hasEmulatedSystemScopeAtomics())
21240 return atomicSupportedIfLegalIntType(RMW);
21241
21242 // On most subtargets, for atomicrmw operations other than add/xchg,
21243 // whether or not the instructions will behave correctly depends on where
21244 // the address physically resides and what interconnect is used in the
21245 // system configuration. On some some targets the instruction will nop,
21246 // and in others synchronization will only occur at degraded device scope.
21247 //
21248 // If the allocation is known local to the device, the instructions should
21249 // work correctly.
21250 if (RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
21251 return atomicSupportedIfLegalIntType(RMW);
21252
21253 // If fine-grained remote memory works at device scope, we don't need to
21254 // do anything.
21255 if (!HasSystemScope &&
21256 Subtarget->hasAgentScopeFineGrainedRemoteMemoryAtomics())
21257 return atomicSupportedIfLegalIntType(RMW);
21258
21259 // If we are targeting a remote allocated address, it depends what kind of
21260 // allocation the address belongs to.
21261 //
21262 // If the allocation is fine-grained (in host memory, or in PCIe peer
21263 // device memory), the operation will fail depending on the target.
21264 //
21265 // Note fine-grained host memory access does work on APUs or if XGMI is
21266 // used, but we do not know if we are targeting an APU or the system
21267 // configuration from the ISA version/target-cpu.
21268 if (RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
21269 return atomicSupportedIfLegalIntType(RMW);
21270
21271 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
21272 Op == AtomicRMWInst::Xor) {
21273 // Atomic sub/or/xor do not work over PCI express, but atomic add
21274 // does. InstCombine transforms these with 0 to or, so undo that.
21275 // Sub-word types are not selectable and take the cmpxchg expansion.
21276 if (const Constant *ConstVal = dyn_cast<Constant>(Val: RMW->getValOperand());
21277 ConstVal && ConstVal->isNullValue() &&
21278 isAtomicRMWLegalIntTy(Ty: RMW->getType()))
21279 return AtomicExpansionKind::CustomExpand;
21280 }
21281
21282 // If the allocation could be in remote, fine-grained memory, the rmw
21283 // instructions may fail. cmpxchg should work, so emit that. On some
21284 // system configurations, PCIe atomics aren't supported so cmpxchg won't
21285 // even work, so you're out of luck anyway.
21286
21287 // In summary:
21288 //
21289 // Cases that may fail:
21290 // - fine-grained pinned host memory
21291 // - fine-grained migratable host memory
21292 // - fine-grained PCIe peer device
21293 //
21294 // Cases that should work, but may be treated overly conservatively.
21295 // - fine-grained host memory on an APU
21296 // - fine-grained XGMI peer device
21297 return AtomicExpansionKind::CmpXChg;
21298 }
21299
21300 return atomicSupportedIfLegalIntType(RMW);
21301 }
21302 case AtomicRMWInst::FAdd: {
21303 Type *Ty = RMW->getType();
21304
21305 // TODO: Handle REGION_ADDRESS
21306 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
21307 // DS F32 FP atomics do respect the denormal mode, but the rounding mode
21308 // is fixed to round-to-nearest-even.
21309 //
21310 // F64 / PK_F16 / PK_BF16 never flush and are also fixed to
21311 // round-to-nearest-even.
21312 //
21313 // We ignore the rounding mode problem, even in strictfp. The C++ standard
21314 // suggests it is OK if the floating-point mode may not match the calling
21315 // thread.
21316 if (Ty->isFloatTy()) {
21317 return Subtarget->hasLDSFPAtomicAddF32() ? AtomicExpansionKind::None
21318 : AtomicExpansionKind::CmpXChg;
21319 }
21320
21321 if (Ty->isDoubleTy()) {
21322 // Ignores denormal mode, but we don't consider flushing mandatory.
21323 return Subtarget->hasLdsAtomicAddF64() ? AtomicExpansionKind::None
21324 : AtomicExpansionKind::CmpXChg;
21325 }
21326
21327 if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
21328 return AtomicExpansionKind::None;
21329
21330 return AtomicExpansionKind::CmpXChg;
21331 }
21332
21333 // LDS atomics respect the denormal mode from the mode register.
21334 //
21335 // Traditionally f32 global/buffer memory atomics would unconditionally
21336 // flush denormals, but newer targets do not flush. f64/f16/bf16 cases never
21337 // flush.
21338 //
21339 // On targets with flat atomic fadd, denormals would flush depending on
21340 // whether the target address resides in LDS or global memory. We consider
21341 // this flat-maybe-flush as will-flush.
21342 if (Ty->isFloatTy() &&
21343 !Subtarget->hasMemoryAtomicFaddF32DenormalSupport()) {
21344 DenormReason = getAtomicFlushDenormalReason(RMW);
21345 if (DenormReason == AtomicFlushDenormalReason::IEEE)
21346 return AtomicExpansionKind::CmpXChg;
21347 }
21348
21349 MemLegality =
21350 getGlobalMemoryFPAtomicLegality(Subtarget: *Subtarget, RMW, HasSystemScope);
21351 if (MemLegality != GlobalFPAtomicLegality::Illegal) {
21352 if (AS == AMDGPUAS::FLAT_ADDRESS) {
21353 // gfx942, gfx12
21354 if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
21355 return ReportHWInst(AtomicExpansionKind::None);
21356 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
21357 // gfx90a, gfx942, gfx12
21358 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
21359 return ReportHWInst(AtomicExpansionKind::None);
21360
21361 // gfx942, gfx12
21362 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
21363 return ReportHWInst(AtomicExpansionKind::None);
21364 } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21365 // gfx90a, gfx942, gfx12
21366 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
21367 return ReportHWInst(AtomicExpansionKind::None);
21368
21369 // While gfx90a/gfx942 supports v2bf16 for global/flat, it does not for
21370 // buffer. gfx12 does have the buffer version.
21371 if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
21372 return ReportHWInst(AtomicExpansionKind::None);
21373 }
21374
21375 // global and flat atomic fadd f64: gfx90a, gfx942.
21376 if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy())
21377 return ReportHWInst(AtomicExpansionKind::None);
21378
21379 if (AS != AMDGPUAS::FLAT_ADDRESS) {
21380 if (Ty->isFloatTy()) {
21381 // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx942,
21382 // gfx11+.
21383 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21384 return ReportHWInst(AtomicExpansionKind::None);
21385 // global/buffer atomic fadd f32 rtn: gfx90a, gfx942, gfx11+.
21386 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21387 return ReportHWInst(AtomicExpansionKind::None);
21388 } else {
21389 // gfx908
21390 if (RMW->use_empty() &&
21391 Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
21392 isV2F16(Ty))
21393 return ReportHWInst(AtomicExpansionKind::None);
21394 }
21395 }
21396
21397 // flat atomic fadd f32: gfx942, gfx11+.
21398 if (AS == AMDGPUAS::FLAT_ADDRESS && Ty->isFloatTy()) {
21399 if (Subtarget->hasFlatAtomicFaddF32Inst())
21400 return ReportHWInst(AtomicExpansionKind::None);
21401
21402 // If it is in flat address space, and the type is float, we will try to
21403 // expand it, if the target supports global and lds atomic fadd. The
21404 // reason we need that is, in the expansion, we emit the check of
21405 // address space. If it is in global address space, we emit the global
21406 // atomic fadd; if it is in shared address space, we emit the LDS atomic
21407 // fadd.
21408 if (Subtarget->hasLDSFPAtomicAddF32()) {
21409 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
21410 return AtomicExpansionKind::CustomExpand;
21411 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
21412 return AtomicExpansionKind::CustomExpand;
21413 }
21414 }
21415 }
21416
21417 return AtomicExpansionKind::CmpXChg;
21418 }
21419 case AtomicRMWInst::FMin:
21420 case AtomicRMWInst::FMax: {
21421 Type *Ty = RMW->getType();
21422
21423 // LDS float and double fmin/fmax were always supported.
21424 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
21425 return Ty->isFloatTy() || Ty->isDoubleTy() ? AtomicExpansionKind::None
21426 : AtomicExpansionKind::CmpXChg;
21427 }
21428
21429 MemLegality =
21430 getGlobalMemoryFPAtomicLegality(Subtarget: *Subtarget, RMW, HasSystemScope);
21431 if (MemLegality != GlobalFPAtomicLegality::Illegal) {
21432 // For flat and global cases:
21433 // float, double in gfx7. Manual claims denormal support.
21434 // Removed in gfx8.
21435 // float, double restored in gfx10.
21436 // double removed again in gfx11, so only f32 for gfx11/gfx12.
21437 //
21438 // For gfx9, gfx90a and gfx942 support f64 for global (same as fadd), but
21439 // no f32.
21440 if (AS == AMDGPUAS::FLAT_ADDRESS) {
21441 if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy())
21442 return ReportHWInst(AtomicExpansionKind::None);
21443 if (Subtarget->hasAtomicFMinFMaxF64FlatInsts() && Ty->isDoubleTy())
21444 return ReportHWInst(AtomicExpansionKind::None);
21445 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS) ||
21446 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
21447 if (Subtarget->hasAtomicFMinFMaxF32GlobalInsts() && Ty->isFloatTy())
21448 return ReportHWInst(AtomicExpansionKind::None);
21449 if (Subtarget->hasAtomicFMinFMaxF64GlobalInsts() && Ty->isDoubleTy())
21450 return ReportHWInst(AtomicExpansionKind::None);
21451 }
21452 }
21453
21454 return AtomicExpansionKind::CmpXChg;
21455 }
21456 case AtomicRMWInst::Nand:
21457 case AtomicRMWInst::FSub:
21458 default:
21459 return AtomicExpansionKind::CmpXChg;
21460 }
21461
21462 llvm_unreachable("covered atomicrmw op switch");
21463}
21464
21465TargetLowering::AtomicExpansionKind
21466SITargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
21467 if (LI->getPointerAddressSpace() == AMDGPUAS::VGPR)
21468 return AtomicExpansionKind::NotAtomic;
21469 return LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
21470 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
21471 : AtomicExpansionKind::None;
21472}
21473
21474TargetLowering::AtomicExpansionKind
21475SITargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
21476 if (SI->getPointerAddressSpace() == AMDGPUAS::VGPR)
21477 return AtomicExpansionKind::NotAtomic;
21478 return SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
21479 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
21480 : AtomicExpansionKind::None;
21481}
21482
21483TargetLowering::AtomicExpansionKind
21484SITargetLowering::shouldExpandAtomicCmpXchgInIR(
21485 const AtomicCmpXchgInst *CmpX) const {
21486 unsigned AddrSpace = CmpX->getPointerAddressSpace();
21487 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
21488 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
21489
21490 if (AddrSpace == AMDGPUAS::VGPR)
21491 return AtomicExpansionKind::NotAtomic;
21492
21493 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS || !flatInstrMayAccessPrivate(I: CmpX))
21494 return AtomicExpansionKind::None;
21495
21496 const DataLayout &DL = CmpX->getDataLayout();
21497
21498 Type *ValTy = CmpX->getNewValOperand()->getType();
21499
21500 // If a 64-bit flat atomic may alias private, we need to avoid using the
21501 // atomic in the private case.
21502 return DL.getTypeSizeInBits(Ty: ValTy) == 64 ? AtomicExpansionKind::CustomExpand
21503 : AtomicExpansionKind::None;
21504}
21505
21506const TargetRegisterClass *
21507SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
21508 const TargetRegisterClass *RC = TargetLoweringBase::getRegClassFor(VT, isDivergent: false);
21509 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
21510 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
21511 return Subtarget->isWave64() ? &AMDGPU::SReg_64RegClass
21512 : &AMDGPU::SReg_32RegClass;
21513 if (!TRI->isSGPRClass(RC) && !isDivergent)
21514 return TRI->getEquivalentSGPRClass(VRC: RC);
21515 if (TRI->isSGPRClass(RC) && isDivergent) {
21516 if (Subtarget->hasGFX90AInsts())
21517 return TRI->getEquivalentAVClass(SRC: RC);
21518 return TRI->getEquivalentVGPRClass(SRC: RC);
21519 }
21520
21521 return RC;
21522}
21523
21524// FIXME: This is a workaround for DivergenceAnalysis not understanding always
21525// uniform values (as produced by the mask results of control flow intrinsics)
21526// used outside of divergent blocks. The phi users need to also be treated as
21527// always uniform.
21528//
21529// FIXME: DA is no longer in-use. Does this still apply to UniformityAnalysis?
21530static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited,
21531 unsigned WaveSize) {
21532 // FIXME: We assume we never cast the mask results of a control flow
21533 // intrinsic.
21534 // Early exit if the type won't be consistent as a compile time hack.
21535 IntegerType *IT = dyn_cast<IntegerType>(Val: V->getType());
21536 if (!IT || IT->getBitWidth() != WaveSize)
21537 return false;
21538
21539 if (!isa<Instruction>(Val: V))
21540 return false;
21541 if (!Visited.insert(Ptr: V).second)
21542 return false;
21543 bool Result = false;
21544 for (const auto *U : V->users()) {
21545 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: U)) {
21546 if (V == U->getOperand(i: 1)) {
21547 switch (Intrinsic->getIntrinsicID()) {
21548 default:
21549 Result = false;
21550 break;
21551 case Intrinsic::amdgcn_if_break:
21552 case Intrinsic::amdgcn_if:
21553 case Intrinsic::amdgcn_else:
21554 Result = true;
21555 break;
21556 }
21557 }
21558 if (V == U->getOperand(i: 0)) {
21559 switch (Intrinsic->getIntrinsicID()) {
21560 default:
21561 Result = false;
21562 break;
21563 case Intrinsic::amdgcn_end_cf:
21564 case Intrinsic::amdgcn_loop:
21565 Result = true;
21566 break;
21567 }
21568 }
21569 } else {
21570 Result = hasCFUser(V: U, Visited, WaveSize);
21571 }
21572 if (Result)
21573 break;
21574 }
21575 return Result;
21576}
21577
21578bool SITargetLowering::requiresUniformRegister(MachineFunction &MF,
21579 const Value *V) const {
21580 if (const CallInst *CI = dyn_cast<CallInst>(Val: V)) {
21581 if (CI->isInlineAsm()) {
21582 // FIXME: This cannot give a correct answer. This should only trigger in
21583 // the case where inline asm returns mixed SGPR and VGPR results, used
21584 // outside the defining block. We don't have a specific result to
21585 // consider, so this assumes if any value is SGPR, the overall register
21586 // also needs to be SGPR.
21587 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
21588 TargetLowering::AsmOperandInfoVector TargetConstraints = ParseConstraints(
21589 DL: MF.getDataLayout(), TRI: Subtarget->getRegisterInfo(), Call: *CI);
21590 for (auto &TC : TargetConstraints) {
21591 if (TC.Type == InlineAsm::isOutput) {
21592 ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
21593 const TargetRegisterClass *RC =
21594 getRegForInlineAsmConstraint(TRI_: SIRI, Constraint: TC.ConstraintCode,
21595 VT: TC.ConstraintVT)
21596 .second;
21597 if (RC && SIRI->isSGPRClass(RC))
21598 return true;
21599 }
21600 }
21601 }
21602 }
21603 SmallPtrSet<const Value *, 16> Visited;
21604 return hasCFUser(V, Visited, WaveSize: Subtarget->getWavefrontSize());
21605}
21606
21607bool SITargetLowering::hasMemSDNodeUser(SDNode *N) const {
21608 for (SDUse &Use : N->uses()) {
21609 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: Use.getUser())) {
21610 if (getBasePtrIndex(N: M) == Use.getOperandNo())
21611 return true;
21612 }
21613 }
21614 return false;
21615}
21616
21617bool SITargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
21618 SDValue N1) const {
21619 if (!N0.hasOneUse())
21620 return false;
21621 // Take care of the opportunity to keep N0 uniform
21622 if (N0->isDivergent() || !N1->isDivergent())
21623 return true;
21624 // Check if we have a good chance to form the memory access pattern with the
21625 // base and offset
21626 return (DAG.isBaseWithConstantOffset(Op: N0) &&
21627 hasMemSDNodeUser(N: *N0->user_begin()));
21628}
21629
21630bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
21631 Register N0, Register N1) const {
21632 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
21633}
21634
21635MachineMemOperand::Flags
21636SITargetLowering::getTargetMMOFlags(const Instruction &I) const {
21637 // Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
21638 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
21639 if (I.getMetadata(Kind: "amdgpu.noclobber"))
21640 Flags |= MONoClobber;
21641 if (I.getMetadata(Kind: "amdgpu.last.use"))
21642 Flags |= MOLastUse;
21643 if (MDNode *N = I.getMetadata(Kind: "amdgpu.cfs")) {
21644 const ConstantAsMetadata *CAM = cast<ConstantAsMetadata>(Val: N->getOperand(I: 0));
21645 ConstantInt *CI = cast<ConstantInt>(Val: CAM->getValue());
21646 unsigned CFS = CI->getZExtValue() & AMDGPU::CPol::CFS_MASK;
21647 static_assert((MOCFSB0 << 1) == MOCFSB1,
21648 "MOCFSB0 and MOCFSB1 are not adjacent bits");
21649 Flags |= static_cast<MachineMemOperand::Flags>(CFS * MOCFSB0);
21650 }
21651 return Flags;
21652}
21653
21654void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
21655 Instruction *AI) const {
21656 // Given: atomicrmw fadd ptr %addr, float %val ordering
21657 //
21658 // With this expansion we produce the following code:
21659 // [...]
21660 // %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %addr)
21661 // br i1 %is.shared, label %atomicrmw.shared, label %atomicrmw.check.private
21662 //
21663 // atomicrmw.shared:
21664 // %cast.shared = addrspacecast ptr %addr to ptr addrspace(3)
21665 // %loaded.shared = atomicrmw fadd ptr addrspace(3) %cast.shared,
21666 // float %val ordering
21667 // br label %atomicrmw.phi
21668 //
21669 // atomicrmw.check.private:
21670 // %is.private = call i1 @llvm.amdgcn.is.private(ptr %int8ptr)
21671 // br i1 %is.private, label %atomicrmw.private, label %atomicrmw.global
21672 //
21673 // atomicrmw.private:
21674 // %cast.private = addrspacecast ptr %addr to ptr addrspace(5)
21675 // %loaded.private = load float, ptr addrspace(5) %cast.private
21676 // %val.new = fadd float %loaded.private, %val
21677 // store float %val.new, ptr addrspace(5) %cast.private
21678 // br label %atomicrmw.phi
21679 //
21680 // atomicrmw.global:
21681 // %cast.global = addrspacecast ptr %addr to ptr addrspace(1)
21682 // %loaded.global = atomicrmw fadd ptr addrspace(1) %cast.global,
21683 // float %val ordering
21684 // br label %atomicrmw.phi
21685 //
21686 // atomicrmw.phi:
21687 // %loaded.phi = phi float [ %loaded.shared, %atomicrmw.shared ],
21688 // [ %loaded.private, %atomicrmw.private ],
21689 // [ %loaded.global, %atomicrmw.global ]
21690 // br label %atomicrmw.end
21691 //
21692 // atomicrmw.end:
21693 // [...]
21694 //
21695 //
21696 // For 64-bit atomics which may reside in private memory, we perform a simpler
21697 // version that only inserts the private check, and uses the flat operation.
21698
21699 IRBuilder<> Builder(AI);
21700 LLVMContext &Ctx = Builder.getContext();
21701
21702 auto *RMW = dyn_cast<AtomicRMWInst>(Val: AI);
21703 const unsigned PtrOpIdx = RMW ? AtomicRMWInst::getPointerOperandIndex()
21704 : AtomicCmpXchgInst::getPointerOperandIndex();
21705 Value *Addr = AI->getOperand(i: PtrOpIdx);
21706
21707 /// TODO: Only need to check private, then emit flat-known-not private (no
21708 /// need for shared block, or cast to global).
21709 AtomicCmpXchgInst *CX = dyn_cast<AtomicCmpXchgInst>(Val: AI);
21710
21711 Align Alignment;
21712 if (RMW)
21713 Alignment = RMW->getAlign();
21714 else if (CX)
21715 Alignment = CX->getAlign();
21716 else
21717 llvm_unreachable("unhandled atomic operation");
21718
21719 // FullFlatEmulation is true if we need to issue the private, shared, and
21720 // global cases.
21721 //
21722 // If this is false, we are only dealing with the flat-targeting-private case,
21723 // where we only insert a check for private and still use the flat instruction
21724 // for global and shared.
21725
21726 bool FullFlatEmulation =
21727 RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
21728 ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
21729 (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
21730 RMW->getType()->isDoubleTy()));
21731
21732 // If the return value isn't used, do not introduce a false use in the phi.
21733 bool ReturnValueIsUsed = !AI->use_empty();
21734
21735 BasicBlock *BB = Builder.GetInsertBlock();
21736 Function *F = BB->getParent();
21737 BasicBlock *ExitBB =
21738 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
21739 BasicBlock *SharedBB = nullptr;
21740
21741 BasicBlock *CheckPrivateBB = BB;
21742 if (FullFlatEmulation) {
21743 SharedBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.shared", Parent: F, InsertBefore: ExitBB);
21744 CheckPrivateBB =
21745 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.check.private", Parent: F, InsertBefore: ExitBB);
21746 }
21747
21748 BasicBlock *PrivateBB =
21749 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.private", Parent: F, InsertBefore: ExitBB);
21750 BasicBlock *GlobalBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.global", Parent: F, InsertBefore: ExitBB);
21751 BasicBlock *PhiBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.phi", Parent: F, InsertBefore: ExitBB);
21752
21753 std::prev(x: BB->end())->eraseFromParent();
21754 Builder.SetInsertPoint(BB);
21755
21756 Value *LoadedShared = nullptr;
21757 if (FullFlatEmulation) {
21758 Value *IsShared = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_shared,
21759 Args: {Addr}, FMFSource: nullptr, Name: "is.shared");
21760 Builder.CreateCondBr(Cond: IsShared, True: SharedBB, False: CheckPrivateBB);
21761 Builder.SetInsertPoint(SharedBB);
21762 Value *CastToLocal = Builder.CreateAddrSpaceCast(
21763 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS));
21764
21765 Instruction *Clone = AI->clone();
21766 Clone->insertInto(ParentBB: SharedBB, It: SharedBB->end());
21767 Clone->getOperandUse(i: PtrOpIdx).set(CastToLocal);
21768 LoadedShared = Clone;
21769
21770 Builder.CreateBr(Dest: PhiBB);
21771 Builder.SetInsertPoint(CheckPrivateBB);
21772 }
21773
21774 Value *IsPrivate = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_private,
21775 Args: {Addr}, FMFSource: nullptr, Name: "is.private");
21776 Builder.CreateCondBr(Cond: IsPrivate, True: PrivateBB, False: GlobalBB);
21777
21778 Builder.SetInsertPoint(PrivateBB);
21779
21780 Value *CastToPrivate = Builder.CreateAddrSpaceCast(
21781 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::PRIVATE_ADDRESS));
21782
21783 Value *LoadedPrivate;
21784 if (RMW) {
21785 LoadedPrivate = Builder.CreateAlignedLoad(
21786 Ty: RMW->getType(), Ptr: CastToPrivate, Align: RMW->getAlign(), isVolatile: RMW->isVolatile(),
21787 Name: "loaded.private");
21788
21789 Value *NewVal = buildAtomicRMWValue(Op: RMW->getOperation(), Builder,
21790 Loaded: LoadedPrivate, Val: RMW->getValOperand());
21791
21792 Builder.CreateAlignedStore(Val: NewVal, Ptr: CastToPrivate, Align: RMW->getAlign(),
21793 isVolatile: RMW->isVolatile());
21794 } else {
21795 auto [ResultLoad, Equal] = buildCmpXchgValue(
21796 Builder, Ptr: CastToPrivate, Cmp: CX->getCompareOperand(), Val: CX->getNewValOperand(),
21797 Alignment: CX->getAlign(), IsVolatile: CX->isVolatile());
21798
21799 Value *Insert = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CX->getType()),
21800 Val: ResultLoad, Idxs: 0);
21801 LoadedPrivate = Builder.CreateInsertValue(Agg: Insert, Val: Equal, Idxs: 1);
21802 }
21803
21804 Builder.CreateBr(Dest: PhiBB);
21805
21806 Builder.SetInsertPoint(GlobalBB);
21807
21808 // Continue using a flat instruction if we only emitted the check for private.
21809 Instruction *LoadedGlobal = AI;
21810 if (FullFlatEmulation) {
21811 Value *CastToGlobal = Builder.CreateAddrSpaceCast(
21812 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
21813 AI->getOperandUse(i: PtrOpIdx).set(CastToGlobal);
21814 }
21815
21816 AI->removeFromParent();
21817 AI->insertInto(ParentBB: GlobalBB, It: GlobalBB->end());
21818
21819 // The new atomicrmw may go through another round of legalization later.
21820 if (!FullFlatEmulation) {
21821 // We inserted the runtime check already, make sure we do not try to
21822 // re-expand this.
21823 // TODO: Should union with any existing metadata.
21824 MDBuilder MDB(F->getContext());
21825 MDNode *RangeNotPrivate =
21826 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
21827 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
21828 LoadedGlobal->setMetadata(KindID: LLVMContext::MD_noalias_addrspace,
21829 Node: RangeNotPrivate);
21830 }
21831
21832 Builder.CreateBr(Dest: PhiBB);
21833
21834 Builder.SetInsertPoint(PhiBB);
21835
21836 if (ReturnValueIsUsed) {
21837 PHINode *Loaded = Builder.CreatePHI(Ty: AI->getType(), NumReservedValues: 3);
21838 AI->replaceAllUsesWith(V: Loaded);
21839 if (FullFlatEmulation)
21840 Loaded->addIncoming(V: LoadedShared, BB: SharedBB);
21841 Loaded->addIncoming(V: LoadedPrivate, BB: PrivateBB);
21842 Loaded->addIncoming(V: LoadedGlobal, BB: GlobalBB);
21843 Loaded->takeName(V: AI);
21844 }
21845
21846 Builder.CreateBr(Dest: ExitBB);
21847}
21848
21849static void convertScratchAtomicToFlatAtomic(Instruction *I,
21850 unsigned PtrOpIdx) {
21851 Value *PtrOp = I->getOperand(i: PtrOpIdx);
21852 assert(PtrOp->getType()->getPointerAddressSpace() ==
21853 AMDGPUAS::PRIVATE_ADDRESS);
21854
21855 Type *FlatPtr = PointerType::get(C&: I->getContext(), AddressSpace: AMDGPUAS::FLAT_ADDRESS);
21856 Value *ASCast = CastInst::CreatePointerCast(S: PtrOp, Ty: FlatPtr, Name: "scratch.ascast",
21857 InsertBefore: I->getIterator());
21858 I->setOperand(i: PtrOpIdx, Val: ASCast);
21859}
21860
21861void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
21862 AtomicRMWInst::BinOp Op = AI->getOperation();
21863
21864 if (AI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21865 return convertScratchAtomicToFlatAtomic(I: AI, PtrOpIdx: AI->getPointerOperandIndex());
21866
21867 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
21868 Op == AtomicRMWInst::Xor) {
21869 if (const auto *ConstVal = dyn_cast<Constant>(Val: AI->getValOperand());
21870 ConstVal && ConstVal->isNullValue() &&
21871 isAtomicRMWLegalIntTy(Ty: AI->getType())) {
21872 // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
21873 AI->setOperation(AtomicRMWInst::Add);
21874
21875 // We may still need the private-alias-flat handling below.
21876
21877 // TODO: Skip this for cases where we cannot access remote memory.
21878 }
21879 }
21880
21881 // The non-flat expansions should only perform the de-canonicalization of
21882 // identity values.
21883 if (AI->getPointerAddressSpace() != AMDGPUAS::FLAT_ADDRESS)
21884 return;
21885
21886 emitExpandAtomicAddrSpacePredicate(AI);
21887}
21888
21889void SITargetLowering::emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const {
21890 if (CI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21891 return convertScratchAtomicToFlatAtomic(I: CI, PtrOpIdx: CI->getPointerOperandIndex());
21892
21893 emitExpandAtomicAddrSpacePredicate(AI: CI);
21894}
21895
21896void SITargetLowering::emitExpandAtomicLoad(LoadInst *LI) const {
21897 if (LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21898 return convertScratchAtomicToFlatAtomic(I: LI, PtrOpIdx: LI->getPointerOperandIndex());
21899
21900 llvm_unreachable(
21901 "Expand Atomic Load only handles SCRATCH -> FLAT conversion");
21902}
21903
21904void SITargetLowering::emitExpandAtomicStore(StoreInst *SI) const {
21905 if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21906 return convertScratchAtomicToFlatAtomic(I: SI, PtrOpIdx: SI->getPointerOperandIndex());
21907
21908 llvm_unreachable(
21909 "Expand Atomic Store only handles SCRATCH -> FLAT conversion");
21910}
21911