1//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Custom DAG lowering for SI
11//
12//===----------------------------------------------------------------------===//
13
14#include "SIISelLowering.h"
15#include "AMDGPU.h"
16#include "AMDGPUInstrInfo.h"
17#include "AMDGPULaneMaskUtils.h"
18#include "AMDGPUMemoryUtils.h"
19#include "AMDGPUSelectionDAGInfo.h"
20#include "AMDGPUTargetMachine.h"
21#include "GCNSubtarget.h"
22#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
23#include "SIMachineFunctionInfo.h"
24#include "SIRegisterInfo.h"
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/FloatingPointMode.h"
27#include "llvm/ADT/Statistic.h"
28#include "llvm/Analysis/OptimizationRemarkEmitter.h"
29#include "llvm/Analysis/UniformityAnalysis.h"
30#include "llvm/CodeGen/Analysis.h"
31#include "llvm/CodeGen/ByteProvider.h"
32#include "llvm/CodeGen/FunctionLoweringInfo.h"
33#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
34#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
35#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineLoopInfo.h"
39#include "llvm/CodeGen/PseudoSourceValueManager.h"
40#include "llvm/CodeGen/SDPatternMatch.h"
41#include "llvm/IR/DiagnosticInfo.h"
42#include "llvm/IR/IRBuilder.h"
43#include "llvm/IR/IntrinsicInst.h"
44#include "llvm/IR/IntrinsicsAMDGPU.h"
45#include "llvm/IR/IntrinsicsR600.h"
46#include "llvm/IR/MDBuilder.h"
47#include "llvm/Support/CommandLine.h"
48#include "llvm/Support/KnownBits.h"
49#include "llvm/Support/ModRef.h"
50#include "llvm/Transforms/Utils/LowerAtomic.h"
51#include <optional>
52
53using namespace llvm;
54using namespace llvm::SDPatternMatch;
55
56#define DEBUG_TYPE "si-lower"
57
58STATISTIC(NumTailCalls, "Number of tail calls");
59
60static cl::opt<bool>
61 DisableLoopAlignment("amdgpu-disable-loop-alignment",
62 cl::desc("Do not align and prefetch loops"),
63 cl::init(Val: false));
64
65static cl::opt<bool> UseDivergentRegisterIndexing(
66 "amdgpu-use-divergent-register-indexing", cl::Hidden,
67 cl::desc("Use indirect register addressing for divergent indexes"),
68 cl::init(Val: false));
69
70static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
71 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
72 return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
73}
74
75static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF) {
76 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
77 return Info->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign();
78}
79
80static unsigned findFirstFreeSGPR(CCState &CCInfo) {
81 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
82 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
83 if (!CCInfo.isAllocated(Reg: AMDGPU::SGPR0 + Reg)) {
84 return AMDGPU::SGPR0 + Reg;
85 }
86 }
87 llvm_unreachable("Cannot allocate sgpr");
88}
89
90SITargetLowering::SITargetLowering(const TargetMachine &TM,
91 const GCNSubtarget &STI)
92 : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI) {
93 addRegisterClass(VT: MVT::i1, RC: &AMDGPU::VReg_1RegClass);
94 addRegisterClass(VT: MVT::i64, RC: &AMDGPU::SReg_64RegClass);
95
96 addRegisterClass(VT: MVT::i32, RC: &AMDGPU::SReg_32RegClass);
97
98 const SIRegisterInfo *TRI = STI.getRegisterInfo();
99 const TargetRegisterClass *V32RegClass =
100 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 32);
101 addRegisterClass(VT: MVT::f32, RC: V32RegClass);
102
103 addRegisterClass(VT: MVT::v2i32, RC: &AMDGPU::SReg_64RegClass);
104
105 const TargetRegisterClass *V64RegClass =
106 TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 64);
107
108 addRegisterClass(VT: MVT::f64, RC: V64RegClass);
109 addRegisterClass(VT: MVT::v2f32, RC: V64RegClass);
110 addRegisterClass(VT: MVT::Untyped, RC: V64RegClass);
111
112 addRegisterClass(VT: MVT::v3i32, RC: &AMDGPU::SGPR_96RegClass);
113 addRegisterClass(VT: MVT::v3f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 96));
114
115 addRegisterClass(VT: MVT::v2i64, RC: &AMDGPU::SGPR_128RegClass);
116 addRegisterClass(VT: MVT::v2f64, RC: &AMDGPU::SGPR_128RegClass);
117
118 addRegisterClass(VT: MVT::v4i32, RC: &AMDGPU::SGPR_128RegClass);
119 addRegisterClass(VT: MVT::v4f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 128));
120
121 addRegisterClass(VT: MVT::v5i32, RC: &AMDGPU::SGPR_160RegClass);
122 addRegisterClass(VT: MVT::v5f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 160));
123
124 addRegisterClass(VT: MVT::v6i32, RC: &AMDGPU::SGPR_192RegClass);
125 addRegisterClass(VT: MVT::v6f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
126
127 addRegisterClass(VT: MVT::v3i64, RC: &AMDGPU::SGPR_192RegClass);
128 addRegisterClass(VT: MVT::v3f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 192));
129
130 addRegisterClass(VT: MVT::v7i32, RC: &AMDGPU::SGPR_224RegClass);
131 addRegisterClass(VT: MVT::v7f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 224));
132
133 addRegisterClass(VT: MVT::v8i32, RC: &AMDGPU::SGPR_256RegClass);
134 addRegisterClass(VT: MVT::v8f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
135
136 addRegisterClass(VT: MVT::v4i64, RC: &AMDGPU::SGPR_256RegClass);
137 addRegisterClass(VT: MVT::v4f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 256));
138
139 addRegisterClass(VT: MVT::v9i32, RC: &AMDGPU::SGPR_288RegClass);
140 addRegisterClass(VT: MVT::v9f32, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 288));
141
142 addRegisterClass(VT: MVT::v10i32, RC: &AMDGPU::SGPR_320RegClass);
143 addRegisterClass(VT: MVT::v10f32,
144 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 320));
145
146 addRegisterClass(VT: MVT::v11i32, RC: &AMDGPU::SGPR_352RegClass);
147 addRegisterClass(VT: MVT::v11f32,
148 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 352));
149
150 addRegisterClass(VT: MVT::v12i32, RC: &AMDGPU::SGPR_384RegClass);
151 addRegisterClass(VT: MVT::v12f32,
152 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 384));
153
154 addRegisterClass(VT: MVT::v16i32, RC: &AMDGPU::SGPR_512RegClass);
155 addRegisterClass(VT: MVT::v16f32,
156 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
157
158 addRegisterClass(VT: MVT::v8i64, RC: &AMDGPU::SGPR_512RegClass);
159 addRegisterClass(VT: MVT::v8f64, RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 512));
160
161 addRegisterClass(VT: MVT::v16i64, RC: &AMDGPU::SGPR_1024RegClass);
162 addRegisterClass(VT: MVT::v16f64,
163 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
164
165 if (Subtarget->has16BitInsts()) {
166 if (Subtarget->useRealTrue16Insts()) {
167 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::VGPR_16RegClass);
168 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::VGPR_16RegClass);
169 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::VGPR_16RegClass);
170 } else {
171 addRegisterClass(VT: MVT::i16, RC: &AMDGPU::SReg_32RegClass);
172 addRegisterClass(VT: MVT::f16, RC: &AMDGPU::SReg_32RegClass);
173 addRegisterClass(VT: MVT::bf16, RC: &AMDGPU::SReg_32RegClass);
174 }
175
176 // Unless there are also VOP3P operations, not operations are really legal.
177 addRegisterClass(VT: MVT::v2i16, RC: &AMDGPU::SReg_32RegClass);
178 addRegisterClass(VT: MVT::v2f16, RC: &AMDGPU::SReg_32RegClass);
179 addRegisterClass(VT: MVT::v2bf16, RC: &AMDGPU::SReg_32RegClass);
180 addRegisterClass(VT: MVT::v4i16, RC: &AMDGPU::SReg_64RegClass);
181 addRegisterClass(VT: MVT::v4f16, RC: &AMDGPU::SReg_64RegClass);
182 addRegisterClass(VT: MVT::v4bf16, RC: &AMDGPU::SReg_64RegClass);
183 addRegisterClass(VT: MVT::v8i16, RC: &AMDGPU::SGPR_128RegClass);
184 addRegisterClass(VT: MVT::v8f16, RC: &AMDGPU::SGPR_128RegClass);
185 addRegisterClass(VT: MVT::v8bf16, RC: &AMDGPU::SGPR_128RegClass);
186 addRegisterClass(VT: MVT::v16i16, RC: &AMDGPU::SGPR_256RegClass);
187 addRegisterClass(VT: MVT::v16f16, RC: &AMDGPU::SGPR_256RegClass);
188 addRegisterClass(VT: MVT::v16bf16, RC: &AMDGPU::SGPR_256RegClass);
189 addRegisterClass(VT: MVT::v32i16, RC: &AMDGPU::SGPR_512RegClass);
190 addRegisterClass(VT: MVT::v32f16, RC: &AMDGPU::SGPR_512RegClass);
191 addRegisterClass(VT: MVT::v32bf16, RC: &AMDGPU::SGPR_512RegClass);
192 }
193
194 addRegisterClass(VT: MVT::v32i32, RC: &AMDGPU::VReg_1024RegClass);
195 addRegisterClass(VT: MVT::v32f32,
196 RC: TRI->getDefaultVectorSuperClassForBitWidth(BitWidth: 1024));
197
198 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
199
200 setMinFunctionAlignment(Align(4));
201 setPrefFunctionAlignment(Align(STI.getInstCacheLineSize()));
202
203 // The boolean content concept here is too inflexible. Compares only ever
204 // really produce a 1-bit result. Any copy/extend from these will turn into a
205 // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
206 // it's what most targets use.
207 setBooleanContents(ZeroOrOneBooleanContent);
208 setBooleanVectorContents(ZeroOrOneBooleanContent);
209
210 // We need to custom lower vector stores from local memory
211 setOperationAction(Ops: ISD::LOAD,
212 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
213 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
214 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
215 MVT::i1, MVT::v32i32},
216 Action: Custom);
217
218 setOperationAction(Ops: ISD::STORE,
219 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
220 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
221 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
222 MVT::i1, MVT::v32i32},
223 Action: Custom);
224
225 if (isTypeLegal(VT: MVT::bf16)) {
226 for (unsigned Opc :
227 {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
228 ISD::FREM, ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
229 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FSQRT, ISD::FCBRT,
230 ISD::FSIN, ISD::FCOS, ISD::FPOW, ISD::FPOWI,
231 ISD::FLDEXP, ISD::FFREXP, ISD::FLOG, ISD::FLOG2,
232 ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10,
233 ISD::FCEIL, ISD::FTRUNC, ISD::FRINT, ISD::FNEARBYINT,
234 ISD::FROUND, ISD::FROUNDEVEN, ISD::FFLOOR, ISD::FCANONICALIZE,
235 ISD::SETCC}) {
236 setOperationAction(Op: Opc, VT: MVT::bf16, Action: Promote);
237 }
238
239 // Only targets with packed bf16 instructions, e.g. gfx13.
240 if (Subtarget->hasBF16PackedInsts()) {
241 // Turn fsub into fadd(x, fneg y) so it reuses the packed v_pk_add_bf16
242 // path instead of promoting to f32.
243 setOperationAction(Op: ISD::FSUB, VT: MVT::bf16, Action: Expand);
244 // Widen scalar fadd to a v2bf16 operation with an unused high lane.
245 setOperationAction(Op: ISD::FADD, VT: MVT::bf16, Action: Custom);
246 }
247
248 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::bf16, Action: Expand);
249
250 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Promote);
251 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::bf16, DestVT: MVT::i16);
252
253 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Legal);
254 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Legal);
255 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Legal);
256
257 // We only need to custom lower because we can't specify an action for bf16
258 // sources.
259 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
260 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
261 }
262
263 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i16, Action: Expand);
264 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i16, Action: Expand);
265 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Expand);
266 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i16, Action: Expand);
267 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i16, Action: Expand);
268 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i16, Action: Expand);
269 setTruncStoreAction(ValVT: MVT::v2i32, MemVT: MVT::v2i8, Action: Expand);
270 setTruncStoreAction(ValVT: MVT::v4i32, MemVT: MVT::v4i8, Action: Expand);
271 setTruncStoreAction(ValVT: MVT::v8i32, MemVT: MVT::v8i8, Action: Expand);
272 setTruncStoreAction(ValVT: MVT::v16i32, MemVT: MVT::v16i8, Action: Expand);
273 setTruncStoreAction(ValVT: MVT::v32i32, MemVT: MVT::v32i8, Action: Expand);
274 setTruncStoreAction(ValVT: MVT::v2i16, MemVT: MVT::v2i8, Action: Expand);
275 setTruncStoreAction(ValVT: MVT::v4i16, MemVT: MVT::v4i8, Action: Expand);
276 setTruncStoreAction(ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Expand);
277 setTruncStoreAction(ValVT: MVT::v16i16, MemVT: MVT::v16i8, Action: Expand);
278 setTruncStoreAction(ValVT: MVT::v32i16, MemVT: MVT::v32i8, Action: Expand);
279
280 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
281 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
282 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i8, Action: Expand);
283 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i8, Action: Expand);
284 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i16, Action: Expand);
285 setTruncStoreAction(ValVT: MVT::v8i64, MemVT: MVT::v8i32, Action: Expand);
286 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i32, Action: Expand);
287
288 setOperationAction(Ops: ISD::GlobalAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
289 setOperationAction(Ops: ISD::BlockAddress, VTs: {MVT::i32, MVT::i64}, Action: Custom);
290 setOperationAction(Ops: ISD::ExternalSymbol, VTs: {MVT::i32, MVT::i64}, Action: Custom);
291
292 setOperationAction(Op: ISD::SELECT, VT: MVT::i1, Action: Promote);
293 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Custom);
294 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Promote);
295 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::f64, DestVT: MVT::i64);
296
297 setOperationAction(Ops: ISD::FSQRT, VTs: {MVT::f32, MVT::f64}, Action: Custom);
298
299 setOperationAction(Ops: ISD::SELECT_CC,
300 VTs: {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Action: Expand);
301
302 setOperationAction(Op: ISD::SETCC, VT: MVT::i1, Action: Promote);
303 setOperationAction(Ops: ISD::SETCC, VTs: {MVT::v2i1, MVT::v4i1}, Action: Expand);
304 AddPromotedToType(Opc: ISD::SETCC, OrigVT: MVT::i1, DestVT: MVT::i32);
305
306 setOperationAction(Ops: ISD::TRUNCATE,
307 VTs: {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
308 MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
309 MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
310 Action: Expand);
311 setOperationAction(Ops: ISD::FP_ROUND,
312 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
313 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
314 MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
315 Action: Expand);
316
317 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG,
318 VTs: {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
319 MVT::v3i16, MVT::v4i16, MVT::Other},
320 Action: Custom);
321
322 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
323 setOperationAction(Ops: ISD::BR_CC,
324 VTs: {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Action: Expand);
325
326 setOperationAction(Ops: {ISD::ABS, ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Legal);
327 setOperationAction(Ops: {ISD::UADDO, ISD::USUBO}, VT: MVT::i64, Action: Legal);
328
329 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i32, Action: Legal);
330 setOperationAction(Ops: {ISD::UADDO_CARRY, ISD::USUBO_CARRY}, VT: MVT::i64, Action: Legal);
331
332 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, VT: MVT::i64,
333 Action: Expand);
334
335 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
336
337 // We only support LOAD/STORE and vector manipulation ops for vectors
338 // with > 4 elements.
339 for (MVT VT :
340 {MVT::v8i32, MVT::v8f32, MVT::v9i32, MVT::v9f32, MVT::v10i32,
341 MVT::v10f32, MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
342 MVT::v16i32, MVT::v16f32, MVT::v2i64, MVT::v2f64, MVT::v4i16,
343 MVT::v4f16, MVT::v4bf16, MVT::v3i64, MVT::v3f64, MVT::v6i32,
344 MVT::v6f32, MVT::v4i64, MVT::v4f64, MVT::v8i64, MVT::v8f64,
345 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
346 MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
347 MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
348 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
349 switch (Op) {
350 case ISD::LOAD:
351 case ISD::STORE:
352 case ISD::BUILD_VECTOR:
353 case ISD::BITCAST:
354 case ISD::UNDEF:
355 case ISD::EXTRACT_VECTOR_ELT:
356 case ISD::INSERT_VECTOR_ELT:
357 case ISD::SCALAR_TO_VECTOR:
358 case ISD::IS_FPCLASS:
359 break;
360 case ISD::EXTRACT_SUBVECTOR:
361 case ISD::INSERT_SUBVECTOR:
362 case ISD::CONCAT_VECTORS:
363 setOperationAction(Op, VT, Action: Custom);
364 break;
365 default:
366 setOperationAction(Op, VT, Action: Expand);
367 break;
368 }
369 }
370 }
371
372 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v4f32, Action: Expand);
373
374 // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
375 // is expanded to avoid having two separate loops in case the index is a VGPR.
376
377 // Most operations are naturally 32-bit vector operations. We only support
378 // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
379 for (MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
380 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
381 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
382
383 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
384 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
385
386 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
387 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v4i32);
388
389 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
390 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v4i32);
391 }
392
393 for (MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
394 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
395 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
396
397 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
398 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
399
400 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
401 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v6i32);
402
403 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
404 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v6i32);
405 }
406
407 for (MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
408 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
409 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
410
411 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
412 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
413
414 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
415 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v8i32);
416
417 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
418 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v8i32);
419 }
420
421 for (MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
422 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
423 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
424
425 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
426 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
427
428 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
429 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v16i32);
430
431 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
432 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v16i32);
433 }
434
435 for (MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
436 setOperationAction(Op: ISD::BUILD_VECTOR, VT: Vec64, Action: Promote);
437 AddPromotedToType(Opc: ISD::BUILD_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
438
439 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: Vec64, Action: Promote);
440 AddPromotedToType(Opc: ISD::EXTRACT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
441
442 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec64, Action: Promote);
443 AddPromotedToType(Opc: ISD::INSERT_VECTOR_ELT, OrigVT: Vec64, DestVT: MVT::v32i32);
444
445 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: Vec64, Action: Promote);
446 AddPromotedToType(Opc: ISD::SCALAR_TO_VECTOR, OrigVT: Vec64, DestVT: MVT::v32i32);
447 }
448
449 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
450 VTs: {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
451 MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
452 Action: Custom);
453
454 if (Subtarget->hasPkMovB32()) {
455 // TODO: 16-bit element vectors should be legal with even aligned elements.
456 // TODO: Can be legal with wider source types than the result with
457 // subregister extracts.
458 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: {MVT::v2i32, MVT::v2f32}, Action: Legal);
459 }
460
461 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT: MVT::v2i32, Action: Legal);
462 // Prevent SELECT v2i32 from being implemented with the above bitwise ops and
463 // instead lower to cndmask in SITargetLowering::LowerSELECT().
464 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i32, Action: Custom);
465 // Enable MatchRotate to produce ISD::ROTR, which is later transformed to
466 // alignbit.
467 setOperationAction(Op: ISD::ROTR, VT: MVT::v2i32, Action: Custom);
468
469 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
470 Action: Custom);
471
472 // Avoid stack access for these.
473 // TODO: Generalize to more vector types.
474 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
475 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
476 MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
477 Action: Custom);
478
479 // Deal with vec3 vector operations when widened to vec4.
480 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
481 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Action: Custom);
482
483 // Deal with vec5/6/7 vector operations when widened to vec8.
484 setOperationAction(Ops: ISD::INSERT_SUBVECTOR,
485 VTs: {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
486 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
487 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
488 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
489 Action: Custom);
490
491 // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
492 // and output demarshalling
493 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP, VTs: {MVT::i32, MVT::i64}, Action: Custom);
494
495 // We can't return success/failure, only the old value,
496 // let LLVM add the comparison
497 setOperationAction(Ops: ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VTs: {MVT::i32, MVT::i64},
498 Action: Expand);
499
500 setOperationAction(Ops: ISD::ADDRSPACECAST, VTs: {MVT::i32, MVT::i64}, Action: Custom);
501
502 setOperationAction(Ops: ISD::BITREVERSE, VTs: {MVT::i32, MVT::i64}, Action: Legal);
503
504 // FIXME: This should be narrowed to i32, but that only happens if i64 is
505 // illegal.
506 // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
507 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i64, MVT::i32}, Action: Legal);
508
509 // On SI this is s_memtime and s_memrealtime on VI.
510 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: Legal);
511
512 if (Subtarget->hasSMemRealTime() ||
513 Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
514 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64, Action: Legal);
515 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Custom);
516
517 if (Subtarget->has16BitInsts()) {
518 setOperationAction(Ops: {ISD::FPOW, ISD::FPOWI}, VT: MVT::f16, Action: Promote);
519 setOperationAction(Ops: {ISD::FLOG, ISD::FEXP, ISD::FLOG10}, VT: MVT::f16, Action: Custom);
520 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Legal);
521 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Legal);
522 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f16, Action: Legal);
523 } else {
524 setOperationAction(Op: ISD::FSQRT, VT: MVT::f16, Action: Custom);
525 }
526
527 if (Subtarget->hasMadMacF32Insts())
528 setOperationAction(Op: ISD::FMAD, VT: MVT::f32, Action: Legal);
529
530 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
531 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
532 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
533
534 // We only really have 32-bit BFE instructions (and 16-bit on VI).
535 //
536 // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
537 // effort to match them now. We want this to be false for i64 cases when the
538 // extraction isn't restricted to the upper or lower half. Ideally we would
539 // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
540 // span the midpoint are probably relatively rare, so don't worry about them
541 // for now.
542 setHasExtractBitsInsn(true);
543
544 // Clamp modifier on add/sub
545 if (Subtarget->hasIntClamp())
546 setOperationAction(Ops: {ISD::UADDSAT, ISD::USUBSAT}, VT: MVT::i32, Action: Legal);
547
548 if (Subtarget->hasAddNoCarryInsts())
549 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT}, VTs: {MVT::i16, MVT::i32},
550 Action: Legal);
551
552 setOperationAction(
553 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
554 VTs: {MVT::f32, MVT::f64}, Action: Custom);
555
556 // These are really only legal for ieee_mode functions. We should be avoiding
557 // them for functions that don't have ieee_mode enabled, so just say they are
558 // legal.
559 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
560 VTs: {MVT::f32, MVT::f64}, Action: Legal);
561
562 if (Subtarget->haveRoundOpsF64())
563 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, VT: MVT::f64,
564 Action: Legal);
565 else
566 setOperationAction(Ops: {ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
567 VT: MVT::f64, Action: Custom);
568
569 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
570 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VTs: {MVT::f32, MVT::f64},
571 Action: Legal);
572 setOperationAction(Ops: ISD::FFREXP, VTs: {MVT::f32, MVT::f64}, Action: Custom);
573
574 setOperationAction(Ops: {ISD::FSIN, ISD::FCOS, ISD::FDIV}, VT: MVT::f32, Action: Custom);
575 setOperationAction(Op: ISD::FDIV, VT: MVT::f64, Action: Custom);
576
577 setOperationAction(Ops: ISD::BF16_TO_FP, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
578 setOperationAction(Ops: ISD::FP_TO_BF16, VTs: {MVT::i16, MVT::f32, MVT::f64}, Action: Expand);
579
580 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i32,
581 Action: Custom);
582 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i16,
583 Action: Custom);
584 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT: MVT::i1,
585 Action: Custom);
586
587 // Custom lower these because we can't specify a rule based on an illegal
588 // source bf16.
589 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f32, Action: Custom);
590 setOperationAction(Ops: {ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, VT: MVT::f64, Action: Custom);
591
592 if (Subtarget->has16BitInsts()) {
593 setOperationAction(Ops: {ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
594 ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
595 VT: MVT::i16, Action: Legal);
596
597 AddPromotedToType(Opc: ISD::SIGN_EXTEND, OrigVT: MVT::i16, DestVT: MVT::i32);
598
599 setOperationAction(Ops: {ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
600 VT: MVT::i16, Action: Expand);
601
602 setOperationAction(Ops: {ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
603 ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
604 ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
605 ISD::CTPOP},
606 VT: MVT::i16, Action: Promote);
607
608 setOperationAction(Op: ISD::LOAD, VT: MVT::i16, Action: Custom);
609
610 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
611
612 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::i16, Action: Promote);
613 AddPromotedToType(Opc: ISD::FP16_TO_FP, OrigVT: MVT::i16, DestVT: MVT::i32);
614 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::i16, Action: Promote);
615 AddPromotedToType(Opc: ISD::FP_TO_FP16, OrigVT: MVT::i16, DestVT: MVT::i32);
616
617 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i16, Action: Custom);
618 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: MVT::i32, Action: Custom);
619 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i16, Action: Custom);
620 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i1, Action: Custom);
621
622 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: MVT::i32, Action: Custom);
623
624 // F16 - Constant Actions.
625 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Legal);
626 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Legal);
627
628 // F16 - Load/Store Actions.
629 setOperationAction(Op: ISD::LOAD, VT: MVT::f16, Action: Promote);
630 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
631 setOperationAction(Op: ISD::STORE, VT: MVT::f16, Action: Promote);
632 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
633
634 // BF16 - Load/Store Actions.
635 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Promote);
636 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
637 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Promote);
638 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
639
640 // F16 - VOP1 Actions.
641 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
642 ISD::FSIN, ISD::FROUND},
643 VT: MVT::f16, Action: Custom);
644
645 // BF16 - VOP1 Actions.
646 if (Subtarget->hasBF16TransInsts())
647 setOperationAction(Ops: {ISD::FCOS, ISD::FSIN, ISD::FDIV}, VT: MVT::bf16, Action: Custom);
648
649 // F16 - VOP2 Actions.
650 setOperationAction(Ops: {ISD::BR_CC, ISD::SELECT_CC}, VTs: {MVT::f16, MVT::bf16},
651 Action: Expand);
652 setOperationAction(Ops: {ISD::FLDEXP, ISD::STRICT_FLDEXP}, VT: MVT::f16, Action: Custom);
653 setOperationAction(Op: ISD::FFREXP, VT: MVT::f16, Action: Custom);
654 setOperationAction(Op: ISD::FDIV, VT: MVT::f16, Action: Custom);
655
656 // F16 - VOP3 Actions.
657 setOperationAction(Op: ISD::FMA, VT: MVT::f16, Action: Legal);
658 if (STI.hasMadF16())
659 setOperationAction(Op: ISD::FMAD, VT: MVT::f16, Action: Legal);
660
661 for (MVT VT :
662 {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
663 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
664 MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
665 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
666 switch (Op) {
667 case ISD::LOAD:
668 case ISD::STORE:
669 case ISD::BUILD_VECTOR:
670 case ISD::BITCAST:
671 case ISD::UNDEF:
672 case ISD::EXTRACT_VECTOR_ELT:
673 case ISD::INSERT_VECTOR_ELT:
674 case ISD::INSERT_SUBVECTOR:
675 case ISD::SCALAR_TO_VECTOR:
676 case ISD::IS_FPCLASS:
677 break;
678 case ISD::EXTRACT_SUBVECTOR:
679 case ISD::CONCAT_VECTORS:
680 case ISD::FSIN:
681 case ISD::FCOS:
682 setOperationAction(Op, VT, Action: Custom);
683 break;
684 default:
685 setOperationAction(Op, VT, Action: Expand);
686 break;
687 }
688 }
689 }
690
691 // v_perm_b32 can handle either of these.
692 setOperationAction(Ops: ISD::BSWAP, VTs: {MVT::i16, MVT::v2i16}, Action: Legal);
693 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i16, Action: Custom);
694
695 // Legalize vector types for sat conversions to select v_cvt_pk_[iu]16_f32.
696 if (Subtarget->hasVCvtPkIU16F32())
697 setOperationAction(
698 Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT},
699 VTs: {MVT::v2i16, MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16},
700 Action: Custom);
701
702 // XXX - Do these do anything? Vector constants turn into build_vector.
703 setOperationAction(Ops: ISD::Constant, VTs: {MVT::v2i16, MVT::v2f16}, Action: Legal);
704
705 setOperationAction(Ops: ISD::UNDEF, VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
706 Action: Legal);
707
708 setOperationAction(Op: ISD::STORE, VT: MVT::v2i16, Action: Promote);
709 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
710 setOperationAction(Op: ISD::STORE, VT: MVT::v2f16, Action: Promote);
711 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
712
713 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i16, Action: Promote);
714 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
715 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f16, Action: Promote);
716 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
717
718 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2i16, Action: Promote);
719 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2i16, DestVT: MVT::i32);
720 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2f16, Action: Promote);
721 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2f16, DestVT: MVT::i32);
722
723 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2i16, Action: Promote);
724 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2i16, DestVT: MVT::i32);
725 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2f16, Action: Promote);
726 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2f16, DestVT: MVT::i32);
727
728 setOperationAction(Op: ISD::AND, VT: MVT::v2i16, Action: Promote);
729 AddPromotedToType(Opc: ISD::AND, OrigVT: MVT::v2i16, DestVT: MVT::i32);
730 setOperationAction(Op: ISD::OR, VT: MVT::v2i16, Action: Promote);
731 AddPromotedToType(Opc: ISD::OR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
732 setOperationAction(Op: ISD::XOR, VT: MVT::v2i16, Action: Promote);
733 AddPromotedToType(Opc: ISD::XOR, OrigVT: MVT::v2i16, DestVT: MVT::i32);
734
735 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i16, Action: Promote);
736 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
737 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f16, Action: Promote);
738 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
739 setOperationAction(Op: ISD::LOAD, VT: MVT::v4bf16, Action: Promote);
740 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
741
742 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v4i16, Action: Promote);
743 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v4i16, DestVT: MVT::i64);
744 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v4f16, Action: Promote);
745 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v4f16, DestVT: MVT::i64);
746
747 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v4i16, Action: Promote);
748 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v4i16, DestVT: MVT::i64);
749 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v4f16, Action: Promote);
750 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v4f16, DestVT: MVT::i64);
751
752 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
753 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
754 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
755 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
756 setOperationAction(Op: ISD::STORE, VT: MVT::v4bf16, Action: Promote);
757 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4bf16, DestVT: MVT::v2i32);
758
759 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i16, Action: Promote);
760 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
761 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f16, Action: Promote);
762 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
763 setOperationAction(Op: ISD::LOAD, VT: MVT::v8bf16, Action: Promote);
764 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
765
766 setOperationAction(Op: ISD::STORE, VT: MVT::v4i16, Action: Promote);
767 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i16, DestVT: MVT::v2i32);
768 setOperationAction(Op: ISD::STORE, VT: MVT::v4f16, Action: Promote);
769 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f16, DestVT: MVT::v2i32);
770
771 setOperationAction(Op: ISD::STORE, VT: MVT::v8i16, Action: Promote);
772 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i16, DestVT: MVT::v4i32);
773 setOperationAction(Op: ISD::STORE, VT: MVT::v8f16, Action: Promote);
774 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f16, DestVT: MVT::v4i32);
775 setOperationAction(Op: ISD::STORE, VT: MVT::v8bf16, Action: Promote);
776 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8bf16, DestVT: MVT::v4i32);
777
778 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i16, Action: Promote);
779 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
780 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f16, Action: Promote);
781 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
782 setOperationAction(Op: ISD::LOAD, VT: MVT::v16bf16, Action: Promote);
783 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
784
785 setOperationAction(Op: ISD::STORE, VT: MVT::v16i16, Action: Promote);
786 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i16, DestVT: MVT::v8i32);
787 setOperationAction(Op: ISD::STORE, VT: MVT::v16f16, Action: Promote);
788 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f16, DestVT: MVT::v8i32);
789 setOperationAction(Op: ISD::STORE, VT: MVT::v16bf16, Action: Promote);
790 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16bf16, DestVT: MVT::v8i32);
791
792 setOperationAction(Op: ISD::LOAD, VT: MVT::v32i16, Action: Promote);
793 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
794 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f16, Action: Promote);
795 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
796 setOperationAction(Op: ISD::LOAD, VT: MVT::v32bf16, Action: Promote);
797 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
798
799 setOperationAction(Op: ISD::STORE, VT: MVT::v32i16, Action: Promote);
800 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32i16, DestVT: MVT::v16i32);
801 setOperationAction(Op: ISD::STORE, VT: MVT::v32f16, Action: Promote);
802 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f16, DestVT: MVT::v16i32);
803 setOperationAction(Op: ISD::STORE, VT: MVT::v32bf16, Action: Promote);
804 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32bf16, DestVT: MVT::v16i32);
805
806 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
807 VT: MVT::v2i32, Action: Expand);
808 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Expand);
809
810 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
811 VT: MVT::v4i32, Action: Expand);
812
813 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
814 VT: MVT::v8i32, Action: Expand);
815
816 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
817 Action: Subtarget->hasVOP3PInsts() ? Legal : Custom);
818
819 setOperationAction(Ops: ISD::FNEG, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
820 // This isn't really legal, but this avoids the legalizer unrolling it (and
821 // allows matching fneg (fabs x) patterns)
822 setOperationAction(Ops: ISD::FABS, VTs: {MVT::v2f16, MVT::v2bf16}, Action: Legal);
823
824 // Can do this in one BFI plus a constant materialize.
825 setOperationAction(Ops: ISD::FCOPYSIGN,
826 VTs: {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
827 MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
828 MVT::v32f16, MVT::v32bf16},
829 Action: Custom);
830
831 setOperationAction(
832 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
833 VT: MVT::f16, Action: Custom);
834 setOperationAction(Ops: {ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, VT: MVT::f16, Action: Legal);
835
836 setOperationAction(Ops: {ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINIMUMNUM,
837 ISD::FMAXIMUMNUM},
838 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
839 Action: Custom);
840
841 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM},
842 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
843 Action: Expand);
844
845 for (MVT Vec16 :
846 {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
847 MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
848 setOperationAction(
849 Ops: {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
850 VT: Vec16, Action: Custom);
851 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: Vec16, Action: Expand);
852 }
853 }
854
855 if (Subtarget->hasVOP3PInsts()) {
856 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
857 ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
858 ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
859 VT: MVT::v2i16, Action: Legal);
860
861 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
862 ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
863 ISD::FCANONICALIZE},
864 VT: MVT::v2f16, Action: Legal);
865
866 setOperationAction(Ops: ISD::EXTRACT_VECTOR_ELT,
867 VTs: {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Action: Custom);
868
869 setOperationAction(Ops: ISD::VECTOR_SHUFFLE,
870 VTs: {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
871 MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
872 MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
873 Action: Custom);
874
875 for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
876 // Split vector operations.
877 setOperationAction(Ops: {ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
878 ISD::MUL, ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN,
879 ISD::UMAX, ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
880 ISD::SSUBSAT},
881 VT, Action: Custom);
882
883 for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
884 // Split vector operations.
885 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
886 ISD::FCANONICALIZE},
887 VT, Action: Custom);
888
889 setOperationAction(
890 Ops: {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
891 VTs: {MVT::v2f16, MVT::v4f16}, Action: Custom);
892
893 setOperationAction(Op: ISD::FEXP, VT: MVT::v2f16, Action: Custom);
894 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
895 Action: Custom);
896
897 if (Subtarget->hasBF16PackedInsts()) {
898 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMAXNUM, ISD::FMINNUM,
899 ISD::FMA, ISD::FNEG, ISD::FABS, ISD::FCANONICALIZE},
900 VT: MVT::v2bf16, Action: Legal);
901
902 for (MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
903 // Split vector operations.
904 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE,
905 ISD::FNEG, ISD::FABS},
906 VT, Action: Custom);
907 }
908
909 if (Subtarget->hasPackedFP32Ops()) {
910 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
911 VT: MVT::v2f32, Action: Legal);
912 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
913 VTs: {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
914 Action: Custom);
915 }
916 if (Subtarget->hasPackedFP64Ops()) {
917 setOperationAction(Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG,
918 ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
919 ISD::FCANONICALIZE, ISD::BUILD_VECTOR},
920 VT: MVT::v2f64, Action: Legal);
921 setOperationAction(
922 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
923 VT: MVT::v2f64, Action: Custom);
924 setOperationAction(
925 Ops: {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FMINNUM_IEEE,
926 ISD::FMAXNUM_IEEE, ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
927 ISD::FMAXIMUMNUM, ISD::FCANONICALIZE},
928 VTs: {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Action: Custom);
929 }
930
931 if (Subtarget->hasPackedU64Ops()) {
932 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL, ISD::BUILD_VECTOR},
933 VT: MVT::v2i64, Action: Legal);
934 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL},
935 VTs: {MVT::v4i64, MVT::v8i64, MVT::v16i64, MVT::v32i64},
936 Action: Custom);
937 }
938 }
939
940 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v4f16, Action: Custom);
941
942 if (Subtarget->has16BitInsts()) {
943 setOperationAction(Op: ISD::SELECT, VT: MVT::v2i16, Action: Promote);
944 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2i16, DestVT: MVT::i32);
945 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f16, Action: Promote);
946 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f16, DestVT: MVT::i32);
947 } else {
948 // Legalization hack.
949 setOperationAction(Ops: ISD::SELECT, VTs: {MVT::v2i16, MVT::v2f16}, Action: Custom);
950
951 setOperationAction(Ops: {ISD::FNEG, ISD::FABS}, VT: MVT::v2f16, Action: Custom);
952 }
953
954 setOperationAction(Ops: ISD::SELECT,
955 VTs: {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
956 MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
957 MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
958 MVT::v32f16, MVT::v32bf16},
959 Action: Custom);
960
961 setOperationAction(Ops: {ISD::SMULO, ISD::UMULO}, VT: MVT::i64, Action: Custom);
962
963 if (Subtarget->hasVMulU64Inst())
964 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Legal);
965 else if (Subtarget->hasScalarSMulU64())
966 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
967
968 if (Subtarget->hasMad64_32())
969 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: MVT::i32, Action: Custom);
970
971 if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
972 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
973
974 if (Subtarget->hasIEEEMinimumMaximumInsts()) {
975 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM},
976 VTs: {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Action: Legal);
977 } else {
978 // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum
979 if (Subtarget->hasMinimum3Maximum3F32())
980 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
981
982 if (Subtarget->hasMinimum3Maximum3PKF16()) {
983 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::v2f16, Action: Legal);
984
985 // If only the vector form is available, we need to widen to a vector.
986 if (!Subtarget->hasMinimum3Maximum3F16())
987 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f16, Action: Custom);
988 }
989 }
990
991 if (Subtarget->hasVOP3PInsts()) {
992 // We want to break these into v2f16 pieces, not scalarize.
993 setOperationAction(Ops: {ISD::FMINIMUM, ISD::FMAXIMUM},
994 VTs: {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
995 Action: Custom);
996 }
997
998 if (Subtarget->hasMinMaxI64Insts())
999 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i64,
1000 Action: Legal);
1001
1002 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN,
1003 VTs: {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
1004 MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
1005 MVT::i8},
1006 Action: Custom);
1007
1008 setOperationAction(Ops: ISD::INTRINSIC_W_CHAIN,
1009 VTs: {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
1010 MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
1011 MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
1012 MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1013 Action: Custom);
1014
1015 setOperationAction(Ops: ISD::INTRINSIC_VOID,
1016 VTs: {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
1017 MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
1018 MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
1019 MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
1020 Action: Custom);
1021
1022 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Custom);
1023 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
1024 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
1025 setOperationAction(Op: ISD::GET_FPENV, VT: MVT::i64, Action: Custom);
1026 setOperationAction(Op: ISD::SET_FPENV, VT: MVT::i64, Action: Custom);
1027
1028 // TODO: Could move this to custom lowering, could benefit from combines on
1029 // extract of relevant bits.
1030 setOperationAction(Op: ISD::GET_FPMODE, VT: MVT::i32, Action: Legal);
1031
1032 setOperationAction(Op: ISD::MUL, VT: MVT::i1, Action: Promote);
1033
1034 if (Subtarget->hasBF16ConversionInsts()) {
1035 setOperationAction(Ops: {ISD::FP_ROUND, ISD::STRICT_FP_ROUND},
1036 VTs: {MVT::bf16, MVT::v2bf16}, Action: Custom);
1037 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2bf16, Action: Legal);
1038 }
1039
1040 if (Subtarget->hasBF16TransInsts()) {
1041 setOperationAction(Ops: {ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, VT: MVT::bf16, Action: Legal);
1042 }
1043
1044 if (Subtarget->hasCvtPkF16F32Inst()) {
1045 setOperationAction(Ops: ISD::FP_ROUND,
1046 VTs: {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
1047 Action: Custom);
1048 }
1049
1050 setTargetDAGCombine({ISD::ADD,
1051 ISD::PTRADD,
1052 ISD::SUB,
1053 ISD::MUL,
1054 ISD::FADD,
1055 ISD::FSUB,
1056 ISD::FDIV,
1057 ISD::FMUL,
1058 ISD::FMINNUM,
1059 ISD::FMAXNUM,
1060 ISD::FMINNUM_IEEE,
1061 ISD::FMAXNUM_IEEE,
1062 ISD::FMINIMUM,
1063 ISD::FMAXIMUM,
1064 ISD::FMINIMUMNUM,
1065 ISD::FMAXIMUMNUM,
1066 ISD::FMA,
1067 ISD::ABS,
1068 ISD::SMIN,
1069 ISD::SMAX,
1070 ISD::UMIN,
1071 ISD::UMAX,
1072 ISD::SETCC,
1073 ISD::SELECT,
1074 ISD::SMIN,
1075 ISD::SMAX,
1076 ISD::UMIN,
1077 ISD::UMAX,
1078 ISD::USUBSAT,
1079 ISD::AND,
1080 ISD::OR,
1081 ISD::XOR,
1082 ISD::SHL,
1083 ISD::SRL,
1084 ISD::SRA,
1085 ISD::FSHR,
1086 ISD::SINT_TO_FP,
1087 ISD::UINT_TO_FP,
1088 ISD::FCANONICALIZE,
1089 ISD::SCALAR_TO_VECTOR,
1090 ISD::ZERO_EXTEND,
1091 ISD::SIGN_EXTEND_INREG,
1092 ISD::ANY_EXTEND,
1093 ISD::EXTRACT_VECTOR_ELT,
1094 ISD::INSERT_VECTOR_ELT,
1095 ISD::FCOPYSIGN});
1096
1097 if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
1098 setTargetDAGCombine(ISD::FP_ROUND);
1099
1100 // All memory operations. Some folding on the pointer operand is done to help
1101 // matching the constant offsets in the addressing modes.
1102 setTargetDAGCombine({ISD::LOAD,
1103 ISD::STORE,
1104 ISD::ATOMIC_LOAD,
1105 ISD::ATOMIC_STORE,
1106 ISD::ATOMIC_CMP_SWAP,
1107 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
1108 ISD::ATOMIC_SWAP,
1109 ISD::ATOMIC_LOAD_ADD,
1110 ISD::ATOMIC_LOAD_SUB,
1111 ISD::ATOMIC_LOAD_AND,
1112 ISD::ATOMIC_LOAD_OR,
1113 ISD::ATOMIC_LOAD_XOR,
1114 ISD::ATOMIC_LOAD_NAND,
1115 ISD::ATOMIC_LOAD_MIN,
1116 ISD::ATOMIC_LOAD_MAX,
1117 ISD::ATOMIC_LOAD_UMIN,
1118 ISD::ATOMIC_LOAD_UMAX,
1119 ISD::ATOMIC_LOAD_FADD,
1120 ISD::ATOMIC_LOAD_FMIN,
1121 ISD::ATOMIC_LOAD_FMAX,
1122 ISD::ATOMIC_LOAD_UINC_WRAP,
1123 ISD::ATOMIC_LOAD_UDEC_WRAP,
1124 ISD::ATOMIC_LOAD_USUB_COND,
1125 ISD::ATOMIC_LOAD_USUB_SAT,
1126 ISD::INTRINSIC_VOID,
1127 ISD::INTRINSIC_W_CHAIN});
1128
1129 // FIXME: In other contexts we pretend this is a per-function property.
1130 setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
1131
1132 setSchedulingPreference(Sched::RegPressure);
1133}
1134
1135const GCNSubtarget *SITargetLowering::getSubtarget() const { return Subtarget; }
1136
1137ArrayRef<MCPhysReg> SITargetLowering::getRoundingControlRegisters() const {
1138 static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
1139 return RCRegs;
1140}
1141
1142//===----------------------------------------------------------------------===//
1143// TargetLowering queries
1144//===----------------------------------------------------------------------===//
1145
1146// v_mad_mix* support a conversion from f16 to f32.
1147//
1148// There is only one special case when denormals are enabled we don't currently,
1149// where this is OK to use.
1150bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
1151 EVT DestVT, EVT SrcVT) const {
1152 return DestVT.getScalarType() == MVT::f32 &&
1153 ((((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
1154 (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
1155 SrcVT.getScalarType() == MVT::f16) ||
1156 (Opcode == ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
1157 SrcVT.getScalarType() == MVT::bf16)) &&
1158 // TODO: This probably only requires no input flushing?
1159 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
1160}
1161
1162bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
1163 LLT DestTy, LLT SrcTy) const {
1164 return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
1165 (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
1166 DestTy.getScalarSizeInBits() == 32 &&
1167 SrcTy.getScalarSizeInBits() == 16 &&
1168 // TODO: This probably only requires no input flushing?
1169 denormalModeIsFlushAllF32(MF: *MI.getMF());
1170}
1171
1172bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
1173 // SI has some legal vector types, but no legal vector operations. Say no
1174 // shuffles are legal in order to prefer scalarizing some vector operations.
1175 return false;
1176}
1177
1178MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1179 CallingConv::ID CC,
1180 EVT VT) const {
1181 if (CC == CallingConv::AMDGPU_KERNEL)
1182 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1183
1184 if (VT.isVector()) {
1185 EVT ScalarVT = VT.getScalarType();
1186 unsigned Size = ScalarVT.getSizeInBits();
1187 if (Size == 16) {
1188 return Subtarget->has16BitInsts()
1189 ? MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), NumElements: 2)
1190 : MVT::i32;
1191 }
1192
1193 if (Size < 16)
1194 return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
1195 return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
1196 }
1197
1198 if (!Subtarget->has16BitInsts() && VT.getSizeInBits() == 16)
1199 return MVT::i32;
1200
1201 if (VT.getSizeInBits() > 32)
1202 return MVT::i32;
1203
1204 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1205}
1206
1207unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1208 CallingConv::ID CC,
1209 EVT VT) const {
1210 if (CC == CallingConv::AMDGPU_KERNEL)
1211 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1212
1213 if (VT.isVector()) {
1214 unsigned NumElts = VT.getVectorNumElements();
1215 EVT ScalarVT = VT.getScalarType();
1216 unsigned Size = ScalarVT.getSizeInBits();
1217
1218 // FIXME: Should probably promote 8-bit vectors to i16.
1219 if (Size == 16)
1220 return (NumElts + 1) / 2;
1221
1222 if (Size <= 32)
1223 return NumElts;
1224
1225 if (Size > 32)
1226 return NumElts * ((Size + 31) / 32);
1227 } else if (VT.getSizeInBits() > 32)
1228 return (VT.getSizeInBits() + 31) / 32;
1229
1230 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1231}
1232
1233unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
1234 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1235 unsigned &NumIntermediates, MVT &RegisterVT) const {
1236 if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
1237 unsigned NumElts = VT.getVectorNumElements();
1238 EVT ScalarVT = VT.getScalarType();
1239 unsigned Size = ScalarVT.getSizeInBits();
1240 // FIXME: We should fix the ABI to be the same on targets without 16-bit
1241 // support, but unless we can properly handle 3-vectors, it will be still be
1242 // inconsistent.
1243 if (Size == 16) {
1244 MVT SimpleIntermediateVT =
1245 MVT::getVectorVT(VT: ScalarVT.getSimpleVT(), EC: ElementCount::getFixed(MinVal: 2));
1246 IntermediateVT = SimpleIntermediateVT;
1247 RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
1248 NumIntermediates = (NumElts + 1) / 2;
1249 return (NumElts + 1) / 2;
1250 }
1251
1252 if (Size == 32) {
1253 RegisterVT = ScalarVT.getSimpleVT();
1254 IntermediateVT = RegisterVT;
1255 NumIntermediates = NumElts;
1256 return NumIntermediates;
1257 }
1258
1259 if (Size < 16 && Subtarget->has16BitInsts()) {
1260 // FIXME: Should probably form v2i16 pieces
1261 RegisterVT = MVT::i16;
1262 IntermediateVT = ScalarVT;
1263 NumIntermediates = NumElts;
1264 return NumIntermediates;
1265 }
1266
1267 if (Size != 16 && Size <= 32) {
1268 RegisterVT = MVT::i32;
1269 IntermediateVT = ScalarVT;
1270 NumIntermediates = NumElts;
1271 return NumIntermediates;
1272 }
1273
1274 if (Size > 32) {
1275 RegisterVT = MVT::i32;
1276 IntermediateVT = RegisterVT;
1277 NumIntermediates = NumElts * ((Size + 31) / 32);
1278 return NumIntermediates;
1279 }
1280 }
1281
1282 return TargetLowering::getVectorTypeBreakdownForCallingConv(
1283 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
1284}
1285
1286static EVT memVTFromLoadIntrData(const SITargetLowering &TLI,
1287 const DataLayout &DL, Type *Ty,
1288 unsigned MaxNumLanes) {
1289 assert(MaxNumLanes != 0);
1290
1291 LLVMContext &Ctx = Ty->getContext();
1292 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
1293 unsigned NumElts = std::min(a: MaxNumLanes, b: VT->getNumElements());
1294 return EVT::getVectorVT(Context&: Ctx, VT: TLI.getValueType(DL, Ty: VT->getElementType()),
1295 NumElements: NumElts);
1296 }
1297
1298 return TLI.getValueType(DL, Ty);
1299}
1300
1301// Peek through TFE struct returns to only use the data size.
1302static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI,
1303 const DataLayout &DL, Type *Ty,
1304 unsigned MaxNumLanes) {
1305 auto *ST = dyn_cast<StructType>(Val: Ty);
1306 if (!ST)
1307 return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
1308
1309 // TFE intrinsics return an aggregate type.
1310 assert(ST->getNumContainedTypes() == 2 &&
1311 ST->getContainedType(1)->isIntegerTy(32));
1312 return memVTFromLoadIntrData(TLI, DL, Ty: ST->getContainedType(i: 0), MaxNumLanes);
1313}
1314
1315/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its
1316/// in-memory representation. This return value is a custom type because there
1317/// is no MVT::i160 and adding one breaks integer promotion logic. While this
1318/// could cause issues during codegen, these address space 7 pointers will be
1319/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer
1320/// in order to allow pre-codegen passes that query TargetTransformInfo, often
1321/// for cost modeling, to work. (This also sets us up decently for doing the
1322/// buffer lowering in GlobalISel if SelectionDAG ever goes away.)
1323MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
1324 if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
1325 return MVT::amdgpuBufferFatPointer;
1326 if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1327 DL.getPointerSizeInBits(AS) == 192)
1328 return MVT::amdgpuBufferStridedPointer;
1329 return AMDGPUTargetLowering::getPointerTy(DL, AS);
1330}
1331/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
1332/// v8i32 when padding is added.
1333/// The in-memory representation of a p9 is {p8, i32, i32}, which is
1334/// also v8i32 with padding.
1335MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
1336 if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
1337 DL.getPointerSizeInBits(AS) == 160) ||
1338 (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
1339 DL.getPointerSizeInBits(AS) == 192))
1340 return MVT::v8i32;
1341 return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
1342}
1343
1344static unsigned getIntrMemWidth(unsigned IntrID) {
1345 switch (IntrID) {
1346 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1347 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1348 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1349 return 8;
1350 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1351 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1352 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1353 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1354 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1355 case Intrinsic::amdgcn_flat_load_monitor_b32:
1356 case Intrinsic::amdgcn_global_load_monitor_b32:
1357 return 32;
1358 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1359 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1360 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1361 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1362 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1363 case Intrinsic::amdgcn_flat_load_monitor_b64:
1364 case Intrinsic::amdgcn_global_load_monitor_b64:
1365 return 64;
1366 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1367 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1368 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1369 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
1370 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
1371 case Intrinsic::amdgcn_flat_load_monitor_b128:
1372 case Intrinsic::amdgcn_global_load_monitor_b128:
1373 return 128;
1374 default:
1375 llvm_unreachable("Unknown width");
1376 }
1377}
1378
1379static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI,
1380 unsigned ArgIdx) {
1381 Value *OrderingArg = CI.getArgOperand(i: ArgIdx);
1382 unsigned Ord = cast<ConstantInt>(Val: OrderingArg)->getZExtValue();
1383 switch (AtomicOrderingCABI(Ord)) {
1384 case AtomicOrderingCABI::acquire:
1385 return AtomicOrdering::Acquire;
1386 break;
1387 case AtomicOrderingCABI::release:
1388 return AtomicOrdering::Release;
1389 break;
1390 case AtomicOrderingCABI::seq_cst:
1391 return AtomicOrdering::SequentiallyConsistent;
1392 break;
1393 default:
1394 return AtomicOrdering::Monotonic;
1395 }
1396}
1397
1398static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
1399 MDNode *ScopeMD = cast<MDNode>(
1400 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ArgIdx))->getMetadata());
1401 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1402 return CI.getContext().getOrInsertSyncScopeID(SSN: Scope);
1403}
1404
1405void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
1406 const CallBase &CI,
1407 MachineFunction &MF,
1408 unsigned IntrID) const {
1409 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
1410 if (CI.hasMetadata(KindID: LLVMContext::MD_invariant_load))
1411 Flags |= MachineMemOperand::MOInvariant;
1412 if (CI.hasMetadata(KindID: LLVMContext::MD_nontemporal))
1413 Flags |= MachineMemOperand::MONonTemporal;
1414 Flags |= getTargetMMOFlags(I: CI);
1415
1416 if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
1417 AMDGPU::lookupRsrcIntrinsic(Intr: IntrID)) {
1418 AttributeSet Attr =
1419 Intrinsic::getFnAttributes(C&: CI.getContext(), id: (Intrinsic::ID)IntrID);
1420 MemoryEffects ME = Attr.getMemoryEffects();
1421 if (ME.doesNotAccessMemory())
1422 return;
1423
1424 bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
1425 if (!IsSPrefetch) {
1426 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1427 if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
1428 Flags |= MachineMemOperand::MOVolatile;
1429 }
1430 Flags |= MachineMemOperand::MODereferenceable;
1431
1432 IntrinsicInfo Info;
1433 // TODO: Should images get their own address space?
1434 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1435
1436 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
1437 if (RsrcIntr->IsImage) {
1438 const AMDGPU::ImageDimIntrinsicInfo *Intr =
1439 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID);
1440 BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Intr->BaseOpcode);
1441 Info.align.reset();
1442 }
1443
1444 Value *RsrcArg = CI.getArgOperand(i: RsrcIntr->RsrcArg);
1445 if (auto *RsrcPtrTy = dyn_cast<PointerType>(Val: RsrcArg->getType())) {
1446 if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
1447 // We conservatively set the memory operand of a buffer intrinsic to the
1448 // base resource pointer, so that we can access alias information about
1449 // those pointers. Cases like "this points at the same value
1450 // but with a different offset" are handled in
1451 // areMemAccessesTriviallyDisjoint.
1452 Info.ptrVal = RsrcArg;
1453 }
1454
1455 if (ME.onlyReadsMemory()) {
1456 if (RsrcIntr->IsImage) {
1457 unsigned MaxNumLanes = 4;
1458
1459 if (!BaseOpcode->Gather4) {
1460 // If this isn't a gather, we may have excess loaded elements in the
1461 // IR type. Check the dmask for the real number of elements loaded.
1462 unsigned DMask =
1463 cast<ConstantInt>(Val: CI.getArgOperand(i: 0))->getZExtValue();
1464 MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1465 }
1466
1467 Info.memVT = memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(),
1468 Ty: CI.getType(), MaxNumLanes);
1469 } else {
1470 Info.memVT =
1471 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1472 MaxNumLanes: std::numeric_limits<unsigned>::max());
1473 }
1474
1475 // FIXME: What does alignment mean for an image?
1476 Info.opc = ISD::INTRINSIC_W_CHAIN;
1477 Info.flags = Flags | MachineMemOperand::MOLoad;
1478 } else if (ME.onlyWritesMemory()) {
1479 Info.opc = ISD::INTRINSIC_VOID;
1480
1481 Type *DataTy = CI.getArgOperand(i: 0)->getType();
1482 if (RsrcIntr->IsImage) {
1483 unsigned DMask = cast<ConstantInt>(Val: CI.getArgOperand(i: 1))->getZExtValue();
1484 unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(Value: DMask);
1485 Info.memVT = memVTFromLoadIntrData(TLI: *this, DL: MF.getDataLayout(), Ty: DataTy,
1486 MaxNumLanes: DMaskLanes);
1487 } else
1488 Info.memVT = getValueType(DL: MF.getDataLayout(), Ty: DataTy);
1489
1490 Info.flags = Flags | MachineMemOperand::MOStore;
1491 } else {
1492 // Atomic, NoReturn Sampler or prefetch
1493 Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
1494 : ISD::INTRINSIC_W_CHAIN;
1495
1496 switch (IntrID) {
1497 default:
1498 Info.flags = Flags | MachineMemOperand::MOLoad;
1499 if (!IsSPrefetch)
1500 Info.flags |= MachineMemOperand::MOStore;
1501
1502 if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
1503 // Fake memory access type for no return sampler intrinsics
1504 Info.memVT = MVT::i32;
1505 } else {
1506 // XXX - Should this be volatile without known ordering?
1507 Info.flags |= MachineMemOperand::MOVolatile;
1508 Info.memVT = MVT::getVT(Ty: CI.getArgOperand(i: 0)->getType());
1509 }
1510 break;
1511 case Intrinsic::amdgcn_raw_buffer_load_lds:
1512 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
1513 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
1514 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
1515 case Intrinsic::amdgcn_struct_buffer_load_lds:
1516 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
1517 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
1518 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
1519 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1520
1521 // Entry 0: Load from buffer.
1522 // Don't set an offset, since the pointer value always represents the
1523 // base of the buffer.
1524 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1525 Info.flags = Flags | MachineMemOperand::MOLoad;
1526 Infos.push_back(Elt: Info);
1527
1528 // Entry 1: Store to LDS.
1529 // Instruction offset is applied, and an additional per-lane offset
1530 // which we simulate using a larger memory type.
1531 Info.memVT = EVT::getIntegerVT(
1532 Context&: CI.getContext(), BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1533 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1534 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 2))
1535 ->getZExtValue();
1536 Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
1537 Info.flags = Flags | MachineMemOperand::MOStore;
1538 Infos.push_back(Elt: Info);
1539 return;
1540 }
1541 case Intrinsic::amdgcn_raw_atomic_buffer_load:
1542 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
1543 case Intrinsic::amdgcn_struct_atomic_buffer_load:
1544 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
1545 Info.memVT =
1546 memVTFromLoadIntrReturn(TLI: *this, DL: MF.getDataLayout(), Ty: CI.getType(),
1547 MaxNumLanes: std::numeric_limits<unsigned>::max());
1548 Info.flags = Flags | MachineMemOperand::MOLoad;
1549 Infos.push_back(Elt: Info);
1550 return;
1551 }
1552 }
1553 }
1554 Infos.push_back(Elt: Info);
1555 return;
1556 }
1557
1558 IntrinsicInfo Info;
1559 switch (IntrID) {
1560 case Intrinsic::amdgcn_ds_ordered_add:
1561 case Intrinsic::amdgcn_ds_ordered_swap: {
1562 Info.opc = ISD::INTRINSIC_W_CHAIN;
1563 Info.memVT = MVT::getVT(Ty: CI.getType());
1564 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1565 Info.align.reset();
1566 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1567
1568 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 4));
1569 if (!Vol->isZero())
1570 Info.flags |= MachineMemOperand::MOVolatile;
1571
1572 Infos.push_back(Elt: Info);
1573 return;
1574 }
1575 case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
1576 case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
1577 Info.opc = ISD::INTRINSIC_W_CHAIN;
1578 Info.memVT = MVT::getVT(Ty: CI.getOperand(i_nocapture: 0)->getType());
1579 Info.ptrVal = nullptr;
1580 Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
1581 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1582 Infos.push_back(Elt: Info);
1583 return;
1584 }
1585 case Intrinsic::amdgcn_ds_append:
1586 case Intrinsic::amdgcn_ds_consume: {
1587 Info.opc = ISD::INTRINSIC_W_CHAIN;
1588 Info.memVT = MVT::getVT(Ty: CI.getType());
1589 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1590 Info.align.reset();
1591 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1592
1593 const ConstantInt *Vol = cast<ConstantInt>(Val: CI.getOperand(i_nocapture: 1));
1594 if (!Vol->isZero())
1595 Info.flags |= MachineMemOperand::MOVolatile;
1596
1597 Infos.push_back(Elt: Info);
1598 return;
1599 }
1600 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1601 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
1602 Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
1603 ? ISD::INTRINSIC_W_CHAIN
1604 : ISD::INTRINSIC_VOID;
1605 Info.memVT = MVT::getVT(Ty: CI.getType());
1606 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1607 Info.memVT = MVT::i64;
1608 Info.size = 8;
1609 Info.align.reset();
1610 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1611 Info.order = AtomicOrdering::Monotonic;
1612 Infos.push_back(Elt: Info);
1613 return;
1614 }
1615 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
1616 case Intrinsic::amdgcn_image_bvh_intersect_ray:
1617 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
1618 Info.opc = ISD::INTRINSIC_W_CHAIN;
1619 Info.memVT =
1620 MVT::getVT(Ty: IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
1621 ? CI.getType()
1622 : cast<StructType>(Val: CI.getType())
1623 ->getElementType(N: 0)); // XXX: what is correct VT?
1624
1625 Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
1626 Info.align.reset();
1627 Info.flags = Flags | MachineMemOperand::MOLoad |
1628 MachineMemOperand::MODereferenceable;
1629 Infos.push_back(Elt: Info);
1630 return;
1631 }
1632 case Intrinsic::amdgcn_global_atomic_fmin_num:
1633 case Intrinsic::amdgcn_global_atomic_fmax_num:
1634 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1635 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1636 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
1637 Info.opc = ISD::INTRINSIC_W_CHAIN;
1638 Info.memVT = MVT::getVT(Ty: CI.getType());
1639 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1640 Info.align.reset();
1641 Info.flags =
1642 Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1643 MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile;
1644 Infos.push_back(Elt: Info);
1645 return;
1646 }
1647 case Intrinsic::amdgcn_cluster_load_b32:
1648 case Intrinsic::amdgcn_cluster_load_b64:
1649 case Intrinsic::amdgcn_cluster_load_b128:
1650 case Intrinsic::amdgcn_ds_load_tr6_b96:
1651 case Intrinsic::amdgcn_ds_load_tr4_b64:
1652 case Intrinsic::amdgcn_ds_load_tr8_b64:
1653 case Intrinsic::amdgcn_ds_load_tr16_b128:
1654 case Intrinsic::amdgcn_global_load_tr6_b96:
1655 case Intrinsic::amdgcn_global_load_tr4_b64:
1656 case Intrinsic::amdgcn_global_load_tr_b64:
1657 case Intrinsic::amdgcn_global_load_tr_b128:
1658 case Intrinsic::amdgcn_ds_read_tr4_b64:
1659 case Intrinsic::amdgcn_ds_read_tr6_b96:
1660 case Intrinsic::amdgcn_ds_read_tr8_b64:
1661 case Intrinsic::amdgcn_ds_read_tr16_b64: {
1662 Info.opc = ISD::INTRINSIC_W_CHAIN;
1663 Info.memVT = MVT::getVT(Ty: CI.getType());
1664 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1665 Info.align.reset();
1666 Info.flags = Flags | MachineMemOperand::MOLoad;
1667 Infos.push_back(Elt: Info);
1668 return;
1669 }
1670 case Intrinsic::amdgcn_flat_load_monitor_b32:
1671 case Intrinsic::amdgcn_flat_load_monitor_b64:
1672 case Intrinsic::amdgcn_flat_load_monitor_b128:
1673 case Intrinsic::amdgcn_global_load_monitor_b32:
1674 case Intrinsic::amdgcn_global_load_monitor_b64:
1675 case Intrinsic::amdgcn_global_load_monitor_b128: {
1676 Info.opc = ISD::INTRINSIC_W_CHAIN;
1677 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1678 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1679 Info.align.reset();
1680 Info.flags = MachineMemOperand::MOLoad;
1681 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1682 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1683 Infos.push_back(Elt: Info);
1684 return;
1685 }
1686 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
1687 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
1688 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
1689 Info.opc = ISD::INTRINSIC_W_CHAIN;
1690 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1691 Info.ptrVal = CI.getOperand(i_nocapture: 0);
1692 Info.align.reset();
1693 Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
1694 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 1);
1695 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 2);
1696 Infos.push_back(Elt: Info);
1697 return;
1698 }
1699 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
1700 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
1701 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
1702 Info.opc = ISD::INTRINSIC_VOID;
1703 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1704 Info.ptrVal = CI.getArgOperand(i: 0);
1705 Info.align.reset();
1706 Info.flags = (MachineMemOperand::MOStore | MOCooperative);
1707 Info.order = parseAtomicOrderingCABIArg(CI, ArgIdx: 2);
1708 Info.ssid = parseSyncscopeMDArg(CI, ArgIdx: 3);
1709 Infos.push_back(Elt: Info);
1710 return;
1711 }
1712 case Intrinsic::amdgcn_ds_gws_init:
1713 case Intrinsic::amdgcn_ds_gws_barrier:
1714 case Intrinsic::amdgcn_ds_gws_sema_v:
1715 case Intrinsic::amdgcn_ds_gws_sema_br:
1716 case Intrinsic::amdgcn_ds_gws_sema_p:
1717 case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1718 Info.opc = ISD::INTRINSIC_VOID;
1719
1720 const GCNTargetMachine &TM =
1721 static_cast<const GCNTargetMachine &>(getTargetMachine());
1722
1723 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1724 Info.ptrVal = MFI->getGWSPSV(TM);
1725
1726 // This is an abstract access, but we need to specify a type and size.
1727 Info.memVT = MVT::i32;
1728 Info.size = 4;
1729 Info.align = Align(4);
1730
1731 if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1732 Info.flags = Flags | MachineMemOperand::MOLoad;
1733 else
1734 Info.flags = Flags | MachineMemOperand::MOStore;
1735 Infos.push_back(Elt: Info);
1736 return;
1737 }
1738 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1739 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1740 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1741 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1742 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1743 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1744 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1745 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
1746 // Entry 0: Load from source (global/flat).
1747 Info.opc = ISD::INTRINSIC_VOID;
1748 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1749 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1750 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1751 Info.flags = Flags | MachineMemOperand::MOLoad;
1752 Infos.push_back(Elt: Info);
1753
1754 // Entry 1: Store to LDS (same offset).
1755 Info.flags = Flags | MachineMemOperand::MOStore;
1756 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1757 Infos.push_back(Elt: Info);
1758 return;
1759 }
1760 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1761 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1762 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1763 case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
1764 // Entry 0: Load from LDS.
1765 Info.opc = ISD::INTRINSIC_VOID;
1766 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: getIntrMemWidth(IntrID));
1767 Info.ptrVal = CI.getArgOperand(i: 1); // LDS pointer
1768 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getSExtValue();
1769 Info.flags = Flags | MachineMemOperand::MOLoad;
1770 Infos.push_back(Elt: Info);
1771
1772 // Entry 1: Store to global (same offset).
1773 Info.flags = Flags | MachineMemOperand::MOStore;
1774 Info.ptrVal = CI.getArgOperand(i: 0); // Global pointer
1775 Infos.push_back(Elt: Info);
1776 return;
1777 }
1778 case Intrinsic::amdgcn_av_load_b128:
1779 case Intrinsic::amdgcn_av_store_b128: {
1780 bool IsStore = IntrID == Intrinsic::amdgcn_av_store_b128;
1781 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
1782 Info.memVT = MVT::v4i32;
1783 Info.ptrVal = CI.getArgOperand(i: 0);
1784 Info.align = Align(16);
1785 Info.flags |=
1786 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
1787 // Pretend to be atomic so that SIMemoryLegalizer::expandStore sets cache
1788 // flags appropriately.
1789 Info.order = AtomicOrdering::Monotonic;
1790
1791 LLVMContext &Ctx = CI.getContext();
1792 unsigned ScopeIdx = CI.arg_size() - 1;
1793 MDNode *ScopeMD = cast<MDNode>(
1794 Val: cast<MetadataAsValue>(Val: CI.getArgOperand(i: ScopeIdx))->getMetadata());
1795 StringRef Scope = cast<MDString>(Val: ScopeMD->getOperand(I: 0))->getString();
1796 Info.ssid = Ctx.getOrInsertSyncScopeID(SSN: Scope);
1797 Infos.push_back(Elt: Info);
1798 return;
1799 }
1800 case Intrinsic::amdgcn_load_to_lds:
1801 case Intrinsic::amdgcn_load_async_to_lds:
1802 case Intrinsic::amdgcn_global_load_lds:
1803 case Intrinsic::amdgcn_global_load_async_lds: {
1804 unsigned Width = cast<ConstantInt>(Val: CI.getArgOperand(i: 2))->getZExtValue();
1805 auto *Aux = cast<ConstantInt>(Val: CI.getArgOperand(i: CI.arg_size() - 1));
1806 bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
1807 if (IsVolatile)
1808 Flags |= MachineMemOperand::MOVolatile;
1809
1810 // Entry 0: Load from source (global/flat).
1811 Info.opc = ISD::INTRINSIC_VOID;
1812 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: Width * 8);
1813 Info.ptrVal = CI.getArgOperand(i: 0); // Source pointer
1814 Info.offset = cast<ConstantInt>(Val: CI.getArgOperand(i: 3))->getSExtValue();
1815 Info.flags = Flags | MachineMemOperand::MOLoad;
1816 Infos.push_back(Elt: Info);
1817
1818 // Entry 1: Store to LDS.
1819 // Same offset from the instruction, but an additional per-lane offset is
1820 // added. Represent that using a wider memory type.
1821 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(),
1822 BitWidth: Width * 8 * Subtarget->getWavefrontSize());
1823 Info.ptrVal = CI.getArgOperand(i: 1); // LDS destination pointer
1824 Info.flags = Flags | MachineMemOperand::MOStore;
1825 Infos.push_back(Elt: Info);
1826 return;
1827 }
1828 case Intrinsic::amdgcn_ds_bvh_stack_rtn:
1829 case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
1830 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
1831 case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
1832 Info.opc = ISD::INTRINSIC_W_CHAIN;
1833
1834 const GCNTargetMachine &TM =
1835 static_cast<const GCNTargetMachine &>(getTargetMachine());
1836
1837 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1838 Info.ptrVal = MFI->getGWSPSV(TM);
1839
1840 // This is an abstract access, but we need to specify a type and size.
1841 Info.memVT = MVT::i32;
1842 Info.size = 4;
1843 Info.align = Align(4);
1844
1845 Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1846 Infos.push_back(Elt: Info);
1847 return;
1848 }
1849 case Intrinsic::amdgcn_s_prefetch_data:
1850 case Intrinsic::amdgcn_s_prefetch_inst:
1851 case Intrinsic::amdgcn_flat_prefetch:
1852 case Intrinsic::amdgcn_global_prefetch: {
1853 Info.opc = ISD::INTRINSIC_VOID;
1854 Info.memVT = EVT::getIntegerVT(Context&: CI.getContext(), BitWidth: 8);
1855 Info.ptrVal = CI.getArgOperand(i: 0);
1856 Info.flags = Flags | MachineMemOperand::MOLoad;
1857 Infos.push_back(Elt: Info);
1858 return;
1859 }
1860 default:
1861 return;
1862 }
1863}
1864
1865void SITargetLowering::CollectTargetIntrinsicOperands(
1866 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
1867 switch (cast<IntrinsicInst>(Val: I).getIntrinsicID()) {
1868 case Intrinsic::amdgcn_addrspacecast_nonnull: {
1869 // The DAG's ValueType loses the addrspaces.
1870 // Add them as 2 extra Constant operands "from" and "to".
1871 unsigned SrcAS = I.getOperand(i_nocapture: 0)->getType()->getPointerAddressSpace();
1872 unsigned DstAS = I.getType()->getPointerAddressSpace();
1873 Ops.push_back(Elt: DAG.getTargetConstant(Val: SrcAS, DL: SDLoc(), VT: MVT::i32));
1874 Ops.push_back(Elt: DAG.getTargetConstant(Val: DstAS, DL: SDLoc(), VT: MVT::i32));
1875 break;
1876 }
1877 default:
1878 break;
1879 }
1880}
1881
1882bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
1883 SmallVectorImpl<Value *> &Ops,
1884 Type *&AccessTy) const {
1885 Value *Ptr = nullptr;
1886 switch (II->getIntrinsicID()) {
1887 case Intrinsic::amdgcn_cluster_load_b128:
1888 case Intrinsic::amdgcn_cluster_load_b64:
1889 case Intrinsic::amdgcn_cluster_load_b32:
1890 case Intrinsic::amdgcn_ds_append:
1891 case Intrinsic::amdgcn_ds_consume:
1892 case Intrinsic::amdgcn_ds_load_tr8_b64:
1893 case Intrinsic::amdgcn_ds_load_tr16_b128:
1894 case Intrinsic::amdgcn_ds_load_tr4_b64:
1895 case Intrinsic::amdgcn_ds_load_tr6_b96:
1896 case Intrinsic::amdgcn_ds_read_tr4_b64:
1897 case Intrinsic::amdgcn_ds_read_tr6_b96:
1898 case Intrinsic::amdgcn_ds_read_tr8_b64:
1899 case Intrinsic::amdgcn_ds_read_tr16_b64:
1900 case Intrinsic::amdgcn_ds_ordered_add:
1901 case Intrinsic::amdgcn_ds_ordered_swap:
1902 case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
1903 case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
1904 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1905 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1906 case Intrinsic::amdgcn_global_atomic_fmax_num:
1907 case Intrinsic::amdgcn_global_atomic_fmin_num:
1908 case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
1909 case Intrinsic::amdgcn_global_load_tr_b64:
1910 case Intrinsic::amdgcn_global_load_tr_b128:
1911 case Intrinsic::amdgcn_global_load_tr4_b64:
1912 case Intrinsic::amdgcn_global_load_tr6_b96:
1913 case Intrinsic::amdgcn_global_store_async_from_lds_b8:
1914 case Intrinsic::amdgcn_global_store_async_from_lds_b32:
1915 case Intrinsic::amdgcn_global_store_async_from_lds_b64:
1916 case Intrinsic::amdgcn_global_store_async_from_lds_b128:
1917 case Intrinsic::amdgcn_av_load_b128:
1918 case Intrinsic::amdgcn_av_store_b128:
1919 Ptr = II->getArgOperand(i: 0);
1920 break;
1921 case Intrinsic::amdgcn_load_to_lds:
1922 case Intrinsic::amdgcn_load_async_to_lds:
1923 case Intrinsic::amdgcn_global_load_lds:
1924 case Intrinsic::amdgcn_global_load_async_lds:
1925 case Intrinsic::amdgcn_global_load_async_to_lds_b8:
1926 case Intrinsic::amdgcn_global_load_async_to_lds_b32:
1927 case Intrinsic::amdgcn_global_load_async_to_lds_b64:
1928 case Intrinsic::amdgcn_global_load_async_to_lds_b128:
1929 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
1930 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
1931 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
1932 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
1933 Ptr = II->getArgOperand(i: 1);
1934 break;
1935 default:
1936 return false;
1937 }
1938 AccessTy = II->getType();
1939 Ops.push_back(Elt: Ptr);
1940 return true;
1941}
1942
1943bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM,
1944 unsigned AddrSpace) const {
1945 if (!Subtarget->hasFlatInstOffsets()) {
1946 // Flat instructions do not have offsets, and only have the register
1947 // address.
1948 return AM.BaseOffs == 0 && AM.Scale == 0;
1949 }
1950
1951 using AMDGPU::FlatAddrSpace;
1952 FlatAddrSpace FlatVariant =
1953 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ? FlatAddrSpace::FlatGlobal
1954 : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? FlatAddrSpace::FlatScratch
1955 : FlatAddrSpace::FLAT;
1956
1957 return AM.Scale == 0 &&
1958 (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
1959 Offset: AM.BaseOffs, AddrSpace, FlatVariant));
1960}
1961
1962bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
1963 if (Subtarget->hasFlatGlobalInsts())
1964 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::GLOBAL_ADDRESS);
1965
1966 if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
1967 // Assume the we will use FLAT for all global memory accesses
1968 // on VI.
1969 // FIXME: This assumption is currently wrong. On VI we still use
1970 // MUBUF instructions for the r + i addressing mode. As currently
1971 // implemented, the MUBUF instructions only work on buffer < 4GB.
1972 // It may be possible to support > 4GB buffers with MUBUF instructions,
1973 // by setting the stride value in the resource descriptor which would
1974 // increase the size limit to (stride * 4GB). However, this is risky,
1975 // because it has never been validated.
1976 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
1977 }
1978
1979 return isLegalMUBUFAddressingMode(AM);
1980}
1981
1982bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
1983 // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
1984 // additionally can do r + r + i with addr64. 32-bit has more addressing
1985 // mode options. Depending on the resource constant, it can also do
1986 // (i64 r0) + (i32 r1) * (i14 i).
1987 //
1988 // Private arrays end up using a scratch buffer most of the time, so also
1989 // assume those use MUBUF instructions. Scratch loads / stores are currently
1990 // implemented as mubuf instructions with offen bit set, so slightly
1991 // different than the normal addr64.
1992 const SIInstrInfo *TII = Subtarget->getInstrInfo();
1993 if (!TII->isLegalMUBUFImmOffset(Imm: AM.BaseOffs))
1994 return false;
1995
1996 // FIXME: Since we can split immediate into soffset and immediate offset,
1997 // would it make sense to allow any immediate?
1998
1999 switch (AM.Scale) {
2000 case 0: // r + i or just i, depending on HasBaseReg.
2001 return true;
2002 case 1:
2003 return true; // We have r + r or r + i.
2004 case 2:
2005 if (AM.HasBaseReg) {
2006 // Reject 2 * r + r.
2007 return false;
2008 }
2009
2010 // Allow 2 * r as r + r
2011 // Or 2 * r + i is allowed as r + r + i.
2012 return true;
2013 default: // Don't allow n * r
2014 return false;
2015 }
2016}
2017
2018bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
2019 const AddrMode &AM, Type *Ty,
2020 unsigned AS,
2021 Instruction *I) const {
2022 // No global is ever allowed as a base.
2023 if (AM.BaseGV)
2024 return false;
2025
2026 if (AS == AMDGPUAS::GLOBAL_ADDRESS)
2027 return isLegalGlobalAddressingMode(AM);
2028
2029 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
2030 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
2031 AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
2032 AS == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2033 // If the offset isn't a multiple of 4, it probably isn't going to be
2034 // correctly aligned.
2035 // FIXME: Can we get the real alignment here?
2036 if (AM.BaseOffs % 4 != 0)
2037 return isLegalMUBUFAddressingMode(AM);
2038
2039 if (!Subtarget->hasScalarSubwordLoads()) {
2040 // There are no SMRD extloads, so if we have to do a small type access we
2041 // will use a MUBUF load.
2042 // FIXME?: We also need to do this if unaligned, but we don't know the
2043 // alignment here.
2044 if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
2045 return isLegalGlobalAddressingMode(AM);
2046 }
2047
2048 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
2049 // SMRD instructions have an 8-bit, dword offset on SI.
2050 if (!isUInt<8>(x: AM.BaseOffs / 4))
2051 return false;
2052 } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
2053 // On CI+, this can also be a 32-bit literal constant offset. If it fits
2054 // in 8-bits, it can use a smaller encoding.
2055 if (!isUInt<32>(x: AM.BaseOffs / 4))
2056 return false;
2057 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
2058 // On VI, these use the SMEM format and the offset is 20-bit in bytes.
2059 if (!isUInt<20>(x: AM.BaseOffs))
2060 return false;
2061 } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
2062 // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
2063 // for S_BUFFER_* instructions).
2064 if (!isInt<21>(x: AM.BaseOffs))
2065 return false;
2066 } else {
2067 // On GFX12, all offsets are signed 24-bit in bytes.
2068 if (!isInt<24>(x: AM.BaseOffs))
2069 return false;
2070 }
2071
2072 if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
2073 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
2074 AM.BaseOffs < 0) {
2075 // Scalar (non-buffer) loads can only use a negative offset if
2076 // soffset+offset is non-negative. Since the compiler can only prove that
2077 // in a few special cases, it is safer to claim that negative offsets are
2078 // not supported.
2079 return false;
2080 }
2081
2082 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2083 return true;
2084
2085 if (AM.Scale == 1 && AM.HasBaseReg)
2086 return true;
2087
2088 return false;
2089 }
2090
2091 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
2092 return Subtarget->hasFlatScratchEnabled()
2093 ? isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS)
2094 : isLegalMUBUFAddressingMode(AM);
2095
2096 if (AS == AMDGPUAS::LOCAL_ADDRESS ||
2097 (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
2098 // Basic, single offset DS instructions allow a 16-bit unsigned immediate
2099 // field.
2100 // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
2101 // an 8-bit dword offset but we don't know the alignment here.
2102 if (!isUInt<16>(x: AM.BaseOffs))
2103 return false;
2104
2105 if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
2106 return true;
2107
2108 if (AM.Scale == 1 && AM.HasBaseReg)
2109 return true;
2110
2111 return false;
2112 }
2113
2114 if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
2115 // For an unknown address space, this usually means that this is for some
2116 // reason being used for pure arithmetic, and not based on some addressing
2117 // computation. We don't have instructions that compute pointers with any
2118 // addressing modes, so treat them as having no offset like flat
2119 // instructions.
2120 return isLegalFlatAddressingMode(AM, AddrSpace: AMDGPUAS::FLAT_ADDRESS);
2121 }
2122
2123 // Assume a user alias of global for unknown address spaces.
2124 return isLegalGlobalAddressingMode(AM);
2125}
2126
2127bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
2128 const MachineFunction &MF) const {
2129 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
2130 return (MemVT.getSizeInBits() <= 4 * 32);
2131 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
2132 unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
2133 return (MemVT.getSizeInBits() <= MaxPrivateBits);
2134 }
2135 if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
2136 return (MemVT.getSizeInBits() <= 2 * 32);
2137 return true;
2138}
2139
2140bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
2141 unsigned Size, unsigned AddrSpace, Align Alignment,
2142 MachineMemOperand::Flags Flags, unsigned *IsFast) const {
2143 if (IsFast)
2144 *IsFast = 0;
2145
2146 if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
2147 AddrSpace == AMDGPUAS::REGION_ADDRESS) {
2148 // Check if alignment requirements for ds_read/write instructions are
2149 // disabled.
2150 if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
2151 return false;
2152
2153 Align RequiredAlignment(
2154 PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))); // Natural alignment.
2155 if (Subtarget->hasLDSMisalignedBugInWGPMode() && Size > 32 &&
2156 Alignment < RequiredAlignment)
2157 return false;
2158
2159 // Either, the alignment requirements are "enabled", or there is an
2160 // unaligned LDS access related hardware bug though alignment requirements
2161 // are "disabled". In either case, we need to check for proper alignment
2162 // requirements.
2163 //
2164 switch (Size) {
2165 case 64:
2166 // SI has a hardware bug in the LDS / GDS bounds checking: if the base
2167 // address is negative, then the instruction is incorrectly treated as
2168 // out-of-bounds even if base + offsets is in bounds. Split vectorized
2169 // loads here to avoid emitting ds_read2_b32. We may re-combine the
2170 // load later in the SILoadStoreOptimizer.
2171 if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
2172 return false;
2173
2174 // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
2175 // can do a 4 byte aligned, 8 byte access in a single operation using
2176 // ds_read2/write2_b32 with adjacent offsets.
2177 RequiredAlignment = Align(4);
2178
2179 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2180 // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
2181 // ds_write2_b32 depending on the alignment. In either case with either
2182 // alignment there is no faster way of doing this.
2183
2184 // The numbers returned here and below are not additive, it is a 'speed
2185 // rank'. They are just meant to be compared to decide if a certain way
2186 // of lowering an operation is faster than another. For that purpose
2187 // naturally aligned operation gets it bitsize to indicate that "it
2188 // operates with a speed comparable to N-bit wide load". With the full
2189 // alignment ds128 is slower than ds96 for example. If underaligned it
2190 // is comparable to a speed of a single dword access, which would then
2191 // mean 32 < 128 and it is faster to issue a wide load regardless.
2192 // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
2193 // wider load which will not be aligned anymore the latter is slower.
2194 if (IsFast)
2195 *IsFast = (Alignment >= RequiredAlignment) ? 64
2196 : (Alignment < Align(4)) ? 32
2197 : 1;
2198 return true;
2199 }
2200
2201 break;
2202 case 96:
2203 if (!Subtarget->hasDS96AndDS128())
2204 return false;
2205
2206 // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
2207 // gfx8 and older.
2208
2209 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2210 // Naturally aligned access is fastest. However, also report it is Fast
2211 // if memory is aligned less than DWORD. A narrow load or store will be
2212 // be equally slow as a single ds_read_b96/ds_write_b96, but there will
2213 // be more of them, so overall we will pay less penalty issuing a single
2214 // instruction.
2215
2216 // See comment on the values above.
2217 if (IsFast)
2218 *IsFast = (Alignment >= RequiredAlignment) ? 96
2219 : (Alignment < Align(4)) ? 32
2220 : 1;
2221 return true;
2222 }
2223
2224 break;
2225 case 128:
2226 if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
2227 return false;
2228
2229 // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
2230 // gfx8 and older, but we can do a 8 byte aligned, 16 byte access in a
2231 // single operation using ds_read2/write2_b64.
2232 RequiredAlignment = Align(8);
2233
2234 if (Subtarget->hasUnalignedDSAccessEnabled()) {
2235 // Naturally aligned access is fastest. However, also report it is Fast
2236 // if memory is aligned less than DWORD. A narrow load or store will be
2237 // be equally slow as a single ds_read_b128/ds_write_b128, but there
2238 // will be more of them, so overall we will pay less penalty issuing a
2239 // single instruction.
2240
2241 // See comment on the values above.
2242 if (IsFast)
2243 *IsFast = (Alignment >= RequiredAlignment) ? 128
2244 : (Alignment < Align(4)) ? 32
2245 : 1;
2246 return true;
2247 }
2248
2249 break;
2250 default:
2251 if (Size > 32)
2252 return false;
2253
2254 break;
2255 }
2256
2257 // See comment on the values above.
2258 // Note that we have a single-dword or sub-dword here, so if underaligned
2259 // it is a slowest possible access, hence returned value is 0.
2260 if (IsFast)
2261 *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
2262
2263 return Alignment >= RequiredAlignment ||
2264 Subtarget->hasUnalignedDSAccessEnabled();
2265 }
2266
2267 // FIXME: We have to be conservative here and assume that flat operations
2268 // will access scratch. If we had access to the IR function, then we
2269 // could determine if any private memory was used in the function.
2270 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
2271 AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
2272 bool AlignedBy4 = Alignment >= Align(4);
2273 if (Subtarget->hasUnalignedScratchAccessEnabled()) {
2274 if (IsFast)
2275 *IsFast = AlignedBy4 ? Size : 1;
2276 return true;
2277 }
2278
2279 if (IsFast)
2280 *IsFast = AlignedBy4;
2281
2282 return AlignedBy4;
2283 }
2284
2285 // So long as they are correct, wide global memory operations perform better
2286 // than multiple smaller memory ops -- even when misaligned
2287 if (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace)) {
2288 if (IsFast)
2289 *IsFast = Size;
2290
2291 return Alignment >= Align(4) ||
2292 Subtarget->hasUnalignedBufferAccessEnabled();
2293 }
2294
2295 // Ensure robust out-of-bounds guarantees for buffer accesses are met when the
2296 // "amdgpu.buffer.oob.mode" module flag has not enabled relaxed untyped-buffer
2297 // OOB semantics. Normally hardware will ensure proper
2298 // out-of-bounds behavior, but in the edge case where an access starts
2299 // out-of-bounds and then enters in-bounds, the entire access would be treated
2300 // as out-of-bounds. Prevent misaligned memory accesses by requiring the
2301 // natural alignment of buffer accesses.
2302 if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
2303 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
2304 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
2305 if (!Subtarget->hasRelaxedBufferOOBMode() &&
2306 Alignment < Align(PowerOf2Ceil(A: divideCeil(Numerator: Size, Denominator: 8))))
2307 return false;
2308 }
2309
2310 // Smaller than dword value must be aligned.
2311 if (Size < 32)
2312 return false;
2313
2314 // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
2315 // byte-address are ignored, thus forcing Dword alignment.
2316 // This applies to private, global, and constant memory.
2317 if (IsFast)
2318 *IsFast = 1;
2319
2320 return Size >= 32 && Alignment >= Align(4);
2321}
2322
2323bool SITargetLowering::allowsMisalignedMemoryAccesses(
2324 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2325 unsigned *IsFast) const {
2326 return allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace,
2327 Alignment, Flags, IsFast);
2328}
2329
2330EVT SITargetLowering::getOptimalMemOpType(
2331 LLVMContext &Context, const MemOp &Op,
2332 const AttributeList &FuncAttributes) const {
2333 // FIXME: Should account for address space here.
2334
2335 // The default fallback uses the private pointer size as a guess for a type to
2336 // use. Make sure we switch these to 64-bit accesses.
2337
2338 if (Op.size() >= 16 &&
2339 Op.isDstAligned(AlignCheck: Align(4))) // XXX: Should only do for global
2340 return MVT::v4i32;
2341
2342 if (Op.size() >= 8 && Op.isDstAligned(AlignCheck: Align(4)))
2343 return MVT::v2i32;
2344
2345 // Use the default.
2346 return MVT::Other;
2347}
2348
2349bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
2350 const MemSDNode *MemNode = cast<MemSDNode>(Val: N);
2351 return MemNode->getMemOperand()->getFlags() & MONoClobber;
2352}
2353
2354bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
2355 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
2356 AS == AMDGPUAS::PRIVATE_ADDRESS;
2357}
2358
2359bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
2360 unsigned DestAS) const {
2361 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
2362 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
2363 Subtarget->hasGloballyAddressableScratch()) {
2364 // Flat -> private requires subtracting src_flat_scratch_base_lo.
2365 return false;
2366 }
2367
2368 // Flat -> private/local is a simple truncate.
2369 // Flat -> global is no-op
2370 return true;
2371 }
2372
2373 const GCNTargetMachine &TM =
2374 static_cast<const GCNTargetMachine &>(getTargetMachine());
2375 return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
2376}
2377
2378TargetLoweringBase::LegalizeTypeAction
2379SITargetLowering::getPreferredVectorAction(MVT VT) const {
2380 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2381 VT.getScalarType().bitsLE(VT: MVT::i16))
2382 return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
2383 return TargetLoweringBase::getPreferredVectorAction(VT);
2384}
2385
2386bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2387 Type *Ty) const {
2388 // FIXME: Could be smarter if called for vector constants.
2389 return true;
2390}
2391
2392bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2393 unsigned Index) const {
2394 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2395 return false;
2396
2397 // TODO: Add more cases that are cheap.
2398 return Index == 0;
2399}
2400
2401bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
2402 // TODO: This should be more aggressive, particular for 16-bit element
2403 // vectors. However there are some mixed improvements and regressions.
2404 EVT EltTy = VT.getVectorElementType();
2405 unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
2406 return EltTy.getSizeInBits() % MinAlign == 0;
2407}
2408
2409bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
2410 if (Subtarget->has16BitInsts() && VT == MVT::i16) {
2411 switch (Op) {
2412 case ISD::LOAD:
2413 case ISD::STORE:
2414 return true;
2415 default:
2416 return false;
2417 }
2418 }
2419
2420 // SimplifySetCC uses this function to determine whether or not it should
2421 // create setcc with i1 operands. We don't have instructions for i1 setcc.
2422 if (VT == MVT::i1 && Op == ISD::SETCC)
2423 return false;
2424
2425 return TargetLowering::isTypeDesirableForOp(Op, VT);
2426}
2427
2428MachinePointerInfo
2429SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
2430 // This isn't really a constant pool but close enough.
2431 MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
2432 PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
2433 return PtrInfo;
2434}
2435
2436SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
2437 const SDLoc &SL,
2438 SDValue Chain,
2439 uint64_t Offset) const {
2440 const DataLayout &DL = DAG.getDataLayout();
2441 MachineFunction &MF = DAG.getMachineFunction();
2442 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2443 MVT PtrVT = getPointerTy(DL, AS: AMDGPUAS::CONSTANT_ADDRESS);
2444
2445 auto [InputPtrReg, RC, ArgTy] =
2446 Info->getPreloadedValue(Value: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
2447
2448 // We may not have the kernarg segment argument if we have no kernel
2449 // arguments.
2450 if (!InputPtrReg)
2451 return DAG.getConstant(Val: Offset, DL: SL, VT: PtrVT);
2452
2453 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
2454 SDValue BasePtr = DAG.getCopyFromReg(
2455 Chain, dl: SL, Reg: MRI.getLiveInVirtReg(PReg: InputPtrReg->getRegister()), VT: PtrVT);
2456
2457 return DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Offset));
2458}
2459
2460SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
2461 const SDLoc &SL) const {
2462 uint64_t Offset =
2463 getImplicitParameterOffset(MF: DAG.getMachineFunction(), Param: FIRST_IMPLICIT);
2464 return lowerKernArgParameterPtr(DAG, SL, Chain: DAG.getEntryNode(), Offset);
2465}
2466
2467SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
2468 const SDLoc &SL) const {
2469
2470 Function &F = DAG.getMachineFunction().getFunction();
2471 std::optional<uint32_t> KnownSize =
2472 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
2473 if (KnownSize.has_value())
2474 return DAG.getConstant(Val: *KnownSize, DL: SL, VT: MVT::i32);
2475 return SDValue();
2476}
2477
2478SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
2479 const SDLoc &SL, SDValue Val,
2480 bool Signed,
2481 const ISD::InputArg *Arg) const {
2482 // First, if it is a widened vector, narrow it.
2483 if (VT.isVector() &&
2484 VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
2485 EVT NarrowedVT =
2486 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(),
2487 NumElements: VT.getVectorNumElements());
2488 Val = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: NarrowedVT, N1: Val,
2489 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
2490 }
2491
2492 // Then convert the vector elements or scalar value.
2493 if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && VT.bitsLT(VT: MemVT)) {
2494 unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
2495 Val = DAG.getNode(Opcode: Opc, DL: SL, VT: MemVT, N1: Val, N2: DAG.getValueType(VT));
2496 }
2497
2498 if (MemVT.isFloatingPoint()) {
2499 if (VT.isFloatingPoint()) {
2500 Val = getFPExtOrFPRound(DAG, Op: Val, DL: SL, VT);
2501 } else {
2502 assert(!MemVT.isVector());
2503 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
2504 SDValue Cast = DAG.getBitcast(VT: IntVT, V: Val);
2505 Val = DAG.getAnyExtOrTrunc(Op: Cast, DL: SL, VT);
2506 }
2507 } else if (Signed)
2508 Val = DAG.getSExtOrTrunc(Op: Val, DL: SL, VT);
2509 else
2510 Val = DAG.getZExtOrTrunc(Op: Val, DL: SL, VT);
2511
2512 return Val;
2513}
2514
2515SDValue SITargetLowering::lowerKernargMemParameter(
2516 SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
2517 uint64_t Offset, Align Alignment, bool Signed,
2518 const ISD::InputArg *Arg) const {
2519
2520 MachinePointerInfo PtrInfo =
2521 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
2522
2523 // Try to avoid using an extload by loading earlier than the argument address,
2524 // and extracting the relevant bits. The load should hopefully be merged with
2525 // the previous argument.
2526 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
2527 // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
2528 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
2529 int64_t OffsetDiff = Offset - AlignDownOffset;
2530
2531 EVT IntVT = MemVT.changeTypeToInteger();
2532
2533 // TODO: If we passed in the base kernel offset we could have a better
2534 // alignment than 4, but we don't really need it.
2535 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset: AlignDownOffset);
2536 SDValue Load = DAG.getLoad(VT: MVT::i32, dl: SL, Chain, Ptr,
2537 PtrInfo: PtrInfo.getWithOffset(O: AlignDownOffset), Alignment: Align(4),
2538 MMOFlags: MachineMemOperand::MODereferenceable |
2539 MachineMemOperand::MOInvariant);
2540
2541 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL: SL, VT: MVT::i32);
2542 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Load, N2: ShiftAmt);
2543
2544 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: IntVT, Operand: Extract);
2545 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MemVT, Operand: ArgVal);
2546 ArgVal = convertArgType(DAG, VT, MemVT, SL, Val: ArgVal, Signed, Arg);
2547
2548 return DAG.getMergeValues(Ops: {ArgVal, Load.getValue(R: 1)}, dl: SL);
2549 }
2550
2551 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
2552 SDValue Load = DAG.getLoad(
2553 VT: MemVT, dl: SL, Chain, Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
2554 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
2555
2556 SDValue Val = convertArgType(DAG, VT, MemVT, SL, Val: Load, Signed, Arg);
2557 return DAG.getMergeValues(Ops: {Val, Load.getValue(R: 1)}, dl: SL);
2558}
2559
2560/// Coerce an argument which was passed in a different ABI type to the original
2561/// expected value type.
2562SDValue SITargetLowering::convertABITypeToValueType(SelectionDAG &DAG,
2563 SDValue Val,
2564 CCValAssign &VA,
2565 const SDLoc &SL) const {
2566 EVT ValVT = VA.getValVT();
2567
2568 // If this is an 8 or 16-bit value, it is really passed promoted
2569 // to 32 bits. Insert an assert[sz]ext to capture this, then
2570 // truncate to the right size.
2571 switch (VA.getLocInfo()) {
2572 case CCValAssign::Full:
2573 return Val;
2574 case CCValAssign::BCvt:
2575 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ValVT, Operand: Val);
2576 case CCValAssign::SExt:
2577 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: SL, VT: VA.getLocVT(), N1: Val,
2578 N2: DAG.getValueType(ValVT));
2579 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2580 case CCValAssign::ZExt:
2581 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: VA.getLocVT(), N1: Val,
2582 N2: DAG.getValueType(ValVT));
2583 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2584 case CCValAssign::AExt:
2585 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ValVT, Operand: Val);
2586 default:
2587 llvm_unreachable("Unknown loc info!");
2588 }
2589}
2590
2591SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG,
2592 CCValAssign &VA, const SDLoc &SL,
2593 SDValue Chain,
2594 const ISD::InputArg &Arg) const {
2595 MachineFunction &MF = DAG.getMachineFunction();
2596 MachineFrameInfo &MFI = MF.getFrameInfo();
2597
2598 if (Arg.Flags.isByVal()) {
2599 unsigned Size = Arg.Flags.getByValSize();
2600 int FrameIdx = MFI.CreateFixedObject(Size, SPOffset: VA.getLocMemOffset(), IsImmutable: false);
2601 return DAG.getFrameIndex(FI: FrameIdx, VT: MVT::i32);
2602 }
2603
2604 unsigned ArgOffset = VA.getLocMemOffset();
2605 unsigned ArgSize = VA.getValVT().getStoreSize();
2606
2607 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: true);
2608
2609 // Create load nodes to retrieve arguments from the stack.
2610 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
2611
2612 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
2613 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
2614 MVT MemVT = VA.getValVT();
2615
2616 switch (VA.getLocInfo()) {
2617 default:
2618 break;
2619 case CCValAssign::BCvt:
2620 MemVT = VA.getLocVT();
2621 break;
2622 case CCValAssign::SExt:
2623 ExtType = ISD::SEXTLOAD;
2624 break;
2625 case CCValAssign::ZExt:
2626 ExtType = ISD::ZEXTLOAD;
2627 break;
2628 case CCValAssign::AExt:
2629 ExtType = ISD::EXTLOAD;
2630 break;
2631 }
2632
2633 SDValue ArgValue = DAG.getExtLoad(
2634 ExtType, dl: SL, VT: VA.getLocVT(), Chain, Ptr: FIN,
2635 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI), MemVT);
2636
2637 SDValue ConvertedVal = convertABITypeToValueType(DAG, Val: ArgValue, VA, SL);
2638 if (ConvertedVal == ArgValue)
2639 return ConvertedVal;
2640
2641 return DAG.getMergeValues(Ops: {ConvertedVal, ArgValue.getValue(R: 1)}, dl: SL);
2642}
2643
2644SDValue SITargetLowering::lowerWorkGroupId(
2645 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2646 AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
2647 AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
2648 AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
2649 if (!Subtarget->hasClusters())
2650 return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2651
2652 // Clusters are supported. Return the global position in the grid. If clusters
2653 // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
2654
2655 // WorkGroupIdXYZ = ClusterId == 0 ?
2656 // ClusterIdXYZ :
2657 // ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
2658 SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
2659 SDLoc SL(ClusterIdXYZ);
2660 SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
2661 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT);
2662 SDValue ClusterSizeXYZ = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterMaxIdXYZ, N2: One);
2663 SDValue ClusterWorkGroupIdXYZ =
2664 getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
2665 SDValue GlobalIdXYZ =
2666 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ClusterWorkGroupIdXYZ,
2667 N2: DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: ClusterIdXYZ, N2: ClusterSizeXYZ));
2668
2669 switch (MFI.getClusterDims().getKind()) {
2670 case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
2671 case AMDGPU::ClusterDimsAttr::Kind::VariableDims:
2672 return GlobalIdXYZ;
2673 case AMDGPU::ClusterDimsAttr::Kind::NoCluster:
2674 return ClusterIdXYZ;
2675 case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
2676 using namespace AMDGPU::Hwreg;
2677 SDValue ClusterIdField =
2678 DAG.getTargetConstant(Val: HwregEncoding::encode(Values: ID_IB_STS2, Values: 6, Values: 4), DL: SL, VT);
2679 SDNode *GetReg =
2680 DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT, Op1: ClusterIdField);
2681 SDValue ClusterId(GetReg, 0);
2682 SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT);
2683 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SL, VT, N1: ClusterId, N2: Zero, N3: ClusterIdXYZ,
2684 N4: GlobalIdXYZ, N5: DAG.getCondCode(Cond: ISD::SETEQ));
2685 }
2686 }
2687
2688 llvm_unreachable("nothing should reach here");
2689}
2690
2691SDValue SITargetLowering::getPreloadedValue(
2692 SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
2693 AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
2694 const ArgDescriptor *Reg = nullptr;
2695 const TargetRegisterClass *RC = nullptr;
2696 LLT Ty;
2697
2698 CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
2699 const ArgDescriptor WorkGroupIDX =
2700 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP9);
2701 // If GridZ is not programmed in an entry function then the hardware will set
2702 // it to all zeros, so there is no need to mask the GridY value in the low
2703 // order bits.
2704 const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
2705 Reg: AMDGPU::TTMP7,
2706 Mask: AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
2707 const ArgDescriptor WorkGroupIDZ =
2708 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP7, Mask: 0xFFFF0000u);
2709 const ArgDescriptor ClusterWorkGroupIDX =
2710 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000000Fu);
2711 const ArgDescriptor ClusterWorkGroupIDY =
2712 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000000F0u);
2713 const ArgDescriptor ClusterWorkGroupIDZ =
2714 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00000F00u);
2715 const ArgDescriptor ClusterWorkGroupMaxIDX =
2716 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0000F000u);
2717 const ArgDescriptor ClusterWorkGroupMaxIDY =
2718 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x000F0000u);
2719 const ArgDescriptor ClusterWorkGroupMaxIDZ =
2720 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x00F00000u);
2721 const ArgDescriptor ClusterWorkGroupMaxFlatID =
2722 ArgDescriptor::createRegister(Reg: AMDGPU::TTMP6, Mask: 0x0F000000u);
2723
2724 auto LoadConstant = [&](unsigned N) {
2725 return DAG.getConstant(Val: N, DL: SDLoc(), VT);
2726 };
2727
2728 if (Subtarget->hasArchitectedSGPRs() &&
2729 (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
2730 AMDGPU::ClusterDimsAttr ClusterDims = MFI.getClusterDims();
2731 bool HasFixedDims = ClusterDims.isFixedDims();
2732
2733 switch (PVID) {
2734 case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
2735 Reg = &WorkGroupIDX;
2736 RC = &AMDGPU::SReg_32RegClass;
2737 Ty = LLT::scalar(SizeInBits: 32);
2738 break;
2739 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
2740 Reg = &WorkGroupIDY;
2741 RC = &AMDGPU::SReg_32RegClass;
2742 Ty = LLT::scalar(SizeInBits: 32);
2743 break;
2744 case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
2745 Reg = &WorkGroupIDZ;
2746 RC = &AMDGPU::SReg_32RegClass;
2747 Ty = LLT::scalar(SizeInBits: 32);
2748 break;
2749 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
2750 if (HasFixedDims && ClusterDims.getDims()[0] == 1)
2751 return LoadConstant(0);
2752 Reg = &ClusterWorkGroupIDX;
2753 RC = &AMDGPU::SReg_32RegClass;
2754 Ty = LLT::scalar(SizeInBits: 32);
2755 break;
2756 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
2757 if (HasFixedDims && ClusterDims.getDims()[1] == 1)
2758 return LoadConstant(0);
2759 Reg = &ClusterWorkGroupIDY;
2760 RC = &AMDGPU::SReg_32RegClass;
2761 Ty = LLT::scalar(SizeInBits: 32);
2762 break;
2763 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
2764 if (HasFixedDims && ClusterDims.getDims()[2] == 1)
2765 return LoadConstant(0);
2766 Reg = &ClusterWorkGroupIDZ;
2767 RC = &AMDGPU::SReg_32RegClass;
2768 Ty = LLT::scalar(SizeInBits: 32);
2769 break;
2770 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
2771 if (HasFixedDims)
2772 return LoadConstant(ClusterDims.getDims()[0] - 1);
2773 Reg = &ClusterWorkGroupMaxIDX;
2774 RC = &AMDGPU::SReg_32RegClass;
2775 Ty = LLT::scalar(SizeInBits: 32);
2776 break;
2777 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
2778 if (HasFixedDims)
2779 return LoadConstant(ClusterDims.getDims()[1] - 1);
2780 Reg = &ClusterWorkGroupMaxIDY;
2781 RC = &AMDGPU::SReg_32RegClass;
2782 Ty = LLT::scalar(SizeInBits: 32);
2783 break;
2784 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
2785 if (HasFixedDims)
2786 return LoadConstant(ClusterDims.getDims()[2] - 1);
2787 Reg = &ClusterWorkGroupMaxIDZ;
2788 RC = &AMDGPU::SReg_32RegClass;
2789 Ty = LLT::scalar(SizeInBits: 32);
2790 break;
2791 case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
2792 Reg = &ClusterWorkGroupMaxFlatID;
2793 RC = &AMDGPU::SReg_32RegClass;
2794 Ty = LLT::scalar(SizeInBits: 32);
2795 break;
2796 default:
2797 break;
2798 }
2799 }
2800
2801 if (!Reg)
2802 std::tie(args&: Reg, args&: RC, args&: Ty) = MFI.getPreloadedValue(Value: PVID);
2803 if (!Reg) {
2804 if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
2805 // It's possible for a kernarg intrinsic call to appear in a kernel with
2806 // no allocated segment, in which case we do not add the user sgpr
2807 // argument, so just return null.
2808 return DAG.getConstant(Val: 0, DL: SDLoc(), VT);
2809 }
2810
2811 // It's undefined behavior if a function marked with the amdgpu-no-*
2812 // attributes uses the corresponding intrinsic.
2813 return DAG.getPOISON(VT);
2814 }
2815
2816 return loadInputValue(DAG, RC, VT, SL: SDLoc(DAG.getEntryNode()), Arg: *Reg);
2817}
2818
2819static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
2820 CallingConv::ID CallConv,
2821 ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
2822 FunctionType *FType,
2823 SIMachineFunctionInfo *Info) {
2824 for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
2825 const ISD::InputArg *Arg = &Ins[I];
2826
2827 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2828 "vector type argument should have been split");
2829
2830 // First check if it's a PS input addr.
2831 if (CallConv == CallingConv::AMDGPU_PS && !Arg->Flags.isInReg() &&
2832 PSInputNum <= 15) {
2833 bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(Index: PSInputNum);
2834
2835 // Inconveniently only the first part of the split is marked as isSplit,
2836 // so skip to the end. We only want to increment PSInputNum once for the
2837 // entire split argument.
2838 if (Arg->Flags.isSplit()) {
2839 while (!Arg->Flags.isSplitEnd()) {
2840 assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
2841 "unexpected vector split in ps argument type");
2842 if (!SkipArg)
2843 Splits.push_back(Elt: *Arg);
2844 Arg = &Ins[++I];
2845 }
2846 }
2847
2848 if (SkipArg) {
2849 // We can safely skip PS inputs.
2850 Skipped.set(Arg->getOrigArgIndex());
2851 ++PSInputNum;
2852 continue;
2853 }
2854
2855 Info->markPSInputAllocated(Index: PSInputNum);
2856 if (Arg->Used)
2857 Info->markPSInputEnabled(Index: PSInputNum);
2858
2859 ++PSInputNum;
2860 }
2861
2862 Splits.push_back(Elt: *Arg);
2863 }
2864}
2865
2866// Allocate special inputs passed in VGPRs.
2867void SITargetLowering::allocateSpecialEntryInputVGPRs(
2868 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2869 SIMachineFunctionInfo &Info) const {
2870 const LLT S32 = LLT::scalar(SizeInBits: 32);
2871 MachineRegisterInfo &MRI = MF.getRegInfo();
2872
2873 if (Info.hasWorkItemIDX()) {
2874 Register Reg = AMDGPU::VGPR0;
2875 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2876
2877 CCInfo.AllocateReg(Reg);
2878 unsigned Mask =
2879 (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
2880 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
2881 }
2882
2883 if (Info.hasWorkItemIDY()) {
2884 assert(Info.hasWorkItemIDX());
2885 if (Subtarget->hasPackedTID()) {
2886 Info.setWorkItemIDY(
2887 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 10));
2888 } else {
2889 unsigned Reg = AMDGPU::VGPR1;
2890 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2891
2892 CCInfo.AllocateReg(Reg);
2893 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
2894 }
2895 }
2896
2897 if (Info.hasWorkItemIDZ()) {
2898 assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
2899 if (Subtarget->hasPackedTID()) {
2900 Info.setWorkItemIDZ(
2901 ArgDescriptor::createRegister(Reg: AMDGPU::VGPR0, Mask: 0x3ff << 20));
2902 } else {
2903 unsigned Reg = AMDGPU::VGPR2;
2904 MRI.setType(VReg: MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass), Ty: S32);
2905
2906 CCInfo.AllocateReg(Reg);
2907 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
2908 }
2909 }
2910}
2911
2912// Try to allocate a VGPR at the end of the argument list, or if no argument
2913// VGPRs are left allocating a stack slot.
2914// If \p Mask is given it indicates bitfield position in the register.
2915// If \p Arg is given use it with new ]p Mask instead of allocating new.
2916static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
2917 ArgDescriptor Arg = ArgDescriptor()) {
2918 if (Arg.isSet())
2919 return ArgDescriptor::createArg(Arg, Mask);
2920
2921 ArrayRef<MCPhysReg> ArgVGPRs = ArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
2922 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgVGPRs);
2923 if (RegIdx == ArgVGPRs.size()) {
2924 // Spill to stack required.
2925 int64_t Offset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
2926
2927 return ArgDescriptor::createStack(Offset, Mask);
2928 }
2929
2930 unsigned Reg = ArgVGPRs[RegIdx];
2931 Reg = CCInfo.AllocateReg(Reg);
2932 assert(Reg != AMDGPU::NoRegister);
2933
2934 MachineFunction &MF = CCInfo.getMachineFunction();
2935 Register LiveInVReg = MF.addLiveIn(PReg: Reg, RC: &AMDGPU::VGPR_32RegClass);
2936 MF.getRegInfo().setType(VReg: LiveInVReg, Ty: LLT::scalar(SizeInBits: 32));
2937 return ArgDescriptor::createRegister(Reg, Mask);
2938}
2939
2940static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
2941 const TargetRegisterClass *RC,
2942 unsigned NumArgRegs) {
2943 ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
2944 unsigned RegIdx = CCInfo.getFirstUnallocated(Regs: ArgSGPRs);
2945 if (RegIdx == ArgSGPRs.size())
2946 report_fatal_error(reason: "ran out of SGPRs for arguments");
2947
2948 unsigned Reg = ArgSGPRs[RegIdx];
2949 Reg = CCInfo.AllocateReg(Reg);
2950 assert(Reg != AMDGPU::NoRegister);
2951
2952 MachineFunction &MF = CCInfo.getMachineFunction();
2953 MF.addLiveIn(PReg: Reg, RC);
2954 return ArgDescriptor::createRegister(Reg);
2955}
2956
2957// If this has a fixed position, we still should allocate the register in the
2958// CCInfo state. Technically we could get away with this for values passed
2959// outside of the normal argument range.
2960static void allocateFixedSGPRInputImpl(CCState &CCInfo,
2961 const TargetRegisterClass *RC,
2962 MCRegister Reg) {
2963 Reg = CCInfo.AllocateReg(Reg);
2964 assert(Reg != AMDGPU::NoRegister);
2965 MachineFunction &MF = CCInfo.getMachineFunction();
2966 MF.addLiveIn(PReg: Reg, RC);
2967}
2968
2969static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
2970 if (Arg) {
2971 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass,
2972 Reg: Arg.getRegister());
2973 } else
2974 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_32RegClass, NumArgRegs: 32);
2975}
2976
2977static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
2978 if (Arg) {
2979 allocateFixedSGPRInputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass,
2980 Reg: Arg.getRegister());
2981 } else
2982 Arg = allocateSGPR32InputImpl(CCInfo, RC: &AMDGPU::SGPR_64RegClass, NumArgRegs: 16);
2983}
2984
2985/// Allocate implicit function VGPR arguments at the end of allocated user
2986/// arguments.
2987void SITargetLowering::allocateSpecialInputVGPRs(
2988 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
2989 SIMachineFunctionInfo &Info) const {
2990 const unsigned Mask = 0x3ff;
2991 ArgDescriptor Arg;
2992
2993 if (Info.hasWorkItemIDX()) {
2994 Arg = allocateVGPR32Input(CCInfo, Mask);
2995 Info.setWorkItemIDX(Arg);
2996 }
2997
2998 if (Info.hasWorkItemIDY()) {
2999 Arg = allocateVGPR32Input(CCInfo, Mask: Mask << 10, Arg);
3000 Info.setWorkItemIDY(Arg);
3001 }
3002
3003 if (Info.hasWorkItemIDZ())
3004 Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask: Mask << 20, Arg));
3005}
3006
3007/// Allocate implicit function VGPR arguments in fixed registers.
3008void SITargetLowering::allocateSpecialInputVGPRsFixed(
3009 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3010 SIMachineFunctionInfo &Info) const {
3011 Register Reg = CCInfo.AllocateReg(Reg: AMDGPU::VGPR31);
3012 if (!Reg)
3013 report_fatal_error(reason: "failed to allocate VGPR for implicit arguments");
3014
3015 const unsigned Mask = 0x3ff;
3016 Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
3017 Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask: Mask << 10));
3018 Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask: Mask << 20));
3019}
3020
3021void SITargetLowering::allocateSpecialInputSGPRs(
3022 CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
3023 SIMachineFunctionInfo &Info) const {
3024 auto &ArgInfo = Info.getArgInfo();
3025 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3026
3027 // TODO: Unify handling with private memory pointers.
3028 if (UserSGPRInfo.hasDispatchPtr())
3029 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchPtr);
3030
3031 if (UserSGPRInfo.hasQueuePtr())
3032 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.QueuePtr);
3033
3034 // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
3035 // constant offset from the kernarg segment.
3036 if (Info.hasImplicitArgPtr())
3037 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.ImplicitArgPtr);
3038
3039 if (UserSGPRInfo.hasDispatchID())
3040 allocateSGPR64Input(CCInfo, Arg&: ArgInfo.DispatchID);
3041
3042 // flat_scratch_init is not applicable for non-kernel functions.
3043
3044 if (Info.hasWorkGroupIDX())
3045 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDX);
3046
3047 if (Info.hasWorkGroupIDY())
3048 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDY);
3049
3050 if (Info.hasWorkGroupIDZ())
3051 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.WorkGroupIDZ);
3052
3053 if (Info.hasLDSKernelId())
3054 allocateSGPR32Input(CCInfo, Arg&: ArgInfo.LDSKernelId);
3055}
3056
3057// Allocate special inputs passed in user SGPRs.
3058void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
3059 MachineFunction &MF,
3060 const SIRegisterInfo &TRI,
3061 SIMachineFunctionInfo &Info) const {
3062 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
3063 if (UserSGPRInfo.hasImplicitBufferPtr()) {
3064 Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
3065 MF.addLiveIn(PReg: ImplicitBufferPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3066 CCInfo.AllocateReg(Reg: ImplicitBufferPtrReg);
3067 }
3068
3069 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
3070 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
3071 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
3072 MF.addLiveIn(PReg: PrivateSegmentBufferReg, RC: &AMDGPU::SGPR_128RegClass);
3073 CCInfo.AllocateReg(Reg: PrivateSegmentBufferReg);
3074 }
3075
3076 if (UserSGPRInfo.hasDispatchPtr()) {
3077 Register DispatchPtrReg = Info.addDispatchPtr(TRI);
3078 MF.addLiveIn(PReg: DispatchPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3079 CCInfo.AllocateReg(Reg: DispatchPtrReg);
3080 }
3081
3082 if (UserSGPRInfo.hasQueuePtr()) {
3083 Register QueuePtrReg = Info.addQueuePtr(TRI);
3084 MF.addLiveIn(PReg: QueuePtrReg, RC: &AMDGPU::SGPR_64RegClass);
3085 CCInfo.AllocateReg(Reg: QueuePtrReg);
3086 }
3087
3088 if (UserSGPRInfo.hasKernargSegmentPtr()) {
3089 MachineRegisterInfo &MRI = MF.getRegInfo();
3090 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
3091 CCInfo.AllocateReg(Reg: InputPtrReg);
3092
3093 Register VReg = MF.addLiveIn(PReg: InputPtrReg, RC: &AMDGPU::SGPR_64RegClass);
3094 MRI.setType(VReg, Ty: LLT::pointer(AddressSpace: AMDGPUAS::CONSTANT_ADDRESS, SizeInBits: 64));
3095 }
3096
3097 if (UserSGPRInfo.hasDispatchID()) {
3098 Register DispatchIDReg = Info.addDispatchID(TRI);
3099 MF.addLiveIn(PReg: DispatchIDReg, RC: &AMDGPU::SGPR_64RegClass);
3100 CCInfo.AllocateReg(Reg: DispatchIDReg);
3101 }
3102
3103 if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
3104 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
3105 MF.addLiveIn(PReg: FlatScratchInitReg, RC: &AMDGPU::SGPR_64RegClass);
3106 CCInfo.AllocateReg(Reg: FlatScratchInitReg);
3107 }
3108
3109 if (UserSGPRInfo.hasPrivateSegmentSize()) {
3110 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
3111 MF.addLiveIn(PReg: PrivateSegmentSizeReg, RC: &AMDGPU::SGPR_32RegClass);
3112 CCInfo.AllocateReg(Reg: PrivateSegmentSizeReg);
3113 }
3114
3115 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
3116 // these from the dispatch pointer.
3117}
3118
3119// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
3120// sequential starting from the first argument.
3121void SITargetLowering::allocatePreloadKernArgSGPRs(
3122 CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
3123 const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
3124 const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
3125 Function &F = MF.getFunction();
3126 unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
3127 GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
3128 bool InPreloadSequence = true;
3129 unsigned InIdx = 0;
3130 bool AlignedForImplictArgs = false;
3131 unsigned ImplicitArgOffset = 0;
3132 for (auto &Arg : F.args()) {
3133 if (!InPreloadSequence || !Arg.hasInRegAttr())
3134 break;
3135
3136 unsigned ArgIdx = Arg.getArgNo();
3137 // Don't preload non-original args or parts not in the current preload
3138 // sequence.
3139 if (InIdx < Ins.size() &&
3140 (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
3141 break;
3142
3143 for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
3144 Ins[InIdx].getOrigArgIndex() == ArgIdx;
3145 InIdx++) {
3146 assert(ArgLocs[ArgIdx].isMemLoc());
3147 auto &ArgLoc = ArgLocs[InIdx];
3148 const Align KernelArgBaseAlign = Align(16);
3149 unsigned ArgOffset = ArgLoc.getLocMemOffset();
3150 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset: ArgOffset);
3151 unsigned NumAllocSGPRs =
3152 alignTo(Value: ArgLoc.getLocVT().getFixedSizeInBits(), Align: 32) / 32;
3153
3154 // Fix alignment for hidden arguments.
3155 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument")) {
3156 if (!AlignedForImplictArgs) {
3157 ImplicitArgOffset =
3158 alignTo(Size: LastExplicitArgOffset,
3159 A: Subtarget->getAlignmentForImplicitArgPtr()) -
3160 LastExplicitArgOffset;
3161 AlignedForImplictArgs = true;
3162 }
3163 ArgOffset += ImplicitArgOffset;
3164 }
3165
3166 // Arg is preloaded into the previous SGPR.
3167 if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
3168 assert(InIdx >= 1 && "No previous SGPR");
3169 Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
3170 Elt: Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
3171 continue;
3172 }
3173
3174 unsigned Padding = ArgOffset - LastExplicitArgOffset;
3175 unsigned PaddingSGPRs = alignTo(Value: Padding, Align: 4) / 4;
3176 // Check for free user SGPRs for preloading.
3177 if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
3178 InPreloadSequence = false;
3179 break;
3180 }
3181
3182 // Preload this argument.
3183 const TargetRegisterClass *RC =
3184 TRI.getSGPRClassForBitWidth(BitWidth: NumAllocSGPRs * 32);
3185 SmallVectorImpl<MCRegister> *PreloadRegs =
3186 Info.addPreloadedKernArg(TRI, RC, AllocSizeDWord: NumAllocSGPRs, KernArgIdx: InIdx, PaddingSGPRs);
3187
3188 if (PreloadRegs->size() > 1)
3189 RC = &AMDGPU::SGPR_32RegClass;
3190 for (auto &Reg : *PreloadRegs) {
3191 assert(Reg);
3192 MF.addLiveIn(PReg: Reg, RC);
3193 CCInfo.AllocateReg(Reg);
3194 }
3195
3196 LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
3197 }
3198 }
3199}
3200
3201void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
3202 const SIRegisterInfo &TRI,
3203 SIMachineFunctionInfo &Info) const {
3204 // Always allocate this last since it is a synthetic preload.
3205 if (Info.hasLDSKernelId()) {
3206 Register Reg = Info.addLDSKernelId();
3207 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3208 CCInfo.AllocateReg(Reg);
3209 }
3210}
3211
3212// Allocate special input registers that are initialized per-wave.
3213void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF,
3214 SIMachineFunctionInfo &Info,
3215 CallingConv::ID CallConv,
3216 bool IsShader) const {
3217 bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
3218 if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
3219 // Note: user SGPRs are handled by the front-end for graphics shaders
3220 // Pad up the used user SGPRs with dead inputs.
3221
3222 // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
3223 // before enabling architected SGPRs for workgroup IDs.
3224 assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
3225
3226 unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
3227 // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
3228 // rely on it to reach 16 since if we end up having no stack usage, it will
3229 // not really be added.
3230 unsigned NumRequiredSystemSGPRs =
3231 Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
3232 Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
3233 for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
3234 Register Reg = Info.addReservedUserSGPR();
3235 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3236 CCInfo.AllocateReg(Reg);
3237 }
3238 }
3239
3240 if (!HasArchitectedSGPRs) {
3241 if (Info.hasWorkGroupIDX()) {
3242 Register Reg = Info.addWorkGroupIDX();
3243 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3244 CCInfo.AllocateReg(Reg);
3245 }
3246
3247 if (Info.hasWorkGroupIDY()) {
3248 Register Reg = Info.addWorkGroupIDY();
3249 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3250 CCInfo.AllocateReg(Reg);
3251 }
3252
3253 if (Info.hasWorkGroupIDZ()) {
3254 Register Reg = Info.addWorkGroupIDZ();
3255 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3256 CCInfo.AllocateReg(Reg);
3257 }
3258 }
3259
3260 if (Info.hasWorkGroupInfo()) {
3261 Register Reg = Info.addWorkGroupInfo();
3262 MF.addLiveIn(PReg: Reg, RC: &AMDGPU::SGPR_32RegClass);
3263 CCInfo.AllocateReg(Reg);
3264 }
3265
3266 if (Info.hasPrivateSegmentWaveByteOffset()) {
3267 // Scratch wave offset passed in system SGPR.
3268 unsigned PrivateSegmentWaveByteOffsetReg;
3269
3270 if (IsShader) {
3271 PrivateSegmentWaveByteOffsetReg =
3272 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
3273
3274 // This is true if the scratch wave byte offset doesn't have a fixed
3275 // location.
3276 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
3277 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
3278 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
3279 }
3280 } else
3281 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
3282
3283 MF.addLiveIn(PReg: PrivateSegmentWaveByteOffsetReg, RC: &AMDGPU::SGPR_32RegClass);
3284 CCInfo.AllocateReg(Reg: PrivateSegmentWaveByteOffsetReg);
3285 }
3286
3287 assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
3288 Info.getNumPreloadedSGPRs() >= 16);
3289}
3290
3291static void reservePrivateMemoryRegs(const TargetMachine &TM,
3292 MachineFunction &MF,
3293 const SIRegisterInfo &TRI,
3294 SIMachineFunctionInfo &Info) {
3295 // Now that we've figured out where the scratch register inputs are, see if
3296 // should reserve the arguments and use them directly.
3297 MachineFrameInfo &MFI = MF.getFrameInfo();
3298 bool HasStackObjects = MFI.hasStackObjects();
3299 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3300
3301 // Record that we know we have non-spill stack objects so we don't need to
3302 // check all stack objects later.
3303 if (HasStackObjects)
3304 Info.setHasNonSpillStackObjects(true);
3305
3306 // Everything live out of a block is spilled with fast regalloc, so it's
3307 // almost certain that spilling will be required.
3308 if (TM.getOptLevel() == CodeGenOptLevel::None)
3309 HasStackObjects = true;
3310
3311 // For now assume stack access is needed in any callee functions, so we need
3312 // the scratch registers to pass in.
3313 bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
3314
3315 if (!ST.hasFlatScratchEnabled()) {
3316 if (RequiresStackAccess && ST.isAmdHsaOrMesa(F: MF.getFunction())) {
3317 // If we have stack objects, we unquestionably need the private buffer
3318 // resource. For the Code Object V2 ABI, this will be the first 4 user
3319 // SGPR inputs. We can reserve those and use them directly.
3320
3321 Register PrivateSegmentBufferReg =
3322 Info.getPreloadedReg(Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
3323 Info.setScratchRSrcReg(PrivateSegmentBufferReg);
3324 } else {
3325 unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
3326 // We tentatively reserve the last registers (skipping the last registers
3327 // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
3328 // we'll replace these with the ones immediately after those which were
3329 // really allocated. In the prologue copies will be inserted from the
3330 // argument to these reserved registers.
3331
3332 // Without HSA, relocations are used for the scratch pointer and the
3333 // buffer resource setup is always inserted in the prologue. Scratch wave
3334 // offset is still in an input SGPR.
3335 Info.setScratchRSrcReg(ReservedBufferReg);
3336 }
3337 }
3338
3339 MachineRegisterInfo &MRI = MF.getRegInfo();
3340
3341 // For entry functions we have to set up the stack pointer if we use it,
3342 // whereas non-entry functions get this "for free". This means there is no
3343 // intrinsic advantage to using S32 over S34 in cases where we do not have
3344 // calls but do need a frame pointer (i.e. if we are requested to have one
3345 // because frame pointer elimination is disabled). To keep things simple we
3346 // only ever use S32 as the call ABI stack pointer, and so using it does not
3347 // imply we need a separate frame pointer.
3348 //
3349 // Try to use s32 as the SP, but move it if it would interfere with input
3350 // arguments. This won't work with calls though.
3351 //
3352 // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
3353 // registers.
3354 if (!MRI.isLiveIn(Reg: AMDGPU::SGPR32)) {
3355 Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
3356 } else {
3357 assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
3358
3359 if (MFI.hasCalls())
3360 report_fatal_error(reason: "call in graphics shader with too many input SGPRs");
3361
3362 for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
3363 if (!MRI.isLiveIn(Reg)) {
3364 Info.setStackPtrOffsetReg(Reg);
3365 break;
3366 }
3367 }
3368
3369 if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
3370 report_fatal_error(reason: "failed to find register for SP");
3371 }
3372
3373 // hasFP should be accurate for entry functions even before the frame is
3374 // finalized, because it does not rely on the known stack size, only
3375 // properties like whether variable sized objects are present.
3376 if (ST.getFrameLowering()->hasFP(MF)) {
3377 Info.setFrameOffsetReg(AMDGPU::SGPR33);
3378 }
3379}
3380
3381bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
3382 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
3383 return !Info->isEntryFunction();
3384}
3385
3386void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {}
3387
3388void SITargetLowering::insertCopiesSplitCSR(
3389 MachineBasicBlock *Entry,
3390 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
3391 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3392
3393 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(MF: Entry->getParent());
3394 if (!IStart)
3395 return;
3396
3397 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3398 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
3399 MachineBasicBlock::iterator MBBI = Entry->begin();
3400 for (const MCPhysReg *I = IStart; *I; ++I) {
3401 const TargetRegisterClass *RC = nullptr;
3402 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3403 RC = &AMDGPU::SGPR_64RegClass;
3404 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3405 RC = &AMDGPU::SGPR_32RegClass;
3406 else
3407 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3408
3409 Register NewVR = MRI->createVirtualRegister(RegClass: RC);
3410 // Create copy from CSR to a virtual register.
3411 Entry->addLiveIn(PhysReg: *I);
3412 BuildMI(BB&: *Entry, I: MBBI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVR)
3413 .addReg(RegNo: *I);
3414
3415 // Insert the copy-back instructions right before the terminator.
3416 for (auto *Exit : Exits)
3417 BuildMI(BB&: *Exit, I: Exit->getFirstTerminator(), MIMD: DebugLoc(),
3418 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: *I)
3419 .addReg(RegNo: NewVR);
3420 }
3421}
3422
3423SDValue SITargetLowering::LowerFormalArguments(
3424 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3425 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3426 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3427 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3428
3429 MachineFunction &MF = DAG.getMachineFunction();
3430 const Function &Fn = MF.getFunction();
3431 FunctionType *FType = MF.getFunction().getFunctionType();
3432 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3433 bool IsError = false;
3434
3435 if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CC: CallConv)) {
3436 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3437 Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()));
3438 IsError = true;
3439 }
3440
3441 SmallVector<ISD::InputArg, 16> Splits;
3442 SmallVector<CCValAssign, 16> ArgLocs;
3443 BitVector Skipped(Ins.size());
3444 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3445 *DAG.getContext());
3446
3447 bool IsGraphics = AMDGPU::isGraphics(CC: CallConv);
3448 bool IsKernel = AMDGPU::isKernel(CC: CallConv);
3449 bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC: CallConv);
3450
3451 if (IsGraphics) {
3452 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
3453 assert(!UserSGPRInfo.hasDispatchPtr() &&
3454 !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
3455 !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
3456 !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
3457 (void)UserSGPRInfo;
3458 if (!Subtarget->hasFlatScratchEnabled())
3459 assert(!UserSGPRInfo.hasFlatScratchInit());
3460 if ((CallConv != CallingConv::AMDGPU_CS &&
3461 CallConv != CallingConv::AMDGPU_Gfx &&
3462 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) ||
3463 !Subtarget->hasArchitectedSGPRs())
3464 assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
3465 !Info->hasWorkGroupIDZ());
3466 }
3467
3468 bool IsWholeWaveFunc = Info->isWholeWaveFunction();
3469
3470 if (CallConv == CallingConv::AMDGPU_PS) {
3471 processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
3472
3473 // At least one interpolation mode must be enabled or else the GPU will
3474 // hang.
3475 //
3476 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
3477 // set PSInputAddr, the user wants to enable some bits after the compilation
3478 // based on run-time states. Since we can't know what the final PSInputEna
3479 // will look like, so we shouldn't do anything here and the user should take
3480 // responsibility for the correct programming.
3481 //
3482 // Otherwise, the following restrictions apply:
3483 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
3484 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
3485 // enabled too.
3486 if ((Info->getPSInputAddr() & 0x7F) == 0 ||
3487 ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(Index: 11))) {
3488 CCInfo.AllocateReg(Reg: AMDGPU::VGPR0);
3489 CCInfo.AllocateReg(Reg: AMDGPU::VGPR1);
3490 Info->markPSInputAllocated(Index: 0);
3491 Info->markPSInputEnabled(Index: 0);
3492 }
3493 if (Subtarget->isAmdPalOS()) {
3494 // For isAmdPalOS, the user does not enable some bits after compilation
3495 // based on run-time states; the register values being generated here are
3496 // the final ones set in hardware. Therefore we need to apply the
3497 // workaround to PSInputAddr and PSInputEnable together. (The case where
3498 // a bit is set in PSInputAddr but not PSInputEnable is where the
3499 // frontend set up an input arg for a particular interpolation mode, but
3500 // nothing uses that input arg. Really we should have an earlier pass
3501 // that removes such an arg.)
3502 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
3503 if ((PsInputBits & 0x7F) == 0 ||
3504 ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
3505 Info->markPSInputEnabled(Index: llvm::countr_zero(Val: Info->getPSInputAddr()));
3506 }
3507 } else if (IsKernel) {
3508 assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
3509 } else {
3510 Splits.append(in_start: IsWholeWaveFunc ? std::next(x: Ins.begin()) : Ins.begin(),
3511 in_end: Ins.end());
3512 }
3513
3514 if (IsKernel)
3515 analyzeFormalArgumentsCompute(State&: CCInfo, Ins);
3516
3517 if (IsEntryFunc) {
3518 allocateSpecialEntryInputVGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3519 allocateHSAUserSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3520 if (IsKernel && Subtarget->hasKernargPreload())
3521 allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, TRI: *TRI, Info&: *Info);
3522
3523 allocateLDSKernelId(CCInfo, MF, TRI: *TRI, Info&: *Info);
3524 } else if (!IsGraphics) {
3525 // For the fixed ABI, pass workitem IDs in the last argument register.
3526 allocateSpecialInputVGPRsFixed(CCInfo, MF, TRI: *TRI, Info&: *Info);
3527
3528 // FIXME: Sink this into allocateSpecialInputSGPRs
3529 if (!Subtarget->hasFlatScratchEnabled())
3530 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
3531
3532 allocateSpecialInputSGPRs(CCInfo, MF, TRI: *TRI, Info&: *Info);
3533 }
3534
3535 if (!IsKernel) {
3536 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg: isVarArg);
3537 CCInfo.AnalyzeFormalArguments(Ins: Splits, Fn: AssignFn);
3538
3539 // This assumes the registers are allocated by CCInfo in ascending order
3540 // with no gaps.
3541 Info->setNumWaveDispatchSGPRs(
3542 CCInfo.getFirstUnallocated(Regs: AMDGPU::SGPR_32RegClass.getRegisters()));
3543 Info->setNumWaveDispatchVGPRs(
3544 CCInfo.getFirstUnallocated(Regs: AMDGPU::VGPR_32RegClass.getRegisters()));
3545 } else if (Info->getNumKernargPreloadedSGPRs()) {
3546 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
3547 }
3548
3549 SmallVector<SDValue, 16> Chains;
3550
3551 if (IsWholeWaveFunc) {
3552 SDValue Setup = DAG.getNode(Opcode: AMDGPUISD::WHOLE_WAVE_SETUP, DL,
3553 ResultTys: {MVT::i1, MVT::Other}, Ops: Chain);
3554 InVals.push_back(Elt: Setup.getValue(R: 0));
3555 Chains.push_back(Elt: Setup.getValue(R: 1));
3556 }
3557
3558 // FIXME: This is the minimum kernel argument alignment. We should improve
3559 // this to the maximum alignment of the arguments.
3560 //
3561 // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
3562 // kern arg offset.
3563 const Align KernelArgBaseAlign = Align(16);
3564
3565 for (unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.size(), ArgIdx = 0; i != e;
3566 ++i) {
3567 const ISD::InputArg &Arg = Ins[i];
3568 if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) {
3569 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
3570 continue;
3571 }
3572
3573 CCValAssign &VA = ArgLocs[ArgIdx++];
3574 MVT VT = VA.getLocVT();
3575
3576 if (IsEntryFunc && VA.isMemLoc()) {
3577 VT = Ins[i].VT;
3578 EVT MemVT = VA.getLocVT();
3579
3580 const uint64_t Offset = VA.getLocMemOffset();
3581 Align Alignment = commonAlignment(A: KernelArgBaseAlign, Offset);
3582
3583 if (Arg.Flags.isByRef()) {
3584 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain, Offset);
3585
3586 const GCNTargetMachine &TM =
3587 static_cast<const GCNTargetMachine &>(getTargetMachine());
3588 if (!TM.isNoopAddrSpaceCast(SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3589 DestAS: Arg.Flags.getPointerAddrSpace())) {
3590 Ptr = DAG.getAddrSpaceCast(dl: DL, VT, Ptr, SrcAS: AMDGPUAS::CONSTANT_ADDRESS,
3591 DestAS: Arg.Flags.getPointerAddrSpace());
3592 }
3593
3594 InVals.push_back(Elt: Ptr);
3595 continue;
3596 }
3597
3598 SDValue NewArg;
3599 if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(Val: i)) {
3600 if (MemVT.getStoreSize() < 4 && Alignment < 4) {
3601 // In this case the argument is packed into the previous preload SGPR.
3602 int64_t AlignDownOffset = alignDown(Value: Offset, Align: 4);
3603 int64_t OffsetDiff = Offset - AlignDownOffset;
3604 EVT IntVT = MemVT.changeTypeToInteger();
3605
3606 const SIMachineFunctionInfo *Info =
3607 MF.getInfo<SIMachineFunctionInfo>();
3608 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3609 Register Reg =
3610 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs[0];
3611
3612 assert(Reg);
3613 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3614 SDValue Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3615
3616 SDValue ShiftAmt = DAG.getConstant(Val: OffsetDiff * 8, DL, VT: MVT::i32);
3617 SDValue Extract = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Copy, N2: ShiftAmt);
3618
3619 SDValue ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Extract);
3620 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MemVT, Operand: ArgVal);
3621 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: ArgVal,
3622 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3623
3624 NewArg = DAG.getMergeValues(Ops: {NewArg, Copy.getValue(R: 1)}, dl: DL);
3625 } else {
3626 const SIMachineFunctionInfo *Info =
3627 MF.getInfo<SIMachineFunctionInfo>();
3628 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
3629 const SmallVectorImpl<MCRegister> &PreloadRegs =
3630 Info->getArgInfo().PreloadKernArgs.find(Val: i)->getSecond().Regs;
3631
3632 SDValue Copy;
3633 if (PreloadRegs.size() == 1) {
3634 Register VReg = MRI.getLiveInVirtReg(PReg: PreloadRegs[0]);
3635 const TargetRegisterClass *RC = MRI.getRegClass(Reg: VReg);
3636 NewArg = DAG.getCopyFromReg(
3637 Chain, dl: DL, Reg: VReg,
3638 VT: EVT::getIntegerVT(Context&: *DAG.getContext(),
3639 BitWidth: TRI->getRegSizeInBits(RC: *RC)));
3640
3641 } else {
3642 // If the kernarg alignment does not match the alignment of the SGPR
3643 // tuple RC that can accommodate this argument, it will be built up
3644 // via copies from from the individual SGPRs that the argument was
3645 // preloaded to.
3646 SmallVector<SDValue, 4> Elts;
3647 for (auto Reg : PreloadRegs) {
3648 Register VReg = MRI.getLiveInVirtReg(PReg: Reg);
3649 Copy = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: MVT::i32);
3650 Elts.push_back(Elt: Copy);
3651 }
3652 NewArg =
3653 DAG.getBuildVector(VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
3654 NumElements: PreloadRegs.size()),
3655 DL, Ops: Elts);
3656 }
3657
3658 // If the argument was preloaded to multiple consecutive 32-bit
3659 // registers because of misalignment between addressable SGPR tuples
3660 // and the argument size, we can still assume that because of kernarg
3661 // segment alignment restrictions that NewArg's size is the same as
3662 // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
3663 // truncate since we cannot preload to less than a single SGPR and the
3664 // MemVT may be smaller.
3665 EVT MemVTInt =
3666 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
3667 if (MemVT.bitsLT(VT: NewArg.getSimpleValueType()))
3668 NewArg = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVTInt, Operand: NewArg);
3669
3670 NewArg = DAG.getBitcast(VT: MemVT, V: NewArg);
3671 NewArg = convertArgType(DAG, VT, MemVT, SL: DL, Val: NewArg,
3672 Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3673 NewArg = DAG.getMergeValues(Ops: {NewArg, Chain}, dl: DL);
3674 }
3675 } else {
3676 // Hidden arguments that are in the kernel signature must be preloaded
3677 // to user SGPRs. Print a diagnostic error if a hidden argument is in
3678 // the argument list and is not preloaded.
3679 if (Arg.isOrigArg()) {
3680 Argument *OrigArg = Fn.getArg(i: Arg.getOrigArgIndex());
3681 if (OrigArg->hasAttribute(Kind: "amdgpu-hidden-argument")) {
3682 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
3683 *OrigArg->getParent(),
3684 "hidden argument in kernel signature was not preloaded",
3685 DL.getDebugLoc()));
3686 }
3687 }
3688
3689 NewArg =
3690 lowerKernargMemParameter(DAG, VT, MemVT, SL: DL, Chain, Offset,
3691 Alignment, Signed: Ins[i].Flags.isSExt(), Arg: &Ins[i]);
3692 }
3693 Chains.push_back(Elt: NewArg.getValue(R: 1));
3694
3695 auto *ParamTy =
3696 dyn_cast<PointerType>(Val: FType->getParamType(i: Ins[i].getOrigArgIndex()));
3697 if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
3698 ParamTy &&
3699 (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
3700 ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
3701 // On SI local pointers are just offsets into LDS, so they are always
3702 // less than 16-bits. On CI and newer they could potentially be
3703 // real pointers, so we can't guarantee their size.
3704 NewArg = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: NewArg.getValueType(), N1: NewArg,
3705 N2: DAG.getValueType(MVT::i16));
3706 }
3707
3708 InVals.push_back(Elt: NewArg);
3709 continue;
3710 }
3711 if (!IsEntryFunc && VA.isMemLoc()) {
3712 SDValue Val = lowerStackParameter(DAG, VA, SL: DL, Chain, Arg);
3713 InVals.push_back(Elt: Val);
3714 if (!Arg.Flags.isByVal())
3715 Chains.push_back(Elt: Val.getValue(R: 1));
3716 continue;
3717 }
3718
3719 assert(VA.isRegLoc() && "Parameter must be in a register!");
3720
3721 Register Reg = VA.getLocReg();
3722 const TargetRegisterClass *RC = nullptr;
3723 if (AMDGPU::VGPR_32RegClass.contains(Reg))
3724 RC = &AMDGPU::VGPR_32RegClass;
3725 else if (AMDGPU::SGPR_32RegClass.contains(Reg))
3726 RC = &AMDGPU::SGPR_32RegClass;
3727 else
3728 llvm_unreachable("Unexpected register class in LowerFormalArguments!");
3729
3730 Reg = MF.addLiveIn(PReg: Reg, RC);
3731 SDValue Val = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT);
3732 if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
3733 // FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
3734 // they will read physical regs before any side effect instructions.
3735 SDValue ReadFirstLane =
3736 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3737 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
3738 N1: ReadFirstLane, N2: Val);
3739 }
3740
3741 if (Arg.Flags.isSRet()) {
3742 // The return object should be reasonably addressable.
3743 Val = annotateStackObjectPointer(Ptr: Val, DAG, DL,
3744 Alignment: Arg.Flags.getNonZeroMemAlign());
3745 }
3746
3747 Val = convertABITypeToValueType(DAG, Val, VA, SL: DL);
3748 InVals.push_back(Elt: Val);
3749 }
3750
3751 // Start adding system SGPRs.
3752 if (IsEntryFunc)
3753 allocateSystemSGPRs(CCInfo, MF, Info&: *Info, CallConv, IsShader: IsGraphics);
3754
3755 unsigned StackArgSize = CCInfo.getStackSize();
3756 Info->setBytesInStackArgArea(StackArgSize);
3757
3758 return Chains.empty() ? Chain
3759 : DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
3760}
3761
3762// TODO: If return values can't fit in registers, we should return as many as
3763// possible in registers before passing on stack.
3764bool SITargetLowering::CanLowerReturn(
3765 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
3766 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
3767 const Type *RetTy) const {
3768 // Replacing returns with sret/stack usage doesn't make sense for shaders.
3769 // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
3770 // for shaders. Vector types should be explicitly handled by CC.
3771 if (AMDGPU::isEntryFunctionCC(CC: CallConv))
3772 return true;
3773
3774 SmallVector<CCValAssign, 16> RVLocs;
3775 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
3776 if (!CCInfo.CheckReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg)))
3777 return false;
3778
3779 // We must use the stack if return would require unavailable registers.
3780 unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
3781 unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
3782 for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
3783 if (CCInfo.isAllocated(Reg: AMDGPU::VGPR_32RegClass.getRegister(i)))
3784 return false;
3785
3786 return true;
3787}
3788
3789SDValue
3790SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3791 bool isVarArg,
3792 const SmallVectorImpl<ISD::OutputArg> &Outs,
3793 const SmallVectorImpl<SDValue> &OutVals,
3794 const SDLoc &DL, SelectionDAG &DAG) const {
3795 MachineFunction &MF = DAG.getMachineFunction();
3796 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
3797 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
3798
3799 if (AMDGPU::isKernel(CC: CallConv)) {
3800 return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
3801 OutVals, DL, DAG);
3802 }
3803
3804 bool IsShader = AMDGPU::isShader(CC: CallConv);
3805
3806 Info->setIfReturnsVoid(Outs.empty());
3807 bool IsWaveEnd = Info->returnsVoid() && IsShader;
3808
3809 // CCValAssign - represent the assignment of the return value to a location.
3810 SmallVector<CCValAssign, 48> RVLocs;
3811
3812 // CCState - Info about the registers and stack slots.
3813 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3814 *DAG.getContext());
3815
3816 // Analyze outgoing return values.
3817 CCInfo.AnalyzeReturn(Outs, Fn: CCAssignFnForReturn(CC: CallConv, IsVarArg: isVarArg));
3818
3819 SDValue Glue;
3820 SmallVector<SDValue, 48> RetOps;
3821 RetOps.push_back(Elt: Chain); // Operand #0 = Chain (updated below)
3822
3823 SDValue ReadFirstLane =
3824 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
3825 // Copy the result values into the output registers.
3826 for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
3827 ++I, ++RealRVLocIdx) {
3828 CCValAssign &VA = RVLocs[I];
3829 assert(VA.isRegLoc() && "Can only return in registers!");
3830 // TODO: Partially return in registers if return values don't fit.
3831 SDValue Arg = OutVals[RealRVLocIdx];
3832
3833 // Copied from other backends.
3834 switch (VA.getLocInfo()) {
3835 case CCValAssign::Full:
3836 break;
3837 case CCValAssign::BCvt:
3838 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
3839 break;
3840 case CCValAssign::SExt:
3841 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3842 break;
3843 case CCValAssign::ZExt:
3844 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3845 break;
3846 case CCValAssign::AExt:
3847 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
3848 break;
3849 default:
3850 llvm_unreachable("Unknown loc info!");
3851 }
3852 if (TRI->isSGPRPhysReg(Reg: VA.getLocReg()))
3853 Arg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Arg.getValueType(),
3854 N1: ReadFirstLane, N2: Arg);
3855 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Arg, Glue);
3856 Glue = Chain.getValue(R: 1);
3857 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
3858 }
3859
3860 // FIXME: Does sret work properly?
3861 if (!Info->isEntryFunction()) {
3862 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3863 const MCPhysReg *I =
3864 TRI->getCalleeSavedRegsViaCopy(MF: &DAG.getMachineFunction());
3865 if (I) {
3866 for (; *I; ++I) {
3867 if (AMDGPU::SReg_64RegClass.contains(Reg: *I))
3868 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i64));
3869 else if (AMDGPU::SReg_32RegClass.contains(Reg: *I))
3870 RetOps.push_back(Elt: DAG.getRegister(Reg: *I, VT: MVT::i32));
3871 else
3872 llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3873 }
3874 }
3875 }
3876
3877 // Update chain and glue.
3878 RetOps[0] = Chain;
3879 if (Glue.getNode())
3880 RetOps.push_back(Elt: Glue);
3881
3882 unsigned Opc = AMDGPUISD::ENDPGM;
3883 if (!IsWaveEnd)
3884 Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
3885 : IsShader ? AMDGPUISD::RETURN_TO_EPILOG
3886 : AMDGPUISD::RET_GLUE;
3887 return DAG.getNode(Opcode: Opc, DL, VT: MVT::Other, Ops: RetOps);
3888}
3889
3890SDValue SITargetLowering::LowerCallResult(
3891 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
3892 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3893 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
3894 SDValue ThisVal) const {
3895 CCAssignFn *RetCC = CCAssignFnForReturn(CC: CallConv, IsVarArg);
3896
3897 // Assign locations to each value returned by this call.
3898 SmallVector<CCValAssign, 16> RVLocs;
3899 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
3900 *DAG.getContext());
3901 CCInfo.AnalyzeCallResult(Ins, Fn: RetCC);
3902
3903 // Copy all of the result registers out of their specified physreg.
3904 for (CCValAssign VA : RVLocs) {
3905 SDValue Val;
3906
3907 if (VA.isRegLoc()) {
3908 Val =
3909 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
3910 Chain = Val.getValue(R: 1);
3911 InGlue = Val.getValue(R: 2);
3912 } else if (VA.isMemLoc()) {
3913 report_fatal_error(reason: "TODO: return values in memory");
3914 } else
3915 llvm_unreachable("unknown argument location type");
3916
3917 switch (VA.getLocInfo()) {
3918 case CCValAssign::Full:
3919 break;
3920 case CCValAssign::BCvt:
3921 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
3922 break;
3923 case CCValAssign::ZExt:
3924 Val = DAG.getNode(Opcode: ISD::AssertZext, DL, VT: VA.getLocVT(), N1: Val,
3925 N2: DAG.getValueType(VA.getValVT()));
3926 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3927 break;
3928 case CCValAssign::SExt:
3929 Val = DAG.getNode(Opcode: ISD::AssertSext, DL, VT: VA.getLocVT(), N1: Val,
3930 N2: DAG.getValueType(VA.getValVT()));
3931 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3932 break;
3933 case CCValAssign::AExt:
3934 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VA.getValVT(), Operand: Val);
3935 break;
3936 default:
3937 llvm_unreachable("Unknown loc info!");
3938 }
3939
3940 InVals.push_back(Elt: Val);
3941 }
3942
3943 return Chain;
3944}
3945
3946// Add code to pass special inputs required depending on used features separate
3947// from the explicit user arguments present in the IR.
3948void SITargetLowering::passSpecialInputs(
3949 CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info,
3950 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
3951 SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const {
3952 // If we don't have a call site, this was a call inserted by
3953 // legalization. These can never use special inputs.
3954 if (!CLI.CB)
3955 return;
3956
3957 SelectionDAG &DAG = CLI.DAG;
3958 const SDLoc &DL = CLI.DL;
3959 const Function &F = DAG.getMachineFunction().getFunction();
3960
3961 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
3962 const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
3963
3964 const AMDGPUFunctionArgInfo &CalleeArgInfo =
3965 AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
3966
3967 // TODO: Unify with private memory register handling. This is complicated by
3968 // the fact that at least in kernels, the input argument is not necessarily
3969 // in the same location as the input.
3970 // clang-format off
3971 static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
3972 std::array<StringLiteral, 2>> ImplicitAttrs[] = {
3973 {AMDGPUFunctionArgInfo::DISPATCH_PTR, {"amdgpu-no-dispatch-ptr", ""}},
3974 {AMDGPUFunctionArgInfo::QUEUE_PTR, {"amdgpu-no-queue-ptr", ""}},
3975 {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, {"amdgpu-no-implicitarg-ptr", ""}},
3976 {AMDGPUFunctionArgInfo::DISPATCH_ID, {"amdgpu-no-dispatch-id", ""}},
3977 {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"}},
3978 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"}},
3979 {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"}},
3980 {AMDGPUFunctionArgInfo::LDS_KERNEL_ID, {"amdgpu-no-lds-kernel-id", ""}},
3981 };
3982 // clang-format on
3983
3984 for (auto [InputID, Attrs] : ImplicitAttrs) {
3985 // If the callee does not use the attribute value, skip copying the value.
3986 if (all_of(Range&: Attrs, P: [&](StringRef Attr) {
3987 return Attr.empty() || CLI.CB->hasFnAttr(Kind: Attr);
3988 }))
3989 continue;
3990
3991 const auto [OutgoingArg, ArgRC, ArgTy] =
3992 CalleeArgInfo.getPreloadedValue(Value: InputID);
3993 if (!OutgoingArg)
3994 continue;
3995
3996 const auto [IncomingArg, IncomingArgRC, Ty] =
3997 CallerArgInfo.getPreloadedValue(Value: InputID);
3998 assert(IncomingArgRC == ArgRC);
3999
4000 // All special arguments are ints for now.
4001 EVT ArgVT = TRI->getSpillSize(RC: *ArgRC) == 8 ? MVT::i64 : MVT::i32;
4002 SDValue InputReg;
4003
4004 if (IncomingArg) {
4005 InputReg = loadInputValue(DAG, RC: ArgRC, VT: ArgVT, SL: DL, Arg: *IncomingArg);
4006 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
4007 // The implicit arg ptr is special because it doesn't have a corresponding
4008 // input for kernels, and is computed from the kernarg segment pointer.
4009 InputReg = getImplicitArgPtr(DAG, SL: DL);
4010 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
4011 std::optional<uint32_t> Id =
4012 AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
4013 if (Id.has_value()) {
4014 InputReg = DAG.getConstant(Val: *Id, DL, VT: ArgVT);
4015 } else {
4016 InputReg = DAG.getPOISON(VT: ArgVT);
4017 }
4018 } else {
4019 // We may have proven the input wasn't needed, although the ABI is
4020 // requiring it. We just need to allocate the register appropriately.
4021 InputReg = DAG.getPOISON(VT: ArgVT);
4022 }
4023
4024 if (OutgoingArg->isRegister()) {
4025 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4026 if (!CCInfo.AllocateReg(Reg: OutgoingArg->getRegister()))
4027 report_fatal_error(reason: "failed to allocate implicit input argument");
4028 } else {
4029 unsigned SpecialArgOffset =
4030 CCInfo.AllocateStack(Size: ArgVT.getStoreSize(), Alignment: Align(4));
4031 SDValue ArgStore =
4032 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4033 MemOpChains.push_back(Elt: ArgStore);
4034 }
4035 }
4036
4037 // Pack workitem IDs into a single register or pass it as is if already
4038 // packed.
4039
4040 auto [OutgoingArg, ArgRC, Ty] =
4041 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X);
4042 if (!OutgoingArg)
4043 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4044 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
4045 if (!OutgoingArg)
4046 std::tie(args&: OutgoingArg, args&: ArgRC, args&: Ty) =
4047 CalleeArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
4048 if (!OutgoingArg)
4049 return;
4050
4051 const ArgDescriptor *IncomingArgX = std::get<0>(
4052 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_X));
4053 const ArgDescriptor *IncomingArgY = std::get<0>(
4054 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
4055 const ArgDescriptor *IncomingArgZ = std::get<0>(
4056 t: CallerArgInfo.getPreloadedValue(Value: AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
4057
4058 SDValue InputReg;
4059 SDLoc SL;
4060
4061 const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-x");
4062 const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-y");
4063 const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr(Kind: "amdgpu-no-workitem-id-z");
4064
4065 // If incoming ids are not packed we need to pack them.
4066 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
4067 NeedWorkItemIDX) {
4068 if (Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 0) != 0) {
4069 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgX);
4070 } else {
4071 InputReg = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
4072 }
4073 }
4074
4075 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
4076 NeedWorkItemIDY && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 1) != 0) {
4077 SDValue Y = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgY);
4078 Y = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Y,
4079 N2: DAG.getShiftAmountConstant(Val: 10, VT: MVT::i32, DL: SL));
4080 InputReg = InputReg.getNode()
4081 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Y)
4082 : Y;
4083 }
4084
4085 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
4086 NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(Kernel: F, Dimension: 2) != 0) {
4087 SDValue Z = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: *IncomingArgZ);
4088 Z = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Z,
4089 N2: DAG.getShiftAmountConstant(Val: 20, VT: MVT::i32, DL: SL));
4090 InputReg = InputReg.getNode()
4091 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: InputReg, N2: Z)
4092 : Z;
4093 }
4094
4095 if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
4096 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
4097 // We're in a situation where the outgoing function requires the workitem
4098 // ID, but the calling function does not have it (e.g a graphics function
4099 // calling a C calling convention function). This is illegal, but we need
4100 // to produce something.
4101 InputReg = DAG.getPOISON(VT: MVT::i32);
4102 } else {
4103 // Workitem ids are already packed, any of present incoming arguments
4104 // will carry all required fields.
4105 ArgDescriptor IncomingArg =
4106 ArgDescriptor::createArg(Arg: IncomingArgX ? *IncomingArgX
4107 : IncomingArgY ? *IncomingArgY
4108 : *IncomingArgZ,
4109 Mask: ~0u);
4110 InputReg = loadInputValue(DAG, RC: ArgRC, VT: MVT::i32, SL: DL, Arg: IncomingArg);
4111 }
4112 }
4113
4114 if (OutgoingArg->isRegister()) {
4115 if (InputReg)
4116 RegsToPass.emplace_back(Args: OutgoingArg->getRegister(), Args&: InputReg);
4117
4118 CCInfo.AllocateReg(Reg: OutgoingArg->getRegister());
4119 } else {
4120 unsigned SpecialArgOffset = CCInfo.AllocateStack(Size: 4, Alignment: Align(4));
4121 if (InputReg) {
4122 SDValue ArgStore =
4123 storeStackInputValue(DAG, SL: DL, Chain, ArgVal: InputReg, Offset: SpecialArgOffset);
4124 MemOpChains.push_back(Elt: ArgStore);
4125 }
4126 }
4127}
4128
4129bool SITargetLowering::isEligibleForTailCallOptimization(
4130 SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
4131 const SmallVectorImpl<ISD::OutputArg> &Outs,
4132 const SmallVectorImpl<SDValue> &OutVals,
4133 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4134 if (AMDGPU::isChainCC(CC: CalleeCC))
4135 return true;
4136
4137 if (!AMDGPU::mayTailCallThisCC(CC: CalleeCC))
4138 return false;
4139
4140 // For a divergent call target, we need to do a waterfall loop over the
4141 // possible callees which precludes us from using a simple jump.
4142 if (Callee->isDivergent())
4143 return false;
4144
4145 MachineFunction &MF = DAG.getMachineFunction();
4146 const Function &CallerF = MF.getFunction();
4147 CallingConv::ID CallerCC = CallerF.getCallingConv();
4148 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
4149 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
4150
4151 // Kernels aren't callable, and don't have a live in return address so it
4152 // doesn't make sense to do a tail call with entry functions.
4153 if (!CallerPreserved)
4154 return false;
4155
4156 bool CCMatch = CallerCC == CalleeCC;
4157
4158 if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
4159 if (AMDGPU::canGuaranteeTCO(CC: CalleeCC) && CCMatch)
4160 return true;
4161 return false;
4162 }
4163
4164 // TODO: Can we handle var args?
4165 if (IsVarArg)
4166 return false;
4167
4168 for (const Argument &Arg : CallerF.args()) {
4169 if (Arg.hasByValAttr())
4170 return false;
4171 }
4172
4173 LLVMContext &Ctx = *DAG.getContext();
4174
4175 // Check that the call results are passed in the same way.
4176 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C&: Ctx, Ins,
4177 CalleeFn: CCAssignFnForCall(CC: CalleeCC, IsVarArg),
4178 CallerFn: CCAssignFnForCall(CC: CallerCC, IsVarArg)))
4179 return false;
4180
4181 // The callee has to preserve all registers the caller needs to preserve.
4182 if (!CCMatch) {
4183 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
4184 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
4185 return false;
4186 }
4187
4188 // Nothing more to check if the callee is taking no arguments.
4189 if (Outs.empty())
4190 return true;
4191
4192 SmallVector<CCValAssign, 16> ArgLocs;
4193 CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
4194
4195 // FIXME: We are not allocating special input registers, so we will be
4196 // deciding based on incorrect register assignments.
4197 CCInfo.AnalyzeCallOperands(Outs, Fn: CCAssignFnForCall(CC: CalleeCC, IsVarArg));
4198
4199 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
4200 // If the stack arguments for this call do not fit into our own save area then
4201 // the call cannot be made tail.
4202 // TODO: Is this really necessary?
4203 if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
4204 return false;
4205
4206 for (const auto &[CCVA, ArgVal] : zip_equal(t&: ArgLocs, u: OutVals)) {
4207 // FIXME: What about inreg arguments that end up passed in memory?
4208 if (!CCVA.isRegLoc())
4209 continue;
4210
4211 // If we are passing an argument in an SGPR, and the value is divergent,
4212 // this call requires a waterfall loop.
4213 if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(Reg: CCVA.getLocReg())) {
4214 LLVM_DEBUG(
4215 dbgs() << "Cannot tail call due to divergent outgoing argument in "
4216 << printReg(CCVA.getLocReg(), TRI) << '\n');
4217 return false;
4218 }
4219 }
4220
4221 const MachineRegisterInfo &MRI = MF.getRegInfo();
4222 return parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals);
4223}
4224
4225bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
4226 if (!CI->isTailCall())
4227 return false;
4228
4229 const Function *ParentFn = CI->getFunction();
4230 if (AMDGPU::isEntryFunctionCC(CC: ParentFn->getCallingConv()))
4231 return false;
4232 return true;
4233}
4234
4235namespace {
4236// Chain calls have special arguments that we need to handle. These are
4237// tagging along at the end of the arguments list(s), after the SGPR and VGPR
4238// arguments (index 0 and 1 respectively).
4239enum ChainCallArgIdx {
4240 Exec = 2,
4241 Flags,
4242 NumVGPRs,
4243 FallbackExec,
4244 FallbackCallee
4245};
4246} // anonymous namespace
4247
4248// The wave scratch offset register is used as the global base pointer.
4249SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
4250 SmallVectorImpl<SDValue> &InVals) const {
4251 CallingConv::ID CallConv = CLI.CallConv;
4252 bool IsChainCallConv = AMDGPU::isChainCC(CC: CallConv);
4253
4254 SelectionDAG &DAG = CLI.DAG;
4255
4256 const SDLoc &DL = CLI.DL;
4257 SDValue Chain = CLI.Chain;
4258 SDValue Callee = CLI.Callee;
4259
4260 llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
4261 bool UsesDynamicVGPRs = false;
4262 if (IsChainCallConv) {
4263 // The last arguments should be the value that we need to put in EXEC,
4264 // followed by the flags and any other arguments with special meanings.
4265 // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so
4266 // we don't treat them like the "real" arguments.
4267 auto RequestedExecIt =
4268 llvm::find_if(Range&: CLI.Outs, P: [](const ISD::OutputArg &Arg) {
4269 return Arg.OrigArgIndex == 2;
4270 });
4271 assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC");
4272
4273 size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin();
4274 CLI.OutVals.erase(CS: CLI.OutVals.begin() + SpecialArgsBeginIdx,
4275 CE: CLI.OutVals.end());
4276 CLI.Outs.erase(CS: RequestedExecIt, CE: CLI.Outs.end());
4277
4278 assert(CLI.Outs.back().OrigArgIndex < 2 &&
4279 "Haven't popped all the special args");
4280
4281 TargetLowering::ArgListEntry RequestedExecArg =
4282 CLI.Args[ChainCallArgIdx::Exec];
4283 if (!RequestedExecArg.Ty->isIntegerTy(BitWidth: Subtarget->getWavefrontSize()))
4284 return lowerUnhandledCall(CLI, InVals, Reason: "Invalid value for EXEC");
4285
4286 // Convert constants into TargetConstants, so they become immediate operands
4287 // instead of being selected into S_MOV.
4288 auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) {
4289 if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Val&: Arg.Node)) {
4290 ChainCallSpecialArgs.push_back(Elt: DAG.getTargetConstant(
4291 Val: ArgNode->getAPIntValue(), DL, VT: ArgNode->getValueType(ResNo: 0)));
4292 } else
4293 ChainCallSpecialArgs.push_back(Elt: Arg.Node);
4294 };
4295
4296 PushNodeOrTargetConstant(RequestedExecArg);
4297
4298 // Process any other special arguments depending on the value of the flags.
4299 TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags];
4300
4301 const APInt &FlagsValue = cast<ConstantSDNode>(Val&: Flags.Node)->getAPIntValue();
4302 if (FlagsValue.isZero()) {
4303 if (CLI.Args.size() > ChainCallArgIdx::Flags + 1)
4304 return lowerUnhandledCall(CLI, InVals,
4305 Reason: "no additional args allowed if flags == 0");
4306 } else if (FlagsValue.isOneBitSet(BitNo: 0)) {
4307 if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
4308 return lowerUnhandledCall(CLI, InVals, Reason: "expected 3 additional args");
4309 }
4310
4311 if (!Subtarget->isWave32()) {
4312 return lowerUnhandledCall(
4313 CLI, InVals, Reason: "dynamic VGPR mode is only supported for wave32");
4314 }
4315
4316 UsesDynamicVGPRs = true;
4317 std::for_each(first: CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
4318 last: CLI.Args.end(), f: PushNodeOrTargetConstant);
4319 }
4320 }
4321
4322 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
4323 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
4324 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
4325 bool &IsTailCall = CLI.IsTailCall;
4326 bool IsVarArg = CLI.IsVarArg;
4327 bool IsSibCall = false;
4328 MachineFunction &MF = DAG.getMachineFunction();
4329
4330 if (Callee.isUndef() || isNullConstant(V: Callee)) {
4331 if (!CLI.IsTailCall) {
4332 for (ISD::InputArg &Arg : CLI.Ins)
4333 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
4334 }
4335
4336 return Chain;
4337 }
4338
4339 if (IsVarArg) {
4340 return lowerUnhandledCall(CLI, InVals,
4341 Reason: "unsupported call to variadic function ");
4342 }
4343
4344 if (!CLI.CB)
4345 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported libcall legalization");
4346
4347 if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
4348 return lowerUnhandledCall(CLI, InVals,
4349 Reason: "unsupported required tail call to function ");
4350 }
4351
4352 if (IsTailCall) {
4353 IsTailCall = isEligibleForTailCallOptimization(Callee, CalleeCC: CallConv, IsVarArg,
4354 Outs, OutVals, Ins, DAG);
4355 if (!IsTailCall &&
4356 ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
4357 report_fatal_error(reason: "failed to perform tail call elimination on a call "
4358 "site marked musttail or on llvm.amdgcn.cs.chain");
4359 }
4360
4361 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
4362
4363 // A sibling call is one where we're under the usual C ABI and not planning
4364 // to change that but can still do a tail call:
4365 if (!TailCallOpt && IsTailCall)
4366 IsSibCall = true;
4367
4368 if (IsTailCall)
4369 ++NumTailCalls;
4370 }
4371
4372 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4373 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
4374 SmallVector<SDValue, 8> MemOpChains;
4375
4376 // Analyze operands of the call, assigning locations to each operand.
4377 SmallVector<CCValAssign, 16> ArgLocs;
4378 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
4379 CCAssignFn *AssignFn = CCAssignFnForCall(CC: CallConv, IsVarArg);
4380
4381 if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CC: CallConv) &&
4382 CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {
4383 // With a fixed ABI, allocate fixed registers before user arguments.
4384 passSpecialInputs(CLI, CCInfo, Info: *Info, RegsToPass, MemOpChains, Chain);
4385 }
4386
4387 // Mark the scratch resource descriptor as allocated so the CC analysis
4388 // does not assign user arguments to these registers, matching the callee.
4389 if (!Subtarget->hasFlatScratchEnabled())
4390 CCInfo.AllocateReg(Reg: Info->getScratchRSrcReg());
4391
4392 CCInfo.AnalyzeCallOperands(Outs, Fn: AssignFn);
4393
4394 // Get a count of how many bytes are to be pushed on the stack.
4395 unsigned NumBytes = CCInfo.getStackSize();
4396
4397 if (IsSibCall) {
4398 // Since we're not changing the ABI to make this a tail call, the memory
4399 // operands are already available in the caller's incoming argument space.
4400 NumBytes = 0;
4401 }
4402
4403 // FPDiff is the byte offset of the call's argument area from the callee's.
4404 // Stores to callee stack arguments will be placed in FixedStackSlots offset
4405 // by this amount for a tail call. In a sibling call it must be 0 because the
4406 // caller will deallocate the entire stack and the callee still expects its
4407 // arguments to begin at SP+0. Completely unused for non-tail calls.
4408 int32_t FPDiff = 0;
4409 MachineFrameInfo &MFI = MF.getFrameInfo();
4410 auto *TRI = Subtarget->getRegisterInfo();
4411
4412 // Adjust the stack pointer for the new arguments...
4413 // These operations are automatically eliminated by the prolog/epilog pass
4414 if (!IsSibCall)
4415 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL);
4416
4417 if (!IsSibCall || IsChainCallConv) {
4418 if (!Subtarget->hasFlatScratchEnabled()) {
4419 SmallVector<SDValue, 4> CopyFromChains;
4420
4421 // In the HSA case, this should be an identity copy.
4422 SDValue ScratchRSrcReg =
4423 DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getScratchRSrcReg(), VT: MVT::v4i32);
4424 RegsToPass.emplace_back(Args: IsChainCallConv
4425 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
4426 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3,
4427 Args&: ScratchRSrcReg);
4428 CopyFromChains.push_back(Elt: ScratchRSrcReg.getValue(R: 1));
4429 Chain = DAG.getTokenFactor(DL, Vals&: CopyFromChains);
4430 }
4431 }
4432
4433 const unsigned NumSpecialInputs = RegsToPass.size();
4434
4435 MVT PtrVT = MVT::i32;
4436
4437 // Walk the register/memloc assignments, inserting copies/loads.
4438 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4439 CCValAssign &VA = ArgLocs[i];
4440 SDValue Arg = OutVals[i];
4441
4442 // Promote the value if needed.
4443 switch (VA.getLocInfo()) {
4444 case CCValAssign::Full:
4445 break;
4446 case CCValAssign::BCvt:
4447 Arg = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getLocVT(), Operand: Arg);
4448 break;
4449 case CCValAssign::ZExt:
4450 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4451 break;
4452 case CCValAssign::SExt:
4453 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4454 break;
4455 case CCValAssign::AExt:
4456 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4457 break;
4458 case CCValAssign::FPExt:
4459 Arg = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: VA.getLocVT(), Operand: Arg);
4460 break;
4461 default:
4462 llvm_unreachable("Unknown loc info!");
4463 }
4464
4465 if (VA.isRegLoc()) {
4466 RegsToPass.push_back(Elt: std::pair(VA.getLocReg(), Arg));
4467 } else {
4468 assert(VA.isMemLoc());
4469
4470 SDValue DstAddr;
4471 MachinePointerInfo DstInfo;
4472
4473 unsigned LocMemOffset = VA.getLocMemOffset();
4474 int32_t Offset = LocMemOffset;
4475
4476 SDValue PtrOff = DAG.getConstant(Val: Offset, DL, VT: PtrVT);
4477 MaybeAlign Alignment;
4478
4479 if (IsTailCall) {
4480 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4481 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
4482 : VA.getValVT().getStoreSize();
4483
4484 // FIXME: We can have better than the minimum byval required alignment.
4485 Alignment =
4486 Flags.isByVal()
4487 ? Flags.getNonZeroByValAlign()
4488 : commonAlignment(A: Subtarget->getStackAlignment(), Offset);
4489
4490 Offset = Offset + FPDiff;
4491 int FI = MFI.CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
4492
4493 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
4494 DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
4495
4496 // Make sure any stack arguments overlapping with where we're storing
4497 // are loaded before this eventual operation. Otherwise they'll be
4498 // clobbered.
4499
4500 // FIXME: Why is this really necessary? This seems to just result in a
4501 // lot of code to copy the stack and write them back to the same
4502 // locations, which are supposed to be immutable?
4503 Chain = addTokenForArgument(Chain, DAG, MFI, ClobberedFI: FI);
4504 } else {
4505 // Stores to the argument stack area are relative to the stack pointer.
4506 SDValue SP = DAG.getCopyFromReg(Chain, dl: DL, Reg: Info->getStackPtrOffsetReg(),
4507 VT: MVT::i32);
4508 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SP, N2: PtrOff);
4509 DstInfo = MachinePointerInfo::getStack(MF, Offset: LocMemOffset);
4510 Alignment =
4511 commonAlignment(A: Subtarget->getStackAlignment(), Offset: LocMemOffset);
4512 }
4513
4514 if (Outs[i].Flags.isByVal()) {
4515 SDValue SizeNode =
4516 DAG.getConstant(Val: Outs[i].Flags.getByValSize(), DL, VT: MVT::i32);
4517 SDValue Cpy =
4518 DAG.getMemcpy(Chain, dl: DL, Dst: DstAddr, Src: Arg, Size: SizeNode,
4519 DstAlign: Outs[i].Flags.getNonZeroByValAlign(),
4520 SrcAlign: Outs[i].Flags.getNonZeroByValAlign(),
4521 /*isVol = */ false, /*AlwaysInline = */ true,
4522 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: DstInfo,
4523 SrcPtrInfo: MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
4524
4525 MemOpChains.push_back(Elt: Cpy);
4526 } else {
4527 SDValue Store =
4528 DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: DstAddr, PtrInfo: DstInfo, Alignment);
4529 MemOpChains.push_back(Elt: Store);
4530 }
4531 }
4532 }
4533
4534 if (!MemOpChains.empty())
4535 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
4536
4537 SDValue ReadFirstLaneID =
4538 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4539
4540 SDValue TokenGlue;
4541 if (CLI.ConvergenceControlToken) {
4542 TokenGlue = DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL, VT: MVT::Glue,
4543 Operand: CLI.ConvergenceControlToken);
4544 }
4545
4546 // Build a sequence of copy-to-reg nodes chained together with token chain
4547 // and flag operands which copy the outgoing args into the appropriate regs.
4548 SDValue InGlue;
4549
4550 unsigned ArgIdx = 0;
4551 for (auto [Reg, Val] : RegsToPass) {
4552 if (ArgIdx++ >= NumSpecialInputs &&
4553 (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
4554 // For chain calls, the inreg arguments are required to be
4555 // uniform. Speculatively Insert a readfirstlane in case we cannot prove
4556 // they are uniform.
4557 //
4558 // For other calls, if an inreg arguments is known to be uniform,
4559 // speculatively insert a readfirstlane in case it is in a VGPR.
4560 //
4561 // FIXME: We need to execute this in a waterfall loop if it is a divergent
4562 // value, so let that continue to produce invalid code.
4563
4564 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
4565 if (TokenGlue)
4566 ReadfirstlaneArgs.push_back(Elt: TokenGlue);
4567 Val = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Val.getValueType(),
4568 Ops: ReadfirstlaneArgs);
4569 }
4570
4571 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg, N: Val, Glue: InGlue);
4572 InGlue = Chain.getValue(R: 1);
4573 }
4574
4575 // We don't usually want to end the call-sequence here because we would tidy
4576 // the frame up *after* the call, however in the ABI-changing tail-call case
4577 // we've carefully laid out the parameters so that when sp is reset they'll be
4578 // in the correct location.
4579 if (IsTailCall && !IsSibCall) {
4580 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL);
4581 InGlue = Chain.getValue(R: 1);
4582 }
4583
4584 std::vector<SDValue> Ops({Chain});
4585
4586 // Add a redundant copy of the callee global which will not be legalized, as
4587 // we need direct access to the callee later.
4588 if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
4589 const GlobalValue *GV = GSD->getGlobal();
4590 Ops.push_back(x: Callee);
4591 Ops.push_back(x: DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64));
4592 } else {
4593 if (IsTailCall) {
4594 // isEligibleForTailCallOptimization considered whether the call target is
4595 // divergent, but we may still end up with a uniform value in a VGPR.
4596 // Insert a readfirstlane just in case.
4597 SDValue ReadFirstLaneID =
4598 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
4599
4600 SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
4601 if (TokenGlue)
4602 ReadfirstlaneArgs.push_back(Elt: TokenGlue); // Wire up convergence token.
4603 Callee = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: Callee.getValueType(),
4604 Ops: ReadfirstlaneArgs);
4605 }
4606
4607 Ops.push_back(x: Callee);
4608 Ops.push_back(x: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i64));
4609 }
4610
4611 if (IsTailCall) {
4612 // Each tail call may have to adjust the stack by a different amount, so
4613 // this information must travel along with the operation for eventual
4614 // consumption by emitEpilogue.
4615 Ops.push_back(x: DAG.getTargetConstant(Val: FPDiff, DL, VT: MVT::i32));
4616 }
4617
4618 if (IsChainCallConv)
4619 llvm::append_range(C&: Ops, R&: ChainCallSpecialArgs);
4620
4621 // Add argument registers to the end of the list so that they are known live
4622 // into the call.
4623 for (auto &[Reg, Val] : RegsToPass)
4624 Ops.push_back(x: DAG.getRegister(Reg, VT: Val.getValueType()));
4625
4626 // Add a register mask operand representing the call-preserved registers.
4627 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
4628 assert(Mask && "Missing call preserved mask for calling convention");
4629 Ops.push_back(x: DAG.getRegisterMask(RegMask: Mask));
4630
4631 if (SDValue Token = CLI.ConvergenceControlToken) {
4632 SmallVector<SDValue, 2> GlueOps;
4633 GlueOps.push_back(Elt: Token);
4634 if (InGlue)
4635 GlueOps.push_back(Elt: InGlue);
4636
4637 InGlue = SDValue(DAG.getMachineNode(Opcode: TargetOpcode::CONVERGENCECTRL_GLUE, dl: DL,
4638 VT: MVT::Glue, Ops: GlueOps),
4639 0);
4640 }
4641
4642 if (InGlue)
4643 Ops.push_back(x: InGlue);
4644
4645 // If we're doing a tall call, use a TC_RETURN here rather than an
4646 // actual call instruction.
4647 if (IsTailCall) {
4648 MFI.setHasTailCall();
4649 unsigned OPC = AMDGPUISD::TC_RETURN;
4650 switch (CallConv) {
4651 case CallingConv::AMDGPU_Gfx:
4652 OPC = AMDGPUISD::TC_RETURN_GFX;
4653 break;
4654 case CallingConv::AMDGPU_CS_Chain:
4655 case CallingConv::AMDGPU_CS_ChainPreserve:
4656 OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
4657 : AMDGPUISD::TC_RETURN_CHAIN;
4658 break;
4659 }
4660
4661 // If the caller is a whole wave function, we need to use a special opcode
4662 // so we can patch up EXEC.
4663 if (Info->isWholeWaveFunction())
4664 OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
4665
4666 return DAG.getNode(Opcode: OPC, DL, VT: MVT::Other, Ops);
4667 }
4668
4669 // Returns a chain and a flag for retval copy to use.
4670 SDValue Call = DAG.getNode(Opcode: AMDGPUISD::CALL, DL, ResultTys: {MVT::Other, MVT::Glue}, Ops);
4671 Chain = Call.getValue(R: 0);
4672 InGlue = Call.getValue(R: 1);
4673
4674 uint64_t CalleePopBytes = NumBytes;
4675 Chain = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: CalleePopBytes, Glue: InGlue, DL);
4676 if (!Ins.empty())
4677 InGlue = Chain.getValue(R: 1);
4678
4679 // Handle result values, copying them out of physregs into vregs that we
4680 // return.
4681 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
4682 InVals, /*IsThisReturn=*/false, ThisVal: SDValue());
4683}
4684
4685// This is similar to the default implementation in ExpandDYNAMIC_STACKALLOC,
4686// except for:
4687// 1. Stack growth direction(default: downwards, AMDGPU: upwards), and
4688// 2. Scale size where, scale = wave-reduction(alloca-size) * wave-size
4689SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
4690 SelectionDAG &DAG) const {
4691 const MachineFunction &MF = DAG.getMachineFunction();
4692 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4693
4694 SDLoc dl(Op);
4695 EVT VT = Op.getValueType();
4696 SDValue Chain = Op.getOperand(i: 0);
4697 Register SPReg = Info->getStackPtrOffsetReg();
4698
4699 // Chain the dynamic stack allocation so that it doesn't modify the stack
4700 // pointer when other instructions are using the stack.
4701 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
4702
4703 SDValue Size = Op.getOperand(i: 1);
4704 SDValue BaseAddr = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
4705 Align Alignment = cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getAlignValue();
4706
4707 const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
4708 assert(TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
4709 "Stack grows upwards for AMDGPU");
4710
4711 Chain = BaseAddr.getValue(R: 1);
4712 // When using flat-scratch, the stack offset is unscaled.
4713 const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
4714 const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
4715
4716 Align StackAlign = TFL->getStackAlign();
4717 if (Alignment > StackAlign) {
4718 uint64_t ScaledAlignment = Alignment.value()
4719 << (HasFlatScratch ? 0 : WavefrontSizeLog2);
4720 uint64_t StackAlignMask = ScaledAlignment - 1;
4721 SDValue TmpAddr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr,
4722 N2: DAG.getConstant(Val: StackAlignMask, DL: dl, VT));
4723 BaseAddr = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TmpAddr,
4724 N2: DAG.getSignedConstant(Val: -ScaledAlignment, DL: dl, VT));
4725 }
4726
4727 assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
4728 SDValue NewSP;
4729 if (isa<ConstantSDNode>(Val: Size)) {
4730 // Increase the stack pointer by the size of the alloca.
4731 // If not using flat-scratch, we have to scale the size by the wave-size.
4732 SDValue ScaledSize =
4733 HasFlatScratch
4734 ? Size
4735 : DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4736 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4737 NewSP = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value
4738 } else {
4739 // For dynamic sized alloca, perform wave-wide reduction to get max of
4740 // alloca size(divergent), and then scale it (when not using flat-scratch)
4741 // by wave-size.
4742 SDValue WaveReduction =
4743 DAG.getTargetConstant(Val: Intrinsic::amdgcn_wave_reduce_umax, DL: dl, VT: MVT::i32);
4744 Size = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: WaveReduction,
4745 N2: Size, N3: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32));
4746 SDValue ScaledSize = Size;
4747 if (!HasFlatScratch) {
4748 ScaledSize =
4749 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Size,
4750 N2: DAG.getConstant(Val: WavefrontSizeLog2, DL: dl, VT: MVT::i32));
4751 }
4752 NewSP =
4753 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: BaseAddr, N2: ScaledSize); // Value in vgpr.
4754 SDValue ReadFirstLaneID =
4755 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: dl, VT: MVT::i32);
4756 NewSP = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::i32, N1: ReadFirstLaneID,
4757 N2: NewSP);
4758 }
4759
4760 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: NewSP); // Output chain
4761 SDValue CallSeqEnd = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
4762
4763 return DAG.getMergeValues(Ops: {BaseAddr, CallSeqEnd}, dl);
4764}
4765
4766SDValue SITargetLowering::LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const {
4767 if (Op.getValueType() != MVT::i32)
4768 return Op; // Defer to cannot select error.
4769
4770 Register SP = getStackPointerRegisterToSaveRestore();
4771 SDLoc SL(Op);
4772
4773 SDValue CopyFromSP = DAG.getCopyFromReg(Chain: Op->getOperand(Num: 0), dl: SL, Reg: SP, VT: MVT::i32);
4774
4775 // Convert from wave uniform to swizzled vector address. This should protect
4776 // from any edge cases where the stacksave result isn't directly used with
4777 // stackrestore.
4778 SDValue VectorAddress =
4779 DAG.getNode(Opcode: AMDGPUISD::WAVE_ADDRESS, DL: SL, VT: MVT::i32, Operand: CopyFromSP);
4780 return DAG.getMergeValues(Ops: {VectorAddress, CopyFromSP.getValue(R: 1)}, dl: SL);
4781}
4782
4783SDValue SITargetLowering::lowerGET_ROUNDING(SDValue Op,
4784 SelectionDAG &DAG) const {
4785 SDLoc SL(Op);
4786 assert(Op.getValueType() == MVT::i32);
4787
4788 uint32_t BothRoundHwReg =
4789 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4790 SDValue GetRoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4791
4792 SDValue IntrinID =
4793 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
4794 SDValue GetReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList: Op->getVTList(),
4795 N1: Op.getOperand(i: 0), N2: IntrinID, N3: GetRoundBothImm);
4796
4797 // There are two rounding modes, one for f32 and one for f64/f16. We only
4798 // report in the standard value range if both are the same.
4799 //
4800 // The raw values also differ from the expected FLT_ROUNDS values. Nearest
4801 // ties away from zero is not supported, and the other values are rotated by
4802 // 1.
4803 //
4804 // If the two rounding modes are not the same, report a target defined value.
4805
4806 // Mode register rounding mode fields:
4807 //
4808 // [1:0] Single-precision round mode.
4809 // [3:2] Double/Half-precision round mode.
4810 //
4811 // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
4812 //
4813 // Hardware Spec
4814 // Toward-0 3 0
4815 // Nearest Even 0 1
4816 // +Inf 1 2
4817 // -Inf 2 3
4818 // NearestAway0 N/A 4
4819 //
4820 // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
4821 // table we can index by the raw hardware mode.
4822 //
4823 // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
4824
4825 SDValue BitTable =
4826 DAG.getConstant(Val: AMDGPU::FltRoundConversionTable, DL: SL, VT: MVT::i64);
4827
4828 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4829 SDValue RoundModeTimesNumBits =
4830 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: GetReg, N2: Two);
4831
4832 // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
4833 // knew only one mode was demanded.
4834 SDValue TableValue =
4835 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4836 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4837
4838 SDValue EntryMask = DAG.getConstant(Val: 0xf, DL: SL, VT: MVT::i32);
4839 SDValue TableEntry =
4840 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TruncTable, N2: EntryMask);
4841
4842 // There's a gap in the 4-bit encoded table and actual enum values, so offset
4843 // if it's an extended value.
4844 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4845 SDValue IsStandardValue =
4846 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: TableEntry, RHS: Four, Cond: ISD::SETULT);
4847 SDValue EnumOffset = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: TableEntry, N2: Four);
4848 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: IsStandardValue,
4849 N2: TableEntry, N3: EnumOffset);
4850
4851 return DAG.getMergeValues(Ops: {Result, GetReg.getValue(R: 1)}, dl: SL);
4852}
4853
4854SDValue SITargetLowering::lowerSET_ROUNDING(SDValue Op,
4855 SelectionDAG &DAG) const {
4856 SDLoc SL(Op);
4857
4858 SDValue NewMode = Op.getOperand(i: 1);
4859 assert(NewMode.getValueType() == MVT::i32);
4860
4861 // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
4862 // hardware MODE.fp_round values.
4863 if (auto *ConstMode = dyn_cast<ConstantSDNode>(Val&: NewMode)) {
4864 uint32_t ClampedVal = std::min(
4865 a: static_cast<uint32_t>(ConstMode->getZExtValue()),
4866 b: static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
4867 NewMode = DAG.getConstant(
4868 Val: AMDGPU::decodeFltRoundToHWConversionTable(FltRounds: ClampedVal), DL: SL, VT: MVT::i32);
4869 } else {
4870 // If we know the input can only be one of the supported standard modes in
4871 // the range 0-3, we can use a simplified mapping to hardware values.
4872 KnownBits KB = DAG.computeKnownBits(Op: NewMode);
4873 const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
4874 // The supported standard values are 0-3. The extended values start at 8. We
4875 // need to offset by 4 if the value is in the extended range.
4876
4877 if (UseReducedTable) {
4878 // Truncate to the low 32-bits.
4879 SDValue BitTable = DAG.getConstant(
4880 Val: AMDGPU::FltRoundToHWConversionTable & 0xffff, DL: SL, VT: MVT::i32);
4881
4882 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4883 SDValue RoundModeTimesNumBits =
4884 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewMode, N2: Two);
4885
4886 NewMode =
4887 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: BitTable, N2: RoundModeTimesNumBits);
4888
4889 // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
4890 // the table extracted bits into inline immediates.
4891 } else {
4892 // table_index = umin(value, value - 4)
4893 // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
4894 SDValue BitTable =
4895 DAG.getConstant(Val: AMDGPU::FltRoundToHWConversionTable, DL: SL, VT: MVT::i64);
4896
4897 SDValue Four = DAG.getConstant(Val: 4, DL: SL, VT: MVT::i32);
4898 SDValue OffsetEnum = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewMode, N2: Four);
4899 SDValue IndexVal =
4900 DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewMode, N2: OffsetEnum);
4901
4902 SDValue Two = DAG.getConstant(Val: 2, DL: SL, VT: MVT::i32);
4903 SDValue RoundModeTimesNumBits =
4904 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: IndexVal, N2: Two);
4905
4906 SDValue TableValue =
4907 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i64, N1: BitTable, N2: RoundModeTimesNumBits);
4908 SDValue TruncTable = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: TableValue);
4909
4910 // No need to mask out the high bits since the setreg will ignore them
4911 // anyway.
4912 NewMode = TruncTable;
4913 }
4914
4915 // Insert a readfirstlane in case the value is a VGPR. We could do this
4916 // earlier and keep more operations scalar, but that interferes with
4917 // combining the source.
4918 SDValue ReadFirstLaneID =
4919 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
4920 NewMode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
4921 N1: ReadFirstLaneID, N2: NewMode);
4922 }
4923
4924 // N.B. The setreg will be later folded into s_round_mode on supported
4925 // targets.
4926 SDValue IntrinID =
4927 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
4928 uint32_t BothRoundHwReg =
4929 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 4);
4930 SDValue RoundBothImm = DAG.getTargetConstant(Val: BothRoundHwReg, DL: SL, VT: MVT::i32);
4931
4932 SDValue SetReg =
4933 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VTList: Op->getVTList(), N1: Op.getOperand(i: 0),
4934 N2: IntrinID, N3: RoundBothImm, N4: NewMode);
4935
4936 return SetReg;
4937}
4938
4939SDValue SITargetLowering::lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const {
4940 if (Op->isDivergent() &&
4941 (!Subtarget->hasVmemPrefInsts() || !Op.getConstantOperandVal(i: 4)))
4942 // Cannot do I$ prefetch with divergent pointer.
4943 return SDValue();
4944
4945 switch (cast<MemSDNode>(Val&: Op)->getAddressSpace()) {
4946 case AMDGPUAS::FLAT_ADDRESS:
4947 case AMDGPUAS::GLOBAL_ADDRESS:
4948 case AMDGPUAS::CONSTANT_ADDRESS:
4949 break;
4950 case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
4951 if (Subtarget->hasSafeSmemPrefetch())
4952 break;
4953 [[fallthrough]];
4954 default:
4955 return SDValue();
4956 }
4957
4958 // I$ prefetch
4959 if (!Subtarget->hasSafeSmemPrefetch() && !Op.getConstantOperandVal(i: 4))
4960 return SDValue();
4961
4962 return Op;
4963}
4964
4965// Work around DAG legality rules only based on the result type.
4966SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
4967 bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
4968 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
4969 EVT SrcVT = Src.getValueType();
4970
4971 if (SrcVT.getScalarType() != MVT::bf16)
4972 return Op;
4973
4974 SDLoc SL(Op);
4975 SDValue BitCast =
4976 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcVT.changeTypeToInteger(), Operand: Src);
4977
4978 EVT DstVT = Op.getValueType();
4979 if (IsStrict)
4980 llvm_unreachable("Need STRICT_BF16_TO_FP");
4981
4982 return DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SL, VT: DstVT, Operand: BitCast);
4983}
4984
4985SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
4986 SDLoc SL(Op);
4987 if (Op.getValueType() != MVT::i64)
4988 return Op;
4989
4990 uint32_t ModeHwReg =
4991 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
4992 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
4993 uint32_t TrapHwReg =
4994 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
4995 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
4996
4997 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
4998 SDValue IntrinID =
4999 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_getreg, DL: SL, VT: MVT::i32);
5000 SDValue GetModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5001 N1: Op.getOperand(i: 0), N2: IntrinID, N3: ModeHwRegImm);
5002 SDValue GetTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL: SL, VTList,
5003 N1: Op.getOperand(i: 0), N2: IntrinID, N3: TrapHwRegImm);
5004 SDValue TokenReg =
5005 DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: GetModeReg.getValue(R: 1),
5006 N2: GetTrapReg.getValue(R: 1));
5007
5008 SDValue CvtPtr =
5009 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: GetModeReg, N2: GetTrapReg);
5010 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
5011
5012 return DAG.getMergeValues(Ops: {Result, TokenReg}, dl: SL);
5013}
5014
5015SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const {
5016 SDLoc SL(Op);
5017 if (Op.getOperand(i: 1).getValueType() != MVT::i64)
5018 return Op;
5019
5020 SDValue Input = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
5021 SDValue NewModeReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5022 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
5023 SDValue NewTrapReg = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Input,
5024 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
5025
5026 SDValue ReadFirstLaneID =
5027 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL: SL, VT: MVT::i32);
5028 NewModeReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5029 N1: ReadFirstLaneID, N2: NewModeReg);
5030 NewTrapReg = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
5031 N1: ReadFirstLaneID, N2: NewTrapReg);
5032
5033 unsigned ModeHwReg =
5034 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_MODE, Values: 0, Values: 23);
5035 SDValue ModeHwRegImm = DAG.getTargetConstant(Val: ModeHwReg, DL: SL, VT: MVT::i32);
5036 unsigned TrapHwReg =
5037 AMDGPU::Hwreg::HwregEncoding::encode(Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: 0, Values: 5);
5038 SDValue TrapHwRegImm = DAG.getTargetConstant(Val: TrapHwReg, DL: SL, VT: MVT::i32);
5039
5040 SDValue IntrinID =
5041 DAG.getTargetConstant(Val: Intrinsic::amdgcn_s_setreg, DL: SL, VT: MVT::i32);
5042 SDValue SetModeReg =
5043 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5044 N2: IntrinID, N3: ModeHwRegImm, N4: NewModeReg);
5045 SDValue SetTrapReg =
5046 DAG.getNode(Opcode: ISD::INTRINSIC_VOID, DL: SL, VT: MVT::Other, N1: Op.getOperand(i: 0),
5047 N2: IntrinID, N3: TrapHwRegImm, N4: NewTrapReg);
5048 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: SetTrapReg, N2: SetModeReg);
5049}
5050
5051Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT,
5052 const MachineFunction &MF) const {
5053 const Function &Fn = MF.getFunction();
5054
5055 Register Reg = StringSwitch<Register>(RegName)
5056 .Case(S: "m0", Value: AMDGPU::M0)
5057 .Case(S: "exec", Value: AMDGPU::EXEC)
5058 .Case(S: "exec_lo", Value: AMDGPU::EXEC_LO)
5059 .Case(S: "exec_hi", Value: AMDGPU::EXEC_HI)
5060 .Case(S: "flat_scratch", Value: AMDGPU::FLAT_SCR)
5061 .Case(S: "flat_scratch_lo", Value: AMDGPU::FLAT_SCR_LO)
5062 .Case(S: "flat_scratch_hi", Value: AMDGPU::FLAT_SCR_HI)
5063 .Default(Value: Register());
5064 if (!Reg)
5065 return Reg;
5066
5067 if (!Subtarget->hasFlatScrRegister() &&
5068 Subtarget->getRegisterInfo()->regsOverlap(RegA: Reg, RegB: AMDGPU::FLAT_SCR)) {
5069 Fn.getContext().emitError(ErrorStr: Twine("invalid register \"" + StringRef(RegName) +
5070 "\" for subtarget."));
5071 }
5072
5073 switch (Reg) {
5074 case AMDGPU::M0:
5075 case AMDGPU::EXEC_LO:
5076 case AMDGPU::EXEC_HI:
5077 case AMDGPU::FLAT_SCR_LO:
5078 case AMDGPU::FLAT_SCR_HI:
5079 if (VT.getSizeInBits() == 32)
5080 return Reg;
5081 break;
5082 case AMDGPU::EXEC:
5083 case AMDGPU::FLAT_SCR:
5084 if (VT.getSizeInBits() == 64)
5085 return Reg;
5086 break;
5087 default:
5088 llvm_unreachable("missing register type checking");
5089 }
5090
5091 report_fatal_error(
5092 reason: Twine("invalid type for register \"" + StringRef(RegName) + "\"."));
5093}
5094
5095// If kill is not the last instruction, split the block so kill is always a
5096// proper terminator.
5097MachineBasicBlock *
5098SITargetLowering::splitKillBlock(MachineInstr &MI,
5099 MachineBasicBlock *BB) const {
5100 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, /*UpdateLiveIns=*/true);
5101 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5102 MI.setDesc(TII->getKillTerminatorFromPseudo(Opcode: MI.getOpcode()));
5103 return SplitBB;
5104}
5105
5106// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
5107// \p MI will be the only instruction in the loop body block. Otherwise, it will
5108// be the first instruction in the remainder block.
5109//
5110/// \returns { LoopBody, Remainder }
5111static std::pair<MachineBasicBlock *, MachineBasicBlock *>
5112splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
5113 MachineFunction *MF = MBB.getParent();
5114 MachineBasicBlock::iterator I(&MI);
5115
5116 // To insert the loop we need to split the block. Move everything after this
5117 // point to a new block, and insert a new empty block between the two.
5118 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
5119 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
5120 MachineFunction::iterator MBBI(MBB);
5121 ++MBBI;
5122
5123 MF->insert(MBBI, MBB: LoopBB);
5124 MF->insert(MBBI, MBB: RemainderBB);
5125
5126 LoopBB->addSuccessor(Succ: LoopBB);
5127 LoopBB->addSuccessor(Succ: RemainderBB);
5128
5129 // Move the rest of the block into a new block.
5130 RemainderBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
5131
5132 if (InstInLoop) {
5133 auto Next = std::next(x: I);
5134
5135 // Move instruction to loop body.
5136 LoopBB->splice(Where: LoopBB->begin(), Other: &MBB, From: I, To: Next);
5137
5138 // Move the rest of the block.
5139 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: Next, To: MBB.end());
5140 } else {
5141 RemainderBB->splice(Where: RemainderBB->begin(), Other: &MBB, From: I, To: MBB.end());
5142 }
5143
5144 MBB.addSuccessor(Succ: LoopBB);
5145
5146 return std::pair(LoopBB, RemainderBB);
5147}
5148
5149/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
5150void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
5151 MachineBasicBlock *MBB = MI.getParent();
5152 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5153 auto I = MI.getIterator();
5154 auto E = std::next(x: I);
5155
5156 // clang-format off
5157 BuildMI(BB&: *MBB, I: E, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_WAITCNT))
5158 .addImm(Val: 0);
5159 // clang-format on
5160
5161 MIBundleBuilder Bundler(*MBB, I, E);
5162 finalizeBundle(MBB&: *MBB, FirstMI: Bundler.begin());
5163}
5164
5165MachineBasicBlock *
5166SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
5167 MachineBasicBlock *BB) const {
5168 const DebugLoc &DL = MI.getDebugLoc();
5169
5170 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5171
5172 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
5173
5174 // Apparently kill flags are only valid if the def is in the same block?
5175 if (MachineOperand *Src = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::data0))
5176 Src->setIsKill(false);
5177
5178 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB&: *BB, InstInLoop: true);
5179
5180 MachineBasicBlock::iterator I = LoopBB->end();
5181
5182 const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
5183 Values: AMDGPU::Hwreg::ID_TRAPSTS, Values: AMDGPU::Hwreg::OFFSET_MEM_VIOL, Values: 1);
5184
5185 // Clear TRAP_STS.MEM_VIOL
5186 BuildMI(BB&: *LoopBB, I: LoopBB->begin(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_IMM32_B32))
5187 .addImm(Val: 0)
5188 .addImm(Val: EncodedReg);
5189
5190 bundleInstWithWaitcnt(MI);
5191
5192 Register Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5193
5194 // Load and check TRAP_STS.MEM_VIOL
5195 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: Reg)
5196 .addImm(Val: EncodedReg);
5197
5198 // FIXME: Do we need to use an isel pseudo that may clobber scc?
5199 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
5200 .addReg(RegNo: Reg, Flags: RegState::Kill)
5201 .addImm(Val: 0);
5202 // clang-format off
5203 BuildMI(BB&: *LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
5204 .addMBB(MBB: LoopBB);
5205 // clang-format on
5206
5207 return RemainderBB;
5208}
5209
5210// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
5211// wavefront. If the value is uniform and just happens to be in a VGPR, this
5212// will only do one iteration. In the worst case, this will loop 64 times.
5213//
5214// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
5215static MachineBasicBlock::iterator
5216emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
5217 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
5218 const DebugLoc &DL, const MachineOperand &Idx,
5219 unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
5220 unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
5221 Register &SGPRIdxReg) {
5222
5223 MachineFunction *MF = OrigBB.getParent();
5224 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5225 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5226 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5227 MachineBasicBlock::iterator I = LoopBB.begin();
5228
5229 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5230 Register PhiExec = MRI.createVirtualRegister(RegClass: BoolRC);
5231 Register NewExec = MRI.createVirtualRegister(RegClass: BoolRC);
5232 Register CurrentIdxReg =
5233 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5234 Register CondReg = MRI.createVirtualRegister(RegClass: BoolRC);
5235
5236 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiReg)
5237 .addReg(RegNo: InitReg)
5238 .addMBB(MBB: &OrigBB)
5239 .addReg(RegNo: ResultReg)
5240 .addMBB(MBB: &LoopBB);
5241
5242 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::PHI), DestReg: PhiExec)
5243 .addReg(RegNo: InitSaveExecReg)
5244 .addMBB(MBB: &OrigBB)
5245 .addReg(RegNo: NewExec)
5246 .addMBB(MBB: &LoopBB);
5247
5248 // Read the next variant <- also loop target.
5249 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CurrentIdxReg)
5250 .addReg(RegNo: Idx.getReg(), Flags: getUndefRegState(B: Idx.isUndef()));
5251
5252 // Compare the just read M0 value to all possible Idx values.
5253 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CMP_EQ_U32_e64), DestReg: CondReg)
5254 .addReg(RegNo: CurrentIdxReg)
5255 .addReg(RegNo: Idx.getReg(), Flags: {}, SubReg: Idx.getSubReg());
5256
5257 // Update EXEC, save the original EXEC value to VCC.
5258 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.AndSaveExecOpc), DestReg: NewExec)
5259 .addReg(RegNo: CondReg, Flags: RegState::Kill);
5260
5261 MRI.setSimpleHint(VReg: NewExec, PrefReg: CondReg);
5262
5263 if (UseGPRIdxMode) {
5264 if (Offset == 0) {
5265 SGPRIdxReg = CurrentIdxReg;
5266 } else {
5267 SGPRIdxReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
5268 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: SGPRIdxReg)
5269 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill)
5270 .addImm(Val: Offset);
5271 }
5272 } else {
5273 // Move index from VCC into M0
5274 if (Offset == 0) {
5275 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5276 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill);
5277 } else {
5278 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5279 .addReg(RegNo: CurrentIdxReg, Flags: RegState::Kill)
5280 .addImm(Val: Offset);
5281 }
5282 }
5283
5284 // Update EXEC, switch all done bits to 0 and all todo bits to 1.
5285 MachineInstr *InsertPt =
5286 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.XorTermOpc), DestReg: LMC.ExecReg)
5287 .addReg(RegNo: LMC.ExecReg)
5288 .addReg(RegNo: NewExec);
5289
5290 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
5291 // s_cbranch_scc0?
5292
5293 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
5294 // clang-format off
5295 BuildMI(BB&: LoopBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
5296 .addMBB(MBB: &LoopBB);
5297 // clang-format on
5298
5299 return InsertPt->getIterator();
5300}
5301
5302// This has slightly sub-optimal regalloc when the source vector is killed by
5303// the read. The register allocator does not understand that the kill is
5304// per-workitem, so is kept alive for the whole loop so we end up not re-using a
5305// subregister from it, using 1 more VGPR than necessary. This was saved when
5306// this was expanded after register allocation.
5307static MachineBasicBlock::iterator
5308loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
5309 unsigned InitResultReg, unsigned PhiReg, int Offset,
5310 bool UseGPRIdxMode, Register &SGPRIdxReg) {
5311 MachineFunction *MF = MBB.getParent();
5312 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5313 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5314 MachineRegisterInfo &MRI = MF->getRegInfo();
5315 const DebugLoc &DL = MI.getDebugLoc();
5316 MachineBasicBlock::iterator I(&MI);
5317
5318 const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
5319 Register DstReg = MI.getOperand(i: 0).getReg();
5320 Register SaveExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5321 Register TmpExec = MRI.createVirtualRegister(RegClass: BoolXExecRC);
5322 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
5323
5324 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: TmpExec);
5325
5326 // Save the EXEC mask
5327 // clang-format off
5328 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: SaveExec)
5329 .addReg(RegNo: LMC.ExecReg);
5330 // clang-format on
5331
5332 auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, InstInLoop: false);
5333
5334 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5335
5336 auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, OrigBB&: MBB, LoopBB&: *LoopBB, DL, Idx: *Idx,
5337 InitReg: InitResultReg, ResultReg: DstReg, PhiReg, InitSaveExecReg: TmpExec,
5338 Offset, UseGPRIdxMode, SGPRIdxReg);
5339
5340 MachineBasicBlock *LandingPad = MF->CreateMachineBasicBlock();
5341 MachineFunction::iterator MBBI(LoopBB);
5342 ++MBBI;
5343 MF->insert(MBBI, MBB: LandingPad);
5344 LoopBB->removeSuccessor(Succ: RemainderBB);
5345 LandingPad->addSuccessor(Succ: RemainderBB);
5346 LoopBB->addSuccessor(Succ: LandingPad);
5347 MachineBasicBlock::iterator First = LandingPad->begin();
5348 // clang-format off
5349 BuildMI(BB&: *LandingPad, I: First, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
5350 .addReg(RegNo: SaveExec);
5351 // clang-format on
5352
5353 return InsPt;
5354}
5355
5356// Returns subreg index, offset
5357static std::pair<unsigned, int>
5358computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
5359 const TargetRegisterClass *SuperRC, unsigned VecReg,
5360 int Offset) {
5361 int NumElts = TRI.getRegSizeInBits(RC: *SuperRC) / 32;
5362
5363 // Skip out of bounds offsets, or else we would end up using an undefined
5364 // register.
5365 if (Offset >= NumElts || Offset < 0)
5366 return std::pair(AMDGPU::sub0, Offset);
5367
5368 return std::pair(SIRegisterInfo::getSubRegFromChannel(Channel: Offset), 0);
5369}
5370
5371static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
5372 MachineRegisterInfo &MRI, MachineInstr &MI,
5373 int Offset) {
5374 MachineBasicBlock *MBB = MI.getParent();
5375 const DebugLoc &DL = MI.getDebugLoc();
5376 MachineBasicBlock::iterator I(&MI);
5377
5378 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5379
5380 assert(Idx->getReg() != AMDGPU::NoRegister);
5381
5382 if (Offset == 0) {
5383 // clang-format off
5384 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
5385 .add(MO: *Idx);
5386 // clang-format on
5387 } else {
5388 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: AMDGPU::M0)
5389 .add(MO: *Idx)
5390 .addImm(Val: Offset);
5391 }
5392}
5393
5394static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
5395 MachineRegisterInfo &MRI, MachineInstr &MI,
5396 int Offset) {
5397 MachineBasicBlock *MBB = MI.getParent();
5398 const DebugLoc &DL = MI.getDebugLoc();
5399 MachineBasicBlock::iterator I(&MI);
5400
5401 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5402
5403 if (Offset == 0)
5404 return Idx->getReg();
5405
5406 Register Tmp = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
5407 BuildMI(BB&: *MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: Tmp)
5408 .add(MO: *Idx)
5409 .addImm(Val: Offset);
5410 return Tmp;
5411}
5412
5413static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
5414 MachineBasicBlock &MBB,
5415 const GCNSubtarget &ST) {
5416 const SIInstrInfo *TII = ST.getInstrInfo();
5417 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5418 MachineFunction *MF = MBB.getParent();
5419 MachineRegisterInfo &MRI = MF->getRegInfo();
5420
5421 Register Dst = MI.getOperand(i: 0).getReg();
5422 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5423 Register SrcReg = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src)->getReg();
5424 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5425
5426 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcReg);
5427 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5428
5429 unsigned SubReg;
5430 std::tie(args&: SubReg, args&: Offset) =
5431 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcReg, Offset);
5432
5433 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5434
5435 // Check for a SGPR index.
5436 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5437 MachineBasicBlock::iterator I(&MI);
5438 const DebugLoc &DL = MI.getDebugLoc();
5439
5440 if (UseGPRIdxMode) {
5441 // TODO: Look at the uses to avoid the copy. This may require rescheduling
5442 // to avoid interfering with other uses, so probably requires a new
5443 // optimization pass.
5444 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5445
5446 const MCInstrDesc &GPRIDXDesc =
5447 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5448 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5449 .addReg(RegNo: SrcReg)
5450 .addReg(RegNo: Idx)
5451 .addImm(Val: SubReg);
5452 } else {
5453 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5454
5455 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5456 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5457 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5458 }
5459
5460 MI.eraseFromParent();
5461
5462 return &MBB;
5463 }
5464
5465 // Control flow needs to be inserted if indexing with a VGPR.
5466 const DebugLoc &DL = MI.getDebugLoc();
5467 MachineBasicBlock::iterator I(&MI);
5468
5469 Register PhiReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5470 Register InitReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5471
5472 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: InitReg);
5473
5474 Register SGPRIdxReg;
5475 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: InitReg, PhiReg, Offset,
5476 UseGPRIdxMode, SGPRIdxReg);
5477
5478 MachineBasicBlock *LoopBB = InsPt->getParent();
5479
5480 if (UseGPRIdxMode) {
5481 const MCInstrDesc &GPRIDXDesc =
5482 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: true);
5483
5484 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5485 .addReg(RegNo: SrcReg)
5486 .addReg(RegNo: SGPRIdxReg)
5487 .addImm(Val: SubReg);
5488 } else {
5489 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOVRELS_B32_e32), DestReg: Dst)
5490 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
5491 .addReg(RegNo: SrcReg, Flags: RegState::Implicit);
5492 }
5493
5494 MI.eraseFromParent();
5495
5496 return LoopBB;
5497}
5498
5499static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
5500 MachineBasicBlock &MBB,
5501 const GCNSubtarget &ST) {
5502 const SIInstrInfo *TII = ST.getInstrInfo();
5503 const SIRegisterInfo &TRI = TII->getRegisterInfo();
5504 MachineFunction *MF = MBB.getParent();
5505 MachineRegisterInfo &MRI = MF->getRegInfo();
5506
5507 Register Dst = MI.getOperand(i: 0).getReg();
5508 const MachineOperand *SrcVec = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::src);
5509 const MachineOperand *Idx = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::idx);
5510 const MachineOperand *Val = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::val);
5511 int Offset = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::offset)->getImm();
5512 const TargetRegisterClass *VecRC = MRI.getRegClass(Reg: SrcVec->getReg());
5513 const TargetRegisterClass *IdxRC = MRI.getRegClass(Reg: Idx->getReg());
5514
5515 // This can be an immediate, but will be folded later.
5516 assert(Val->getReg());
5517
5518 unsigned SubReg;
5519 std::tie(args&: SubReg, args&: Offset) =
5520 computeIndirectRegAndOffset(TRI, SuperRC: VecRC, VecReg: SrcVec->getReg(), Offset);
5521 const bool UseGPRIdxMode = ST.useVGPRIndexMode();
5522
5523 if (Idx->getReg() == AMDGPU::NoRegister) {
5524 MachineBasicBlock::iterator I(&MI);
5525 const DebugLoc &DL = MI.getDebugLoc();
5526
5527 assert(Offset == 0);
5528
5529 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: Dst)
5530 .add(MO: *SrcVec)
5531 .add(MO: *Val)
5532 .addImm(Val: SubReg);
5533
5534 MI.eraseFromParent();
5535 return &MBB;
5536 }
5537
5538 // Check for a SGPR index.
5539 if (TII->getRegisterInfo().isSGPRClass(RC: IdxRC)) {
5540 MachineBasicBlock::iterator I(&MI);
5541 const DebugLoc &DL = MI.getDebugLoc();
5542
5543 if (UseGPRIdxMode) {
5544 Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
5545
5546 const MCInstrDesc &GPRIDXDesc =
5547 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5548 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5549 .addReg(RegNo: SrcVec->getReg())
5550 .add(MO: *Val)
5551 .addReg(RegNo: Idx)
5552 .addImm(Val: SubReg);
5553 } else {
5554 setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
5555
5556 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5557 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5558 BuildMI(BB&: MBB, I, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5559 .addReg(RegNo: SrcVec->getReg())
5560 .add(MO: *Val)
5561 .addImm(Val: SubReg);
5562 }
5563 MI.eraseFromParent();
5564 return &MBB;
5565 }
5566
5567 // Control flow needs to be inserted if indexing with a VGPR.
5568 if (Val->isReg())
5569 MRI.clearKillFlags(Reg: Val->getReg());
5570
5571 const DebugLoc &DL = MI.getDebugLoc();
5572
5573 Register PhiReg = MRI.createVirtualRegister(RegClass: VecRC);
5574
5575 Register SGPRIdxReg;
5576 auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitResultReg: SrcVec->getReg(), PhiReg, Offset,
5577 UseGPRIdxMode, SGPRIdxReg);
5578 MachineBasicBlock *LoopBB = InsPt->getParent();
5579
5580 if (UseGPRIdxMode) {
5581 const MCInstrDesc &GPRIDXDesc =
5582 TII->getIndirectGPRIDXPseudo(VecSize: TRI.getRegSizeInBits(RC: *VecRC), IsIndirectSrc: false);
5583
5584 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: GPRIDXDesc, DestReg: Dst)
5585 .addReg(RegNo: PhiReg)
5586 .add(MO: *Val)
5587 .addReg(RegNo: SGPRIdxReg)
5588 .addImm(Val: SubReg);
5589 } else {
5590 const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
5591 VecSize: TRI.getRegSizeInBits(RC: *VecRC), EltSize: 32, IsSGPR: false);
5592 BuildMI(BB&: *LoopBB, I: InsPt, MIMD: DL, MCID: MovRelDesc, DestReg: Dst)
5593 .addReg(RegNo: PhiReg)
5594 .add(MO: *Val)
5595 .addImm(Val: SubReg);
5596 }
5597
5598 MI.eraseFromParent();
5599 return LoopBB;
5600}
5601
5602static MachineBasicBlock *expand64BitScalarArithmetic(MachineInstr &MI,
5603 MachineBasicBlock *BB) {
5604 // For targets older than GFX12, we emit a sequence of 32-bit operations.
5605 // For GFX12, we emit s_add_u64 and s_sub_u64.
5606 MachineFunction *MF = BB->getParent();
5607 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
5608 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5609 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
5610 const DebugLoc &DL = MI.getDebugLoc();
5611 MachineOperand &Dest = MI.getOperand(i: 0);
5612 MachineOperand &Src0 = MI.getOperand(i: 1);
5613 MachineOperand &Src1 = MI.getOperand(i: 2);
5614 bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5615 if (ST.hasScalarAddSub64()) {
5616 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
5617 // clang-format off
5618 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg())
5619 .add(MO: Src0)
5620 .add(MO: Src1);
5621 // clang-format on
5622 } else {
5623 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5624 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
5625
5626 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5627 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5628
5629 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5630 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5631 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5632 MI, MRI, SuperReg: Src0, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5633
5634 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5635 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub0, SubRC: &AMDGPU::SReg_32RegClass);
5636 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5637 MI, MRI, SuperReg: Src1, SuperRC: BoolRC, SubIdx: AMDGPU::sub1, SubRC: &AMDGPU::SReg_32RegClass);
5638
5639 unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
5640 unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
5641 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0).add(MO: Src0Sub0).add(MO: Src1Sub0);
5642 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1).add(MO: Src0Sub1).add(MO: Src1Sub1);
5643 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
5644 .addReg(RegNo: DestSub0)
5645 .addImm(Val: AMDGPU::sub0)
5646 .addReg(RegNo: DestSub1)
5647 .addImm(Val: AMDGPU::sub1);
5648 }
5649 MI.eraseFromParent();
5650 return BB;
5651}
5652
5653static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
5654 MachineFunction *MF = BB->getParent();
5655 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
5656 const SIInstrInfo *TII = ST.getInstrInfo();
5657 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5658 MachineRegisterInfo &MRI = MF->getRegInfo();
5659 const DebugLoc &DL = MI.getDebugLoc();
5660 Register Dst = MI.getOperand(i: 0).getReg();
5661 const MachineOperand &Src0 = MI.getOperand(i: 1);
5662 const MachineOperand &Src1 = MI.getOperand(i: 2);
5663 Register SrcCond = MI.getOperand(i: 3).getReg();
5664
5665 Register DstLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5666 Register DstHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
5667 const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
5668 Register SrcCondCopy = MRI.createVirtualRegister(RegClass: CondRC);
5669
5670 int Src0Idx =
5671 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
5672 int Src1Idx =
5673 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
5674 const TargetRegisterClass *Src0RC =
5675 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src0Idx));
5676 const TargetRegisterClass *Src1RC =
5677 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: Src1Idx));
5678
5679 const TargetRegisterClass *Src0SubRC =
5680 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
5681 const TargetRegisterClass *Src1SubRC =
5682 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
5683
5684 MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
5685 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
5686 MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
5687 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
5688
5689 MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
5690 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
5691 MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
5692 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
5693
5694 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SrcCondCopy).addReg(RegNo: SrcCond);
5695 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstLo)
5696 .addImm(Val: 0)
5697 .add(MO: Src0Sub0)
5698 .addImm(Val: 0)
5699 .add(MO: Src1Sub0)
5700 .addReg(RegNo: SrcCondCopy);
5701
5702 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B32_e64), DestReg: DstHi)
5703 .addImm(Val: 0)
5704 .add(MO: Src0Sub1)
5705 .addImm(Val: 0)
5706 .add(MO: Src1Sub1)
5707 .addReg(RegNo: SrcCondCopy);
5708
5709 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: Dst)
5710 .addReg(RegNo: DstLo)
5711 .addImm(Val: AMDGPU::sub0)
5712 .addReg(RegNo: DstHi)
5713 .addImm(Val: AMDGPU::sub1);
5714 MI.eraseFromParent();
5715}
5716
5717static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
5718 switch (Opc) {
5719 case AMDGPU::S_MIN_U32:
5720 return std::numeric_limits<uint32_t>::max();
5721 case AMDGPU::S_MIN_I32:
5722 return std::numeric_limits<int32_t>::max();
5723 case AMDGPU::S_MAX_U32:
5724 return std::numeric_limits<uint32_t>::min();
5725 case AMDGPU::S_MAX_I32:
5726 return std::numeric_limits<int32_t>::min();
5727 case AMDGPU::V_ADD_F32_e64: // -0.0
5728 return 0x80000000;
5729 case AMDGPU::V_SUB_F32_e64: // +0.0
5730 return 0x0;
5731 case AMDGPU::S_ADD_I32:
5732 case AMDGPU::S_SUB_I32:
5733 case AMDGPU::S_OR_B32:
5734 case AMDGPU::S_XOR_B32:
5735 return std::numeric_limits<uint32_t>::min();
5736 case AMDGPU::S_AND_B32:
5737 return std::numeric_limits<uint32_t>::max();
5738 case AMDGPU::V_MIN_F32_e64:
5739 case AMDGPU::V_MAX_F32_e64:
5740 return 0x7fc00000; // qNAN
5741 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5742 return std::numeric_limits<uint64_t>::max();
5743 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5744 return std::numeric_limits<int64_t>::max();
5745 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5746 return std::numeric_limits<uint64_t>::min();
5747 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5748 return std::numeric_limits<int64_t>::min();
5749 case AMDGPU::V_MIN_F64_e64:
5750 case AMDGPU::V_MAX_F64_e64:
5751 case AMDGPU::V_MIN_NUM_F64_e64:
5752 case AMDGPU::V_MAX_NUM_F64_e64:
5753 return 0x7FF8000000000000; // qNAN
5754 case AMDGPU::S_ADD_U64_PSEUDO:
5755 case AMDGPU::S_SUB_U64_PSEUDO:
5756 case AMDGPU::S_OR_B64:
5757 case AMDGPU::S_XOR_B64:
5758 return std::numeric_limits<uint64_t>::min();
5759 case AMDGPU::S_AND_B64:
5760 return std::numeric_limits<uint64_t>::max();
5761 case AMDGPU::V_ADD_F64_e64:
5762 case AMDGPU::V_ADD_F64_pseudo_e64:
5763 return 0x8000000000000000; // -0.0
5764 default:
5765 llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
5766 }
5767}
5768
5769static bool is32bitWaveReduceOperation(unsigned Opc) {
5770 return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
5771 Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
5772 Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
5773 Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
5774 Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
5775 Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
5776 Opc == AMDGPU::V_SUB_F32_e64;
5777}
5778
5779static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
5780 return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
5781 Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
5782 Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
5783 Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
5784 Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
5785}
5786
5787static std::tuple<unsigned, unsigned>
5788getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
5789 unsigned DPPOpc;
5790 switch (Opc) {
5791 case AMDGPU::S_MIN_U32:
5792 DPPOpc = AMDGPU::V_MIN_U32_dpp;
5793 break;
5794 case AMDGPU::S_MIN_I32:
5795 DPPOpc = AMDGPU::V_MIN_I32_dpp;
5796 break;
5797 case AMDGPU::S_MAX_U32:
5798 DPPOpc = AMDGPU::V_MAX_U32_dpp;
5799 break;
5800 case AMDGPU::S_MAX_I32:
5801 DPPOpc = AMDGPU::V_MAX_I32_dpp;
5802 break;
5803 case AMDGPU::S_ADD_I32:
5804 case AMDGPU::S_SUB_I32:
5805 DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
5806 : AMDGPU::V_ADD_CO_U32_dpp;
5807 break;
5808 case AMDGPU::S_AND_B32:
5809 DPPOpc = AMDGPU::V_AND_B32_dpp;
5810 break;
5811 case AMDGPU::S_OR_B32:
5812 DPPOpc = AMDGPU::V_OR_B32_dpp;
5813 break;
5814 case AMDGPU::S_XOR_B32:
5815 DPPOpc = AMDGPU::V_XOR_B32_dpp;
5816 break;
5817 case AMDGPU::V_ADD_F32_e64:
5818 case AMDGPU::V_SUB_F32_e64:
5819 DPPOpc = AMDGPU::V_ADD_F32_dpp;
5820 break;
5821 case AMDGPU::V_MIN_F32_e64:
5822 DPPOpc = AMDGPU::V_MIN_F32_dpp;
5823 break;
5824 case AMDGPU::V_MAX_F32_e64:
5825 DPPOpc = AMDGPU::V_MAX_F32_dpp;
5826 break;
5827 case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
5828 case AMDGPU::V_CMP_LT_I64_e64: // min.i64
5829 case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
5830 case AMDGPU::V_CMP_GT_I64_e64: // max.i64
5831 case AMDGPU::S_ADD_U64_PSEUDO:
5832 case AMDGPU::S_SUB_U64_PSEUDO:
5833 case AMDGPU::S_AND_B64:
5834 case AMDGPU::S_OR_B64:
5835 case AMDGPU::S_XOR_B64:
5836 case AMDGPU::V_MIN_NUM_F64_e64:
5837 case AMDGPU::V_MIN_F64_e64:
5838 case AMDGPU::V_MAX_NUM_F64_e64:
5839 case AMDGPU::V_MAX_F64_e64:
5840 case AMDGPU::V_ADD_F64_pseudo_e64:
5841 case AMDGPU::V_ADD_F64_e64:
5842 DPPOpc = AMDGPU::V_MOV_B64_DPP_PSEUDO;
5843 break;
5844 default:
5845 llvm_unreachable("unhandled lane op");
5846 }
5847 unsigned ClampOpc = Opc;
5848 if (!ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
5849 if (Opc == AMDGPU::S_SUB_I32)
5850 ClampOpc = AMDGPU::S_ADD_I32;
5851 if (Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO)
5852 ClampOpc = AMDGPU::V_ADD_CO_U32_e64;
5853 else if (Opc == AMDGPU::S_AND_B64)
5854 ClampOpc = AMDGPU::V_AND_B32_e64;
5855 else if (Opc == AMDGPU::S_OR_B64)
5856 ClampOpc = AMDGPU::V_OR_B32_e64;
5857 else if (Opc == AMDGPU::S_XOR_B64)
5858 ClampOpc = AMDGPU::V_XOR_B32_e64;
5859 else
5860 ClampOpc = ST.getInstrInfo()->getVALUOp(Opc: ClampOpc);
5861 }
5862 return {DPPOpc, ClampOpc};
5863}
5864
5865static std::pair<Register, Register>
5866ExtractSubRegs(MachineInstr &MI, MachineOperand &Op,
5867 const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
5868 MachineRegisterInfo &MRI) {
5869 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5870 const SIInstrInfo *TII = ST.getInstrInfo();
5871 const TargetRegisterClass *SrcSubRC =
5872 TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
5873 Register Op1L =
5874 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub0, SubRC: SrcSubRC);
5875 Register Op1H =
5876 TII->buildExtractSubReg(MI, MRI, SuperReg: Op, SuperRC: SrcRC, SubIdx: AMDGPU::sub1, SubRC: SrcSubRC);
5877 return {Op1L, Op1H};
5878}
5879
5880static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
5881 MachineBasicBlock &BB,
5882 const GCNSubtarget &ST,
5883 unsigned Opc) {
5884 MachineRegisterInfo &MRI = BB.getParent()->getRegInfo();
5885 const SIRegisterInfo *TRI = ST.getRegisterInfo();
5886 const DebugLoc &DL = MI.getDebugLoc();
5887 const SIInstrInfo *TII = ST.getInstrInfo();
5888
5889 // Reduction operations depend on whether the input operand is SGPR or VGPR.
5890 Register SrcReg = MI.getOperand(i: 1).getReg();
5891 bool isSGPR = TRI->isSGPRClass(RC: MRI.getRegClass(Reg: SrcReg));
5892 Register DstReg = MI.getOperand(i: 0).getReg();
5893 unsigned Stratergy = static_cast<unsigned>(MI.getOperand(i: 2).getImm());
5894 enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
5895 MachineBasicBlock *RetBB = nullptr;
5896 unsigned MIOpc = MI.getOpcode();
5897 auto BuildRegSequence = [&](MachineBasicBlock &BB,
5898 MachineBasicBlock::iterator MI, Register Dst,
5899 Register Src0, Register Src1) {
5900 auto RegSequence =
5901 BuildMI(BB, I: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dst)
5902 .addReg(RegNo: Src0)
5903 .addImm(Val: AMDGPU::sub0)
5904 .addReg(RegNo: Src1)
5905 .addImm(Val: AMDGPU::sub1);
5906 return RegSequence;
5907 };
5908 if (isSGPR) {
5909 switch (Opc) {
5910 case AMDGPU::S_MIN_U32:
5911 case AMDGPU::S_MIN_I32:
5912 case AMDGPU::V_MIN_F32_e64:
5913 case AMDGPU::S_MAX_U32:
5914 case AMDGPU::S_MAX_I32:
5915 case AMDGPU::V_MAX_F32_e64:
5916 case AMDGPU::S_AND_B32:
5917 case AMDGPU::S_OR_B32: {
5918 // Idempotent operations.
5919 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: DstReg).addReg(RegNo: SrcReg);
5920 RetBB = &BB;
5921 break;
5922 }
5923 case AMDGPU::V_CMP_LT_U64_e64: // umin
5924 case AMDGPU::V_CMP_LT_I64_e64: // min
5925 case AMDGPU::V_CMP_GT_U64_e64: // umax
5926 case AMDGPU::V_CMP_GT_I64_e64: // max
5927 case AMDGPU::V_MIN_F64_e64:
5928 case AMDGPU::V_MIN_NUM_F64_e64:
5929 case AMDGPU::V_MAX_F64_e64:
5930 case AMDGPU::V_MAX_NUM_F64_e64:
5931 case AMDGPU::S_AND_B64:
5932 case AMDGPU::S_OR_B64: {
5933 // Idempotent operations.
5934 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B64), DestReg: DstReg).addReg(RegNo: SrcReg);
5935 RetBB = &BB;
5936 break;
5937 }
5938 case AMDGPU::S_XOR_B32:
5939 case AMDGPU::S_XOR_B64:
5940 case AMDGPU::S_ADD_I32:
5941 case AMDGPU::S_ADD_U64_PSEUDO:
5942 case AMDGPU::V_ADD_F32_e64:
5943 case AMDGPU::V_ADD_F64_e64:
5944 case AMDGPU::V_ADD_F64_pseudo_e64:
5945 case AMDGPU::S_SUB_I32:
5946 case AMDGPU::S_SUB_U64_PSEUDO:
5947 case AMDGPU::V_SUB_F32_e64: {
5948 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
5949 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
5950 Register ExecMask = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
5951 Register NumActiveLanes =
5952 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5953
5954 bool IsWave32 = ST.isWave32();
5955 unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
5956 MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
5957 unsigned BitCountOpc =
5958 IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
5959
5960 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MovOpc), DestReg: ExecMask).addReg(RegNo: ExecReg);
5961
5962 auto NewAccumulator =
5963 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: BitCountOpc), DestReg: NumActiveLanes)
5964 .addReg(RegNo: ExecMask);
5965
5966 switch (Opc) {
5967 case AMDGPU::S_XOR_B32:
5968 case AMDGPU::S_XOR_B64: {
5969 // Performing an XOR operation on a uniform value
5970 // depends on the parity of the number of active lanes.
5971 // For even parity, the result will be 0, for odd
5972 // parity the result will be the same as the input value.
5973 Register ParityRegister =
5974 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5975
5976 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_AND_B32), DestReg: ParityRegister)
5977 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
5978 .addImm(Val: 1)
5979 .setOperandDead(3); // Dead scc
5980 if (Opc == AMDGPU::S_XOR_B32) {
5981 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
5982 .addReg(RegNo: SrcReg)
5983 .addReg(RegNo: ParityRegister);
5984 } else {
5985 Register DestSub0 =
5986 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5987 Register DestSub1 =
5988 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
5989 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
5990 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
5991 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
5992 .addReg(RegNo: Op1L)
5993 .addReg(RegNo: ParityRegister);
5994 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub1)
5995 .addReg(RegNo: Op1H)
5996 .addReg(RegNo: ParityRegister);
5997 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
5998 }
5999 break;
6000 }
6001 case AMDGPU::S_SUB_I32: {
6002 Register NegatedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6003
6004 // Take the negation of the source operand.
6005 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedVal)
6006 .addImm(Val: 0)
6007 .addReg(RegNo: SrcReg);
6008 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6009 .addReg(RegNo: NegatedVal)
6010 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6011 break;
6012 }
6013 case AMDGPU::S_ADD_I32: {
6014 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DstReg)
6015 .addReg(RegNo: SrcReg)
6016 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg());
6017 break;
6018 }
6019 case AMDGPU::S_ADD_U64_PSEUDO:
6020 case AMDGPU::S_SUB_U64_PSEUDO: {
6021 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6022 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6023 Register Op1H_Op0L_Reg =
6024 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6025 Register Op1L_Op0H_Reg =
6026 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6027 Register CarryReg =
6028 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6029 Register AddReg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6030 Register NegatedValLo =
6031 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6032 Register NegatedValHi =
6033 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6034 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6035 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6036 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6037 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedValLo)
6038 .addImm(Val: 0)
6039 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6040 .setOperandDead(3); // Dead scc
6041 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ASHR_I32), DestReg: NegatedValHi)
6042 .addReg(RegNo: NegatedValLo)
6043 .addImm(Val: 31)
6044 .setOperandDead(3); // Dead scc
6045 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1L_Op0H_Reg)
6046 .addReg(RegNo: Op1L)
6047 .addReg(RegNo: NegatedValHi);
6048 }
6049 Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
6050 ? NegatedValLo
6051 : NewAccumulator->getOperand(i: 0).getReg();
6052 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: DestSub0)
6053 .addReg(RegNo: Op1L)
6054 .addReg(RegNo: LowOpcode);
6055 if (ST.hasScalarMulHiInsts()) {
6056 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_HI_U32), DestReg: CarryReg)
6057 .addReg(RegNo: Op1L)
6058 .addReg(RegNo: LowOpcode);
6059 } else {
6060 Register VCarryReg =
6061 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6062 Register LowOpVGPR =
6063 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6064 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LowOpVGPR)
6065 .addReg(RegNo: LowOpcode);
6066 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_HI_U32_e64), DestReg: VCarryReg)
6067 .addReg(RegNo: Op1L)
6068 .addReg(RegNo: LowOpVGPR);
6069 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: CarryReg)
6070 .addReg(RegNo: VCarryReg);
6071 }
6072 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MUL_I32), DestReg: Op1H_Op0L_Reg)
6073 .addReg(RegNo: Op1H)
6074 .addReg(RegNo: LowOpcode);
6075
6076 Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
6077 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: HiVal)
6078 .addReg(RegNo: CarryReg)
6079 .addReg(RegNo: Op1H_Op0L_Reg)
6080 .setOperandDead(3); // Dead scc
6081
6082 if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6083 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: DestSub1)
6084 .addReg(RegNo: HiVal)
6085 .addReg(RegNo: Op1L_Op0H_Reg)
6086 .setOperandDead(3); // Dead scc
6087 }
6088 BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
6089 break;
6090 }
6091 case AMDGPU::V_ADD_F32_e64:
6092 case AMDGPU::V_ADD_F64_e64:
6093 case AMDGPU::V_ADD_F64_pseudo_e64:
6094 case AMDGPU::V_SUB_F32_e64: {
6095 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6096 const TargetRegisterClass *VregRC = TII->getRegClass(MCID: TII->get(Opcode: Opc), OpNum: 0);
6097 Register ActiveLanesVreg = MRI.createVirtualRegister(RegClass: VregRC);
6098 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6099 // Get number of active lanes as a float val.
6100 BuildMI(BB, I&: MI, MIMD: DL,
6101 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
6102 : AMDGPU::V_CVT_F64_I32_e64),
6103 DestReg: ActiveLanesVreg)
6104 .addReg(RegNo: NewAccumulator->getOperand(i: 0).getReg())
6105 .addImm(Val: 0) // clamp
6106 .addImm(Val: 0); // output-modifier
6107
6108 // Take negation of input for SUB reduction
6109 unsigned srcMod = (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6110 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
6111 ? SISrcMods::NEG
6112 : SISrcMods::NONE;
6113 unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
6114 : ST.getGeneration() >= AMDGPUSubtarget::GFX12
6115 ? AMDGPU::V_MUL_F64_pseudo_e64
6116 : AMDGPU::V_MUL_F64_e64;
6117 auto DestVregInst = BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: MulOpc),
6118 DestReg: DstVreg)
6119 .addImm(Val: srcMod) // src0 modifier
6120 .addReg(RegNo: SrcReg)
6121 .addImm(Val: SISrcMods::NONE) // src1 modifier
6122 .addReg(RegNo: ActiveLanesVreg)
6123 .addImm(Val: SISrcMods::NONE) // clamp
6124 .addImm(Val: SISrcMods::NONE); // output-mod
6125 if (is32BitOpc) {
6126 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
6127 .addReg(RegNo: DstVreg);
6128 } else {
6129 Register LaneValueLoReg =
6130 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6131 Register LaneValueHiReg =
6132 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6133 auto [Op1L, Op1H] =
6134 ExtractSubRegs(MI, Op&: DestVregInst->getOperand(i: 0), SrcRC: VregRC, ST, MRI);
6135 // lane value input should be in an sgpr
6136 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6137 DestReg: LaneValueLoReg)
6138 .addReg(RegNo: Op1L);
6139 BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6140 DestReg: LaneValueHiReg)
6141 .addReg(RegNo: Op1H);
6142 NewAccumulator =
6143 BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
6144 }
6145 }
6146 }
6147 RetBB = &BB;
6148 }
6149 }
6150 } else {
6151 MachineBasicBlock::iterator I = BB.end();
6152 Register SrcReg = MI.getOperand(i: 1).getReg();
6153 bool is32BitOpc = is32bitWaveReduceOperation(Opc);
6154 bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
6155 bool NeedsMovDPP = !is32BitOpc;
6156 // Create virtual registers required for lowering.
6157 const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
6158 const TargetRegisterClass *DstRegClass = MRI.getRegClass(Reg: DstReg);
6159 const TargetRegisterClass *SrcRegClass = MRI.getRegClass(Reg: SrcReg);
6160 bool IsWave32 = ST.isWave32();
6161 unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
6162 unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
6163 if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
6164 !ST.hasDPP()) { // If target doesn't support DPP operations, default to
6165 // iterative stratergy
6166
6167 // To reduce the VGPR using iterative approach, we need to iterate
6168 // over all the active lanes. Lowering consists of ComputeLoop,
6169 // which iterate over only active lanes. We use copy of EXEC register
6170 // as induction variable and every active lane modifies it using bitset0
6171 // so that we will get the next active lane for next iteration.
6172
6173 // Create Control flow for loop
6174 // Split MI's Machine Basic block into For loop
6175 auto [ComputeLoop, ComputeEnd] = splitBlockForLoop(MI, MBB&: BB, InstInLoop: true);
6176
6177 Register LoopIterator = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6178 Register IdentityValReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6179 Register AccumulatorReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6180 Register ActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6181 Register NewActiveBitsReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6182 Register FF1Reg = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6183 Register LaneValueReg = MRI.createVirtualRegister(RegClass: DstRegClass);
6184
6185 // Create initial values of induction variable from Exec, Accumulator and
6186 // insert branch instr to newly created ComputeBlock
6187 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: MovOpcForExec), DestReg: LoopIterator).addReg(RegNo: ExecReg);
6188 uint64_t IdentityValue =
6189 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6190 ? 0x0 // +0.0 for double sub reduction
6191 : getIdentityValueForWaveReduction(Opc);
6192 BuildMI(BB, I, MIMD: DL,
6193 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6194 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6195 DestReg: IdentityValReg)
6196 .addImm(Val: IdentityValue);
6197 // clang-format off
6198 BuildMI(BB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_BRANCH))
6199 .addMBB(MBB: ComputeLoop);
6200 // clang-format on
6201
6202 // Start constructing ComputeLoop
6203 I = ComputeLoop->begin();
6204 auto Accumulator =
6205 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: AccumulatorReg)
6206 .addReg(RegNo: IdentityValReg)
6207 .addMBB(MBB: &BB);
6208 auto ActiveBits =
6209 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::PHI), DestReg: ActiveBitsReg)
6210 .addReg(RegNo: LoopIterator)
6211 .addMBB(MBB: &BB);
6212
6213 I = ComputeLoop->end();
6214 MachineInstr *NewAccumulator;
6215 // Perform the computations
6216 unsigned SFFOpc =
6217 IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
6218 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: SFFOpc), DestReg: FF1Reg)
6219 .addReg(RegNo: ActiveBitsReg);
6220 if (is32BitOpc) {
6221 Register OpDstReg = DstReg;
6222 bool hasSrc0Modifier = AMDGPU::getNamedOperandIdx(
6223 Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers) != -1;
6224 bool hasSrc1Modifier = AMDGPU::getNamedOperandIdx(
6225 Opcode: Opc, Name: AMDGPU::OpName::src1_modifiers) != -1;
6226 bool hasClamp =
6227 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::clamp) != -1;
6228 bool hasOpSel =
6229 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel) != -1;
6230 bool hasOMod =
6231 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::omod) != -1;
6232 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6233 DestReg: LaneValueReg)
6234 .addReg(RegNo: SrcReg)
6235 .addReg(RegNo: FF1Reg);
6236 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
6237 // Get the Lane Value in VGPR to avoid the Constant Bus Restriction
6238 Register LaneValVgpr = MRI.createVirtualRegister(RegClass: SrcRegClass);
6239 Register VgprResultReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6240 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LaneValVgpr)
6241 .addReg(RegNo: LaneValueReg);
6242 OpDstReg = VgprResultReg;
6243 LaneValueReg = LaneValVgpr;
6244 }
6245 auto OpInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: OpDstReg);
6246 if (hasSrc0Modifier)
6247 OpInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6248 OpInstr.addReg(RegNo: AccumulatorReg); // src0
6249 if (hasSrc1Modifier)
6250 OpInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6251 OpInstr.addReg(RegNo: LaneValueReg); // src1
6252 if (hasClamp)
6253 OpInstr.addImm(Val: 0); // clamp
6254 if (hasOpSel)
6255 OpInstr.addImm(Val: 0); // opsel
6256 if (hasOMod)
6257 OpInstr.addImm(Val: 0); // omod
6258 if (ST.getInstrInfo()->isVALU(Opcode: Opc, /*AllowLDSDMA=*/true)) {
6259 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32),
6260 DestReg: DstReg)
6261 .addReg(RegNo: OpDstReg);
6262 }
6263 } else {
6264 Register LaneValueLoReg =
6265 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6266 Register LaneValueHiReg =
6267 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6268 Register LaneValReg =
6269 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_64RegClass);
6270 auto [Op1L, Op1H] = ExtractSubRegs(MI, Op&: MI.getOperand(i: 1),
6271 SrcRC: MRI.getRegClass(Reg: SrcReg), ST, MRI);
6272 // lane value input should be in an sgpr
6273 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6274 DestReg: LaneValueLoReg)
6275 .addReg(RegNo: Op1L)
6276 .addReg(RegNo: FF1Reg);
6277 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6278 DestReg: LaneValueHiReg)
6279 .addReg(RegNo: Op1H)
6280 .addReg(RegNo: FF1Reg);
6281 auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
6282 LaneValueLoReg, LaneValueHiReg);
6283 switch (Opc) {
6284 case AMDGPU::S_OR_B64:
6285 case AMDGPU::S_AND_B64:
6286 case AMDGPU::S_XOR_B64: {
6287 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6288 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6289 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6290 .setOperandDead(3); // Dead scc
6291 break;
6292 }
6293 case AMDGPU::V_CMP_GT_I64_e64:
6294 case AMDGPU::V_CMP_GT_U64_e64:
6295 case AMDGPU::V_CMP_LT_I64_e64:
6296 case AMDGPU::V_CMP_LT_U64_e64: {
6297 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6298 Register ComparisonResultReg =
6299 MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6300 int SrcIdx =
6301 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6302 const TargetRegisterClass *VregClass =
6303 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6304 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregClass);
6305 auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
6306 MI, Op&: Accumulator->getOperand(i: 0), SrcRC: VregClass, ST, MRI);
6307 BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
6308 SrcReg0Sub1);
6309 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: LaneMaskReg)
6310 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6311 .addReg(RegNo: AccumulatorVReg);
6312
6313 unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
6314 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AndOpc), DestReg: ComparisonResultReg)
6315 .addReg(RegNo: LaneMaskReg)
6316 .addReg(RegNo: ActiveBitsReg);
6317
6318 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6319 MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B64), DestReg: DstReg)
6320 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6321 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6322 break;
6323 }
6324 case AMDGPU::V_MIN_F64_e64:
6325 case AMDGPU::V_MIN_NUM_F64_e64:
6326 case AMDGPU::V_MAX_F64_e64:
6327 case AMDGPU::V_MAX_NUM_F64_e64:
6328 case AMDGPU::V_ADD_F64_e64:
6329 case AMDGPU::V_ADD_F64_pseudo_e64: {
6330 int SrcIdx =
6331 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src);
6332 const TargetRegisterClass *VregRC =
6333 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: MI.getDesc(), OpNum: SrcIdx));
6334 Register AccumulatorVReg = MRI.createVirtualRegister(RegClass: VregRC);
6335 Register DstVreg = MRI.createVirtualRegister(RegClass: VregRC);
6336 Register LaneValLo =
6337 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6338 Register LaneValHi =
6339 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6340 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AccumulatorVReg)
6341 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg());
6342 unsigned Modifier =
6343 MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
6344 ? SISrcMods::NEG
6345 : SISrcMods::NONE;
6346 auto DstVregInst =
6347 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstVreg)
6348 .addImm(Val: Modifier) // src0 modifiers
6349 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg())
6350 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6351 .addReg(RegNo: AccumulatorVReg)
6352 .addImm(Val: SISrcMods::NONE) // clamp
6353 .addImm(Val: SISrcMods::NONE); // omod
6354 auto ReadLaneLo =
6355 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6356 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValLo);
6357 auto ReadLaneHi =
6358 BuildMI(BB&: *ComputeLoop, I, MIMD: DL,
6359 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: LaneValHi);
6360 MachineBasicBlock::iterator Iters = *ReadLaneLo;
6361 auto [Op1L, Op1H] = ExtractSubRegs(MI&: *Iters, Op&: DstVregInst->getOperand(i: 0),
6362 SrcRC: VregRC, ST, MRI);
6363 ReadLaneLo.addReg(RegNo: Op1L);
6364 ReadLaneHi.addReg(RegNo: Op1H);
6365 NewAccumulator =
6366 BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
6367 break;
6368 }
6369 case AMDGPU::S_ADD_U64_PSEUDO:
6370 case AMDGPU::S_SUB_U64_PSEUDO: {
6371 NewAccumulator = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: DstReg)
6372 .addReg(RegNo: Accumulator->getOperand(i: 0).getReg())
6373 .addReg(RegNo: LaneValue->getOperand(i: 0).getReg());
6374 ComputeLoop =
6375 expand64BitScalarArithmetic(MI&: *NewAccumulator, BB: ComputeLoop);
6376 break;
6377 }
6378 }
6379 }
6380 // Manipulate the iterator to get the next active lane
6381 unsigned BITSETOpc =
6382 IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
6383 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: BITSETOpc), DestReg: NewActiveBitsReg)
6384 .addReg(RegNo: FF1Reg)
6385 .addReg(RegNo: ActiveBitsReg);
6386
6387 // Add phi nodes
6388 Accumulator.addReg(RegNo: DstReg).addMBB(MBB: ComputeLoop);
6389 ActiveBits.addReg(RegNo: NewActiveBitsReg).addMBB(MBB: ComputeLoop);
6390
6391 // Creating branching
6392 MachineInstrBuilder SetSCCInstr;
6393 if (!ST.hasScalarCompareEq64()) {
6394 // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
6395 Register LaneMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6396 unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
6397 SetSCCInstr =
6398 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc), DestReg: LaneMaskReg);
6399 } else {
6400 unsigned CMPOpc =
6401 IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
6402 SetSCCInstr = BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: CMPOpc));
6403 }
6404 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6405 if (ST.hasScalarCompareEq64())
6406 SetSCCInstr.addImm(Val: 0);
6407 else
6408 SetSCCInstr.addReg(RegNo: NewActiveBitsReg);
6409 BuildMI(BB&: *ComputeLoop, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
6410 .addMBB(MBB: ComputeLoop);
6411
6412 RetBB = ComputeEnd;
6413 } else {
6414 assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
6415 MachineBasicBlock *CurrBB = &BB;
6416 Register SrcWithIdentity = MRI.createVirtualRegister(RegClass: SrcRegClass);
6417 Register IdentityVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6418 Register IdentitySGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6419 Register DPPRowShr1 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6420 Register DPPRowShr2 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6421 Register DPPRowShr4 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6422 Register DPPRowShr8 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6423 Register RowBcast15 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6424 Register ReducedValSGPR = MRI.createVirtualRegister(RegClass: DstRegClass);
6425 Register NegatedReducedVal = MRI.createVirtualRegister(RegClass: DstRegClass);
6426 Register RowBcast31 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6427 Register UndefExec = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6428 Register FinalDPPResult;
6429 MachineInstr *SrcWithIdentityInstr;
6430 MachineInstr *LastBcastInstr;
6431 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: UndefExec);
6432
6433 uint64_t IdentityValue = getIdentityValueForWaveReduction(Opc);
6434 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL,
6435 MCID: TII->get(Opcode: is32BitOpc ? AMDGPU::S_MOV_B32
6436 : AMDGPU::S_MOV_B64_IMM_PSEUDO),
6437 DestReg: IdentitySGPR)
6438 .addImm(Val: IdentityValue);
6439 auto IdentityCopyInstr =
6440 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: IdentityVGPR)
6441 .addReg(RegNo: IdentitySGPR);
6442 auto DPPClampOpcPair = getDPPOpcForWaveReduction(Opc, ST);
6443 unsigned DPPOpc = std::get<0>(t&: DPPClampOpcPair);
6444 unsigned ClampOpc = std::get<1>(t&: DPPClampOpcPair);
6445 auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
6446 Register Src1) {
6447 return BuildMI(BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_SET_INACTIVE_B32),
6448 DestReg: Dst)
6449 .addImm(Val: 0) // src0 modifiers
6450 .addReg(RegNo: Src0) // src0
6451 .addImm(Val: 0) // src1 modifiers
6452 .addReg(RegNo: Src1) // identity value for inactive lanes
6453 .addReg(RegNo: UndefExec); // bool i1
6454 };
6455 auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
6456 unsigned DPPCtrl) {
6457 auto DPPInstr =
6458 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: DPPOpc), DestReg: Dst).addReg(RegNo: Src); // old
6459 if (isFPOp && !NeedsMovDPP)
6460 DPPInstr.addImm(Val: SISrcMods::NONE); // src0 modifier
6461 DPPInstr.addReg(RegNo: Src); // src0
6462 if (isFPOp && !NeedsMovDPP)
6463 DPPInstr.addImm(Val: SISrcMods::NONE); // src1 modifier
6464 if (!NeedsMovDPP)
6465 DPPInstr.addReg(RegNo: Src); // src1
6466 if (AMDGPU::getNamedOperandIdx(Opcode: DPPOpc, Name: AMDGPU::OpName::clamp) >= 0)
6467 DPPInstr.addImm(Val: 0); // clamp
6468 DPPInstr
6469 .addImm(Val: DPPCtrl) // dpp-ctrl
6470 .addImm(Val: 0xf) // row-mask
6471 .addImm(Val: 0xf) // bank-mask
6472 .addImm(Val: 0); // bound-control
6473 };
6474 auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1,
6475 bool isAddSub = false,
6476 bool needsCarryIn = false,
6477 Register CarryIn = Register()) {
6478 unsigned InstrOpc = ClampOpc;
6479 Register CarryOutReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6480 if (needsCarryIn)
6481 InstrOpc = AMDGPU::V_ADDC_U32_e64;
6482 auto ClampInstr = BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: InstrOpc), DestReg: Dst);
6483 if (isFPOp)
6484 ClampInstr.addImm(Val: SISrcMods::NONE); // src0 mod
6485 if (isAddSub) {
6486 if (needsCarryIn)
6487 ClampInstr.addReg(RegNo: CarryOutReg,
6488 Flags: RegState::Define |
6489 RegState::Dead); // killed carry-out reg
6490 else
6491 ClampInstr.addReg(RegNo: CarryOutReg, Flags: RegState::Define); // carry-out reg
6492 }
6493 ClampInstr.addReg(RegNo: Src0); // src0
6494 if (isFPOp)
6495 ClampInstr.addImm(Val: SISrcMods::NONE); // src1 mod
6496 ClampInstr.addReg(RegNo: Src1); // src1
6497 if (needsCarryIn)
6498 ClampInstr.addReg(RegNo: CarryIn, Flags: RegState::Kill); // carry-in reg
6499 if (AMDGPU::getNamedOperandIdx(Opcode: InstrOpc, Name: AMDGPU::OpName::clamp) >= 0)
6500 ClampInstr.addImm(Val: 0); // clamp
6501 if (isFPOp)
6502 ClampInstr.addImm(Val: 0); // omod
6503 LastBcastInstr = ClampInstr;
6504 return CarryOutReg;
6505 };
6506 auto BuildPostDPPInstr = [&](Register Src0, Register Src1) {
6507 bool isAddSubOpc =
6508 Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO;
6509 bool isBitWiseOpc = Opc == AMDGPU::S_AND_B64 ||
6510 Opc == AMDGPU::S_OR_B64 || Opc == AMDGPU::S_XOR_B64;
6511 Register ReturnReg = MRI.createVirtualRegister(RegClass: SrcRegClass);
6512 if (isAddSubOpc || isBitWiseOpc) {
6513 Register ResLo = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6514 Register ResHi = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6515 MachineOperand Src0Operand =
6516 MachineOperand::CreateReg(Reg: Src0, /*isDef=*/false);
6517 MachineOperand Src1Operand =
6518 MachineOperand::CreateReg(Reg: Src1, /*isDef=*/false);
6519 auto [Src0Lo, Src0Hi] =
6520 ExtractSubRegs(MI, Op&: Src0Operand, SrcRC: SrcRegClass, ST, MRI);
6521 auto [Src1Lo, Src1Hi] =
6522 ExtractSubRegs(MI, Op&: Src1Operand, SrcRC: SrcRegClass, ST, MRI);
6523 Register CarryReg = BuildClampInstr(
6524 ResLo, Src0Lo, Src1Lo, isAddSubOpc, /*needsCarryIn*/ false);
6525 BuildClampInstr(ResHi, Src0Hi, Src1Hi, isAddSubOpc,
6526 /*needsCarryIn*/ isAddSubOpc, CarryReg);
6527 BuildRegSequence(*CurrBB, MI, ReturnReg, ResLo, ResHi);
6528 } else {
6529 if (isFPOp) {
6530 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: ReturnReg)
6531 .addImm(Val: SISrcMods::NONE) // src0 modifiers
6532 .addReg(RegNo: Src0)
6533 .addImm(Val: SISrcMods::NONE) // src1 modifiers
6534 .addReg(RegNo: Src1)
6535 .addImm(Val: SISrcMods::NONE) // clamp
6536 .addImm(Val: SISrcMods::NONE); // omod
6537 } else {
6538 Register CmpMaskReg = MRI.createVirtualRegister(RegClass: WaveMaskRegClass);
6539 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: CmpMaskReg)
6540 .addReg(RegNo: Src0) // src0
6541 .addReg(RegNo: Src1); // src1
6542 LastBcastInstr =
6543 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_CNDMASK_B64_PSEUDO),
6544 DestReg: ReturnReg)
6545 .addReg(RegNo: Src1) // src0
6546 .addReg(RegNo: Src0) // src1
6547 .addReg(RegNo: CmpMaskReg); // src2
6548 expand64BitV_CNDMASK(MI&: *LastBcastInstr, BB: CurrBB);
6549 }
6550 }
6551 return ReturnReg;
6552 };
6553
6554 // Set inactive lanes to the identity value.
6555 if (is32BitOpc) {
6556 SrcWithIdentityInstr =
6557 BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
6558 } else {
6559 Register SrcWithIdentitylo =
6560 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6561 Register SrcWithIdentityhi =
6562 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6563 auto [Reg0Sub0, Reg0Sub1] = ExtractSubRegs(
6564 MI, Op&: IdentityCopyInstr->getOperand(i: 0), SrcRC: SrcRegClass, ST, MRI);
6565 auto [SrcReg0Sub0, SrcReg0Sub1] =
6566 ExtractSubRegs(MI, Op&: MI.getOperand(i: 1), SrcRC: SrcRegClass, ST, MRI);
6567 MachineInstr *SetInactiveLoInstr =
6568 BuildSetInactiveInstr(SrcWithIdentitylo, SrcReg0Sub0, Reg0Sub0);
6569 MachineInstr *SetInactiveHiInstr =
6570 BuildSetInactiveInstr(SrcWithIdentityhi, SrcReg0Sub1, Reg0Sub1);
6571 SrcWithIdentityInstr =
6572 BuildRegSequence(*CurrBB, MI, SrcWithIdentity,
6573 SetInactiveLoInstr->getOperand(i: 0).getReg(),
6574 SetInactiveHiInstr->getOperand(i: 0).getReg());
6575 }
6576 // DPP reduction
6577 Register SrcWithIdentityReg =
6578 SrcWithIdentityInstr->getOperand(i: 0).getReg();
6579 BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentityReg,
6580 AMDGPU::DPP::ROW_SHR_FIRST);
6581 if (NeedsMovDPP)
6582 DPPRowShr1 = BuildPostDPPInstr(SrcWithIdentityReg, DPPRowShr1);
6583
6584 BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
6585 (AMDGPU::DPP::ROW_SHR_FIRST + 1));
6586 if (NeedsMovDPP)
6587 DPPRowShr2 = BuildPostDPPInstr(DPPRowShr1, DPPRowShr2);
6588
6589 BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
6590 (AMDGPU::DPP::ROW_SHR_FIRST + 3));
6591 if (NeedsMovDPP)
6592 DPPRowShr4 = BuildPostDPPInstr(DPPRowShr2, DPPRowShr4);
6593
6594 BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
6595 (AMDGPU::DPP::ROW_SHR_FIRST + 7));
6596 if (NeedsMovDPP)
6597 DPPRowShr8 = BuildPostDPPInstr(DPPRowShr4, DPPRowShr8);
6598
6599 if (ST.hasDPPBroadcasts()) {
6600 BuildDPPMachineInstr(RowBcast15, DPPRowShr8, AMDGPU::DPP::BCAST15);
6601 if (NeedsMovDPP)
6602 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, RowBcast15);
6603 } else {
6604 // magic constant: 0x1E0
6605 // To Set BIT_MODE : bit 15 = 0
6606 // XOR mask : bit [14:10] = 0
6607 // OR mask : bit [9:5] = 15
6608 // AND mask : bit [4:0] = 0
6609 if (is32BitOpc) {
6610 Register SwizzledValue =
6611 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6612 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6613 DestReg: SwizzledValue)
6614 .addReg(RegNo: DPPRowShr8) // addr
6615 .addImm(Val: 0x1E0) // swizzle offset (i16)
6616 .addImm(Val: 0x0); // gds (i1)
6617 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
6618 } else {
6619 Register SwizzledValuelo =
6620 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6621 Register SwizzledValuehi =
6622 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6623 Register SwizzledValue64 = MRI.createVirtualRegister(RegClass: SrcRegClass);
6624 MachineOperand DPPRowShr8Op =
6625 MachineOperand::CreateReg(Reg: DPPRowShr8, /*isDef=*/false);
6626 auto [Op1L, Op1H] =
6627 ExtractSubRegs(MI, Op&: DPPRowShr8Op, SrcRC: SrcRegClass, ST, MRI);
6628 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6629 DestReg: SwizzledValuelo)
6630 .addReg(RegNo: Op1L) // addr
6631 .addImm(Val: 0x1E0) // swizzle offset (i16)
6632 .addImm(Val: 0x0); // gds (i1)
6633 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_SWIZZLE_B32),
6634 DestReg: SwizzledValuehi)
6635 .addReg(RegNo: Op1H) // addr
6636 .addImm(Val: 0x1E0) // swizzle offset (i16)
6637 .addImm(Val: 0x0); // gds (i1)
6638 BuildRegSequence(*CurrBB, MI, SwizzledValue64, SwizzledValuelo,
6639 SwizzledValuehi);
6640 if (NeedsMovDPP)
6641 RowBcast15 = BuildPostDPPInstr(DPPRowShr8, SwizzledValue64);
6642 else
6643 BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue64);
6644 }
6645 }
6646 FinalDPPResult = RowBcast15;
6647 if (!IsWave32) {
6648 if (ST.hasDPPBroadcasts()) {
6649 BuildDPPMachineInstr(RowBcast31, RowBcast15, AMDGPU::DPP::BCAST31);
6650 if (NeedsMovDPP)
6651 RowBcast31 = BuildPostDPPInstr(RowBcast15, RowBcast31);
6652 } else {
6653 Register ShiftedThreadID =
6654 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6655 Register PermuteByteOffset =
6656 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6657 Register PermutedValue = MRI.createVirtualRegister(RegClass: SrcRegClass);
6658 Register Lane32Offset =
6659 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6660 Register WordSizeConst =
6661 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
6662 Register ThreadIDRegLo =
6663 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6664 Register ThreadIDReg =
6665 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6666 // Get the thread ID.
6667 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_LO_U32_B32_e64),
6668 DestReg: ThreadIDRegLo)
6669 .addImm(Val: -1)
6670 .addImm(Val: 0);
6671 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MBCNT_HI_U32_B32_e64),
6672 DestReg: ThreadIDReg)
6673 .addImm(Val: -1)
6674 .addReg(RegNo: ThreadIDRegLo);
6675 // shift each lane over by 32 positions, so value in 31st lane is
6676 // present in 63rd lane.
6677 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: Lane32Offset)
6678 .addImm(Val: 0x20);
6679 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_ADD_U32_e64),
6680 DestReg: ShiftedThreadID)
6681 .addReg(RegNo: ThreadIDReg)
6682 .addReg(RegNo: Lane32Offset)
6683 .addImm(Val: 0); // clamp
6684 // multiply by reg size.
6685 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: WordSizeConst)
6686 .addImm(Val: 0x4);
6687 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MUL_LO_U32_e64),
6688 DestReg: PermuteByteOffset)
6689 .addReg(RegNo: WordSizeConst)
6690 .addReg(RegNo: ShiftedThreadID);
6691 // Permute the lanes
6692 if (is32BitOpc) {
6693 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6694 DestReg: PermutedValue)
6695 .addReg(RegNo: PermuteByteOffset) // addr
6696 .addReg(RegNo: RowBcast15) // data
6697 .addImm(Val: 0); // offset
6698 } else {
6699 Register PermutedValuelo =
6700 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6701 Register PermutedValuehi =
6702 MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6703 MachineOperand RowBcast15Op =
6704 MachineOperand::CreateReg(Reg: RowBcast15, /*isDef=*/false);
6705 auto [RowBcast15Lo, RowBcast15Hi] =
6706 ExtractSubRegs(MI, Op&: RowBcast15Op, SrcRC: SrcRegClass, ST, MRI);
6707 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6708 DestReg: PermutedValuelo)
6709 .addReg(RegNo: PermuteByteOffset) // addr
6710 .addReg(RegNo: RowBcast15Lo) // swizzle offset (i16)
6711 .addImm(Val: 0x0); // gds (i1)
6712 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::DS_PERMUTE_B32),
6713 DestReg: PermutedValuehi)
6714 .addReg(RegNo: PermuteByteOffset) // addr
6715 .addReg(RegNo: RowBcast15Hi) // swizzle offset (i16)
6716 .addImm(Val: 0x0); // gds (i1)
6717 BuildRegSequence(*CurrBB, MI, PermutedValue, PermutedValuelo,
6718 PermutedValuehi);
6719 }
6720 if (NeedsMovDPP)
6721 RowBcast31 = BuildPostDPPInstr(RowBcast15, PermutedValue);
6722 else
6723 BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
6724 }
6725 FinalDPPResult = RowBcast31;
6726 }
6727 if (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
6728 MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64) {
6729 Register NegatedValVGPR = MRI.createVirtualRegister(RegClass: SrcRegClass);
6730 // Opc for f32 reduction is V_SUB_F32.
6731 // For f64, there is no equivalent V_SUB_F64 opcode, so use
6732 // V_ADD_F64/V_ADD_F64_pseudo, and negate the second operand.
6733 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc),
6734 DestReg: NegatedValVGPR)
6735 .addImm(Val: SISrcMods::NONE) // src0 mods
6736 .addReg(RegNo: IdentityVGPR) // src0
6737 .addImm(Val: is32BitOpc ? SISrcMods::NONE : SISrcMods::NEG) // src1 mods
6738 .addReg(RegNo: IsWave32 ? RowBcast15 : RowBcast31) // src1
6739 .addImm(Val: SISrcMods::NONE) // clamp
6740 .addImm(Val: SISrcMods::NONE); // omod
6741 FinalDPPResult = NegatedValVGPR;
6742 }
6743 // The final reduced value is in the last lane.
6744 if (is32BitOpc) {
6745 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6746 DestReg: ReducedValSGPR)
6747 .addReg(RegNo: FinalDPPResult)
6748 .addImm(Val: ST.getWavefrontSize() - 1);
6749 } else {
6750 Register LaneValueLoReg =
6751 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6752 Register LaneValueHiReg =
6753 MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
6754 const TargetRegisterClass *SrcRC = MRI.getRegClass(Reg: SrcReg);
6755 MachineOperand FinalDPPResultOperand =
6756 MachineOperand::CreateReg(Reg: FinalDPPResult, /*isDef=*/false);
6757 auto [Op1L, Op1H] =
6758 ExtractSubRegs(MI, Op&: FinalDPPResultOperand, SrcRC, ST, MRI);
6759 // lane value input should be in an sgpr
6760 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6761 DestReg: LaneValueLoReg)
6762 .addReg(RegNo: Op1L)
6763 .addImm(Val: ST.getWavefrontSize() - 1);
6764 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READLANE_B32),
6765 DestReg: LaneValueHiReg)
6766 .addReg(RegNo: Op1H)
6767 .addImm(Val: ST.getWavefrontSize() - 1);
6768 BuildRegSequence(*CurrBB, MI, ReducedValSGPR, LaneValueLoReg,
6769 LaneValueHiReg);
6770 }
6771 if (Opc == AMDGPU::S_SUB_I32) {
6772 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SUB_I32), DestReg: NegatedReducedVal)
6773 .addImm(Val: 0)
6774 .addReg(RegNo: ReducedValSGPR);
6775 } else if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
6776 auto NegatedValInstr =
6777 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: NegatedReducedVal)
6778 .addImm(Val: 0)
6779 .addReg(RegNo: ReducedValSGPR);
6780 CurrBB = expand64BitScalarArithmetic(MI&: *NegatedValInstr, BB: CurrBB);
6781 }
6782 // Mark the final result as a whole-wave-mode calculation.
6783 BuildMI(BB&: *CurrBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::STRICT_WWM), DestReg: DstReg)
6784 .addReg(RegNo: Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U64_PSEUDO
6785 ? NegatedReducedVal
6786 : ReducedValSGPR);
6787 RetBB = CurrBB;
6788 }
6789 }
6790 MI.eraseFromParent();
6791 return RetBB;
6792}
6793
6794MachineBasicBlock *
6795SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
6796 MachineBasicBlock *BB) const {
6797 MachineFunction *MF = BB->getParent();
6798 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
6799 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
6800 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
6801 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
6802 MachineRegisterInfo &MRI = MF->getRegInfo();
6803 const DebugLoc &DL = MI.getDebugLoc();
6804
6805 switch (MI.getOpcode()) {
6806 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
6807 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_U32);
6808 case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
6809 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_U64_e64);
6810 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
6811 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MIN_I32);
6812 case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
6813 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_LT_I64_e64);
6814 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
6815 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MIN_F32_e64);
6816 case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
6817 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6818 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6819 ? AMDGPU::V_MIN_NUM_F64_e64
6820 : AMDGPU::V_MIN_F64_e64);
6821 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
6822 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_U32);
6823 case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
6824 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_U64_e64);
6825 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
6826 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_MAX_I32);
6827 case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
6828 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_CMP_GT_I64_e64);
6829 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
6830 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_MAX_F32_e64);
6831 case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
6832 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6833 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6834 ? AMDGPU::V_MAX_NUM_F64_e64
6835 : AMDGPU::V_MAX_F64_e64);
6836 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
6837 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_I32);
6838 case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
6839 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_ADD_U64_PSEUDO);
6840 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
6841 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_ADD_F32_e64);
6842 case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
6843 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6844 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6845 ? AMDGPU::V_ADD_F64_pseudo_e64
6846 : AMDGPU::V_ADD_F64_e64);
6847 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
6848 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_I32);
6849 case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
6850 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_SUB_U64_PSEUDO);
6851 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
6852 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::V_SUB_F32_e64);
6853 case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
6854 // There is no S/V_SUB_F64 opcode. Double type subtraction is expanded as
6855 // fadd + neg, by setting the NEG bit in the instruction.
6856 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(),
6857 Opc: ST.getGeneration() >= AMDGPUSubtarget::GFX12
6858 ? AMDGPU::V_ADD_F64_pseudo_e64
6859 : AMDGPU::V_ADD_F64_e64);
6860 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
6861 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B32);
6862 case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
6863 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_AND_B64);
6864 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
6865 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B32);
6866 case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
6867 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_OR_B64);
6868 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
6869 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B32);
6870 case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
6871 return lowerWaveReduce(MI, BB&: *BB, ST: *getSubtarget(), Opc: AMDGPU::S_XOR_B64);
6872 case AMDGPU::S_UADDO_PSEUDO:
6873 case AMDGPU::S_USUBO_PSEUDO: {
6874 MachineOperand &Dest0 = MI.getOperand(i: 0);
6875 MachineOperand &Dest1 = MI.getOperand(i: 1);
6876 MachineOperand &Src0 = MI.getOperand(i: 2);
6877 MachineOperand &Src1 = MI.getOperand(i: 3);
6878
6879 unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
6880 ? AMDGPU::S_ADD_U32
6881 : AMDGPU::S_SUB_U32;
6882 // clang-format off
6883 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest0.getReg())
6884 .add(MO: Src0)
6885 .add(MO: Src1);
6886 // clang-format on
6887
6888 unsigned SelOpc =
6889 Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
6890 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: Dest1.getReg()).addImm(Val: -1).addImm(Val: 0);
6891
6892 MI.eraseFromParent();
6893 return BB;
6894 }
6895 case AMDGPU::S_ADD_U64_PSEUDO:
6896 case AMDGPU::S_SUB_U64_PSEUDO: {
6897 return expand64BitScalarArithmetic(MI, BB);
6898 }
6899 case AMDGPU::V_ADD_U64_PSEUDO:
6900 case AMDGPU::V_SUB_U64_PSEUDO: {
6901 bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
6902
6903 MachineOperand &Dest = MI.getOperand(i: 0);
6904 MachineOperand &Src0 = MI.getOperand(i: 1);
6905 MachineOperand &Src1 = MI.getOperand(i: 2);
6906
6907 if (ST.hasAddSubU64Insts()) {
6908 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL,
6909 MCID: TII->get(Opcode: IsAdd ? AMDGPU::V_ADD_U64_e64
6910 : AMDGPU::V_SUB_U64_e64),
6911 DestReg: Dest.getReg())
6912 .add(MO: Src0)
6913 .add(MO: Src1)
6914 .addImm(Val: 0); // clamp
6915 TII->legalizeOperands(MI&: *I);
6916 MI.eraseFromParent();
6917 return BB;
6918 }
6919
6920 if (IsAdd && ST.hasLshlAddU64Inst()) {
6921 auto Add = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_LSHL_ADD_U64_e64),
6922 DestReg: Dest.getReg())
6923 .add(MO: Src0)
6924 .addImm(Val: 0)
6925 .add(MO: Src1);
6926 TII->legalizeOperands(MI&: *Add);
6927 MI.eraseFromParent();
6928 return BB;
6929 }
6930
6931 const auto *CarryRC = TRI->getWaveMaskRegClass();
6932
6933 Register DestSub0 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6934 Register DestSub1 = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
6935
6936 Register CarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
6937 Register DeadCarryReg = MRI.createVirtualRegister(RegClass: CarryRC);
6938
6939 const TargetRegisterClass *Src0RC = Src0.isReg()
6940 ? MRI.getRegClass(Reg: Src0.getReg())
6941 : &AMDGPU::VReg_64RegClass;
6942 const TargetRegisterClass *Src1RC = Src1.isReg()
6943 ? MRI.getRegClass(Reg: Src1.getReg())
6944 : &AMDGPU::VReg_64RegClass;
6945
6946 const TargetRegisterClass *Src0SubRC =
6947 TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
6948 const TargetRegisterClass *Src1SubRC =
6949 TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
6950
6951 MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
6952 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub0, SubRC: Src0SubRC);
6953 MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
6954 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub0, SubRC: Src1SubRC);
6955
6956 MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
6957 MI, MRI, SuperReg: Src0, SuperRC: Src0RC, SubIdx: AMDGPU::sub1, SubRC: Src0SubRC);
6958 MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
6959 MI, MRI, SuperReg: Src1, SuperRC: Src1RC, SubIdx: AMDGPU::sub1, SubRC: Src1SubRC);
6960
6961 unsigned LoOpc =
6962 IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
6963 MachineInstr *LoHalf = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LoOpc), DestReg: DestSub0)
6964 .addReg(RegNo: CarryReg, Flags: RegState::Define)
6965 .add(MO: SrcReg0Sub0)
6966 .add(MO: SrcReg1Sub0)
6967 .addImm(Val: 0); // clamp bit
6968
6969 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
6970 MachineInstr *HiHalf =
6971 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: HiOpc), DestReg: DestSub1)
6972 .addReg(RegNo: DeadCarryReg, Flags: RegState::Define | RegState::Dead)
6973 .add(MO: SrcReg0Sub1)
6974 .add(MO: SrcReg1Sub1)
6975 .addReg(RegNo: CarryReg, Flags: RegState::Kill)
6976 .addImm(Val: 0); // clamp bit
6977
6978 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::REG_SEQUENCE), DestReg: Dest.getReg())
6979 .addReg(RegNo: DestSub0)
6980 .addImm(Val: AMDGPU::sub0)
6981 .addReg(RegNo: DestSub1)
6982 .addImm(Val: AMDGPU::sub1);
6983 TII->legalizeOperands(MI&: *LoHalf);
6984 TII->legalizeOperands(MI&: *HiHalf);
6985 MI.eraseFromParent();
6986 return BB;
6987 }
6988 case AMDGPU::S_ADD_CO_PSEUDO:
6989 case AMDGPU::S_SUB_CO_PSEUDO: {
6990 // This pseudo has a chance to be selected
6991 // only from uniform add/subcarry node. All the VGPR operands
6992 // therefore assumed to be splat vectors.
6993 MachineBasicBlock::iterator MII = MI;
6994 MachineOperand &Dest = MI.getOperand(i: 0);
6995 MachineOperand &CarryDest = MI.getOperand(i: 1);
6996 MachineOperand &Src0 = MI.getOperand(i: 2);
6997 MachineOperand &Src1 = MI.getOperand(i: 3);
6998 MachineOperand &Src2 = MI.getOperand(i: 4);
6999 if (Src0.isReg() && TRI->isVectorRegister(MRI, Reg: Src0.getReg())) {
7000 Register RegOp0 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7001 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp0)
7002 .addReg(RegNo: Src0.getReg());
7003 Src0.setReg(RegOp0);
7004 }
7005 if (Src1.isReg() && TRI->isVectorRegister(MRI, Reg: Src1.getReg())) {
7006 Register RegOp1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7007 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp1)
7008 .addReg(RegNo: Src1.getReg());
7009 Src1.setReg(RegOp1);
7010 }
7011 Register RegOp2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
7012 if (TRI->isVectorRegister(MRI, Reg: Src2.getReg())) {
7013 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: RegOp2)
7014 .addReg(RegNo: Src2.getReg());
7015 Src2.setReg(RegOp2);
7016 }
7017
7018 if (ST.isWave64()) {
7019 if (ST.hasScalarCompareEq64()) {
7020 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U64))
7021 .addReg(RegNo: Src2.getReg())
7022 .addImm(Val: 0);
7023 } else {
7024 const TargetRegisterClass *Src2RC = MRI.getRegClass(Reg: Src2.getReg());
7025 const TargetRegisterClass *SubRC =
7026 TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
7027 MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
7028 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub0, SubRC);
7029 MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
7030 MI: MII, MRI, SuperReg: Src2, SuperRC: Src2RC, SubIdx: AMDGPU::sub1, SubRC);
7031 Register Src2_32 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7032
7033 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_OR_B32), DestReg: Src2_32)
7034 .add(MO: Src2Sub0)
7035 .add(MO: Src2Sub1);
7036
7037 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7038 .addReg(RegNo: Src2_32, Flags: RegState::Kill)
7039 .addImm(Val: 0);
7040 }
7041 } else {
7042 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_LG_U32))
7043 .addReg(RegNo: Src2.getReg())
7044 .addImm(Val: 0);
7045 }
7046
7047 unsigned Opc = MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
7048 ? AMDGPU::S_ADDC_U32
7049 : AMDGPU::S_SUBB_U32;
7050
7051 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: Dest.getReg()).add(MO: Src0).add(MO: Src1);
7052
7053 unsigned SelOpc =
7054 ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
7055
7056 BuildMI(BB&: *BB, I: MII, MIMD: DL, MCID: TII->get(Opcode: SelOpc), DestReg: CarryDest.getReg())
7057 .addImm(Val: -1)
7058 .addImm(Val: 0);
7059
7060 MI.eraseFromParent();
7061 return BB;
7062 }
7063 case AMDGPU::SI_INIT_M0: {
7064 MachineOperand &M0Init = MI.getOperand(i: 0);
7065 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7066 MCID: TII->get(Opcode: M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
7067 DestReg: AMDGPU::M0)
7068 .add(MO: M0Init);
7069 MI.eraseFromParent();
7070 return BB;
7071 }
7072 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
7073 // Set SCC to true, in case the barrier instruction gets converted to a NOP.
7074 BuildMI(BB&: *BB, I: MI.getIterator(), MIMD: MI.getDebugLoc(),
7075 MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7076 .addImm(Val: 0)
7077 .addImm(Val: 0);
7078 return BB;
7079 }
7080 case AMDGPU::GET_GROUPSTATICSIZE: {
7081 assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
7082 getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
7083 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32))
7084 .add(MO: MI.getOperand(i: 0))
7085 .addImm(Val: MFI->getLDSSize());
7086 MI.eraseFromParent();
7087 return BB;
7088 }
7089 case AMDGPU::GET_SHADERCYCLESHILO: {
7090 assert(MF->getSubtarget<GCNSubtarget>().hasShaderCyclesHiLoRegisters());
7091 // The algorithm is:
7092 //
7093 // hi1 = getreg(SHADER_CYCLES_HI)
7094 // lo1 = getreg(SHADER_CYCLES_LO)
7095 // hi2 = getreg(SHADER_CYCLES_HI)
7096 //
7097 // If hi1 == hi2 then there was no overflow and the result is hi2:lo1.
7098 // Otherwise there was overflow and the result is hi2:0. In both cases the
7099 // result should represent the actual time at some point during the sequence
7100 // of three getregs.
7101 using namespace AMDGPU::Hwreg;
7102 Register RegHi1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7103 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi1)
7104 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7105 Register RegLo1 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7106 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegLo1)
7107 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES, Values: 0, Values: 32));
7108 Register RegHi2 = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7109 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_GETREG_B32), DestReg: RegHi2)
7110 .addImm(Val: HwregEncoding::encode(Values: ID_SHADER_CYCLES_HI, Values: 0, Values: 32));
7111 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
7112 .addReg(RegNo: RegHi1)
7113 .addReg(RegNo: RegHi2);
7114 Register RegLo = MRI.createVirtualRegister(RegClass: &AMDGPU::SReg_32RegClass);
7115 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: RegLo)
7116 .addReg(RegNo: RegLo1)
7117 .addImm(Val: 0);
7118 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE))
7119 .add(MO: MI.getOperand(i: 0))
7120 .addReg(RegNo: RegLo)
7121 .addImm(Val: AMDGPU::sub0)
7122 .addReg(RegNo: RegHi2)
7123 .addImm(Val: AMDGPU::sub1);
7124 MI.eraseFromParent();
7125 return BB;
7126 }
7127 case AMDGPU::SI_INDIRECT_SRC_V1:
7128 case AMDGPU::SI_INDIRECT_SRC_V2:
7129 case AMDGPU::SI_INDIRECT_SRC_V3:
7130 case AMDGPU::SI_INDIRECT_SRC_V4:
7131 case AMDGPU::SI_INDIRECT_SRC_V5:
7132 case AMDGPU::SI_INDIRECT_SRC_V6:
7133 case AMDGPU::SI_INDIRECT_SRC_V7:
7134 case AMDGPU::SI_INDIRECT_SRC_V8:
7135 case AMDGPU::SI_INDIRECT_SRC_V9:
7136 case AMDGPU::SI_INDIRECT_SRC_V10:
7137 case AMDGPU::SI_INDIRECT_SRC_V11:
7138 case AMDGPU::SI_INDIRECT_SRC_V12:
7139 case AMDGPU::SI_INDIRECT_SRC_V16:
7140 case AMDGPU::SI_INDIRECT_SRC_V32:
7141 return emitIndirectSrc(MI, MBB&: *BB, ST: *getSubtarget());
7142 case AMDGPU::SI_INDIRECT_DST_V1:
7143 case AMDGPU::SI_INDIRECT_DST_V2:
7144 case AMDGPU::SI_INDIRECT_DST_V3:
7145 case AMDGPU::SI_INDIRECT_DST_V4:
7146 case AMDGPU::SI_INDIRECT_DST_V5:
7147 case AMDGPU::SI_INDIRECT_DST_V6:
7148 case AMDGPU::SI_INDIRECT_DST_V7:
7149 case AMDGPU::SI_INDIRECT_DST_V8:
7150 case AMDGPU::SI_INDIRECT_DST_V9:
7151 case AMDGPU::SI_INDIRECT_DST_V10:
7152 case AMDGPU::SI_INDIRECT_DST_V11:
7153 case AMDGPU::SI_INDIRECT_DST_V12:
7154 case AMDGPU::SI_INDIRECT_DST_V16:
7155 case AMDGPU::SI_INDIRECT_DST_V32:
7156 return emitIndirectDst(MI, MBB&: *BB, ST: *getSubtarget());
7157 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
7158 case AMDGPU::SI_KILL_I1_PSEUDO:
7159 return splitKillBlock(MI, BB);
7160 case AMDGPU::V_CNDMASK_B64_PSEUDO: {
7161 expand64BitV_CNDMASK(MI, BB);
7162 return BB;
7163 }
7164 case AMDGPU::SI_BR_UNDEF: {
7165 MachineInstr *Br = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_SCC1))
7166 .add(MO: MI.getOperand(i: 0));
7167 Br->getOperand(i: 1).setIsUndef(); // read undef SCC
7168 MI.eraseFromParent();
7169 return BB;
7170 }
7171 case AMDGPU::ADJCALLSTACKUP:
7172 case AMDGPU::ADJCALLSTACKDOWN: {
7173 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
7174 MachineInstrBuilder MIB(*MF, &MI);
7175 MIB.addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::ImplicitDefine)
7176 .addReg(RegNo: Info->getStackPtrOffsetReg(), Flags: RegState::Implicit);
7177 return BB;
7178 }
7179 case AMDGPU::SI_CALL_ISEL: {
7180 unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(MF: *MF);
7181
7182 MachineInstrBuilder MIB;
7183 MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_CALL), DestReg: ReturnAddrReg);
7184
7185 for (const MachineOperand &MO : MI.operands())
7186 MIB.add(MO);
7187
7188 MIB.cloneMemRefs(OtherMI: MI);
7189 MI.eraseFromParent();
7190 return BB;
7191 }
7192 case AMDGPU::V_ADD_CO_U32_e32:
7193 case AMDGPU::V_SUB_CO_U32_e32:
7194 case AMDGPU::V_SUBREV_CO_U32_e32: {
7195 // TODO: Define distinct V_*_I32_Pseudo instructions instead.
7196 unsigned Opc = MI.getOpcode();
7197
7198 bool NeedClampOperand = false;
7199 if (TII->pseudoToMCOpcode(Opcode: Opc) == -1) {
7200 Opc = AMDGPU::getVOPe64(Opcode: Opc);
7201 NeedClampOperand = true;
7202 }
7203
7204 auto I = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: MI.getOperand(i: 0).getReg());
7205 if (TII->isVOP3(MI: *I)) {
7206 I.addReg(RegNo: TRI->getVCC(), Flags: RegState::Define);
7207 }
7208 I.add(MO: MI.getOperand(i: 1)).add(MO: MI.getOperand(i: 2));
7209 if (NeedClampOperand)
7210 I.addImm(Val: 0); // clamp bit for e64 encoding
7211
7212 TII->legalizeOperands(MI&: *I);
7213
7214 MI.eraseFromParent();
7215 return BB;
7216 }
7217 case AMDGPU::V_ADDC_U32_e32:
7218 case AMDGPU::V_SUBB_U32_e32:
7219 case AMDGPU::V_SUBBREV_U32_e32:
7220 // These instructions have an implicit use of vcc which counts towards the
7221 // constant bus limit.
7222 TII->legalizeOperands(MI);
7223 return BB;
7224 case AMDGPU::DS_GWS_INIT:
7225 case AMDGPU::DS_GWS_SEMA_BR:
7226 case AMDGPU::DS_GWS_BARRIER:
7227 case AMDGPU::DS_GWS_SEMA_V:
7228 case AMDGPU::DS_GWS_SEMA_P:
7229 case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
7230 // A s_waitcnt 0 is required to be the instruction immediately following.
7231 if (getSubtarget()->hasGWSAutoReplay()) {
7232 bundleInstWithWaitcnt(MI);
7233 return BB;
7234 }
7235
7236 return emitGWSMemViolTestLoop(MI, BB);
7237 case AMDGPU::S_SETREG_B32: {
7238 // Try to optimize cases that only set the denormal mode or rounding mode.
7239 //
7240 // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
7241 // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
7242 // instead.
7243 //
7244 // FIXME: This could be predicates on the immediate, but tablegen doesn't
7245 // allow you to have a no side effect instruction in the output of a
7246 // sideeffecting pattern.
7247 auto [ID, Offset, Width] =
7248 AMDGPU::Hwreg::HwregEncoding::decode(Encoded: MI.getOperand(i: 1).getImm());
7249 if (ID != AMDGPU::Hwreg::ID_MODE)
7250 return BB;
7251
7252 const unsigned WidthMask = maskTrailingOnes<unsigned>(N: Width);
7253 const unsigned SetMask = WidthMask << Offset;
7254
7255 if (getSubtarget()->hasDenormModeInst()) {
7256 unsigned SetDenormOp = 0;
7257 unsigned SetRoundOp = 0;
7258
7259 // The dedicated instructions can only set the whole denorm or round mode
7260 // at once, not a subset of bits in either.
7261 if (SetMask ==
7262 (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) {
7263 // If this fully sets both the round and denorm mode, emit the two
7264 // dedicated instructions for these.
7265 SetRoundOp = AMDGPU::S_ROUND_MODE;
7266 SetDenormOp = AMDGPU::S_DENORM_MODE;
7267 } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
7268 SetRoundOp = AMDGPU::S_ROUND_MODE;
7269 } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
7270 SetDenormOp = AMDGPU::S_DENORM_MODE;
7271 }
7272
7273 if (SetRoundOp || SetDenormOp) {
7274 MachineInstr *Def = MRI.getVRegDef(Reg: MI.getOperand(i: 0).getReg());
7275 if (Def && Def->isMoveImmediate() && Def->getOperand(i: 1).isImm()) {
7276 unsigned ImmVal = Def->getOperand(i: 1).getImm();
7277 if (SetRoundOp) {
7278 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetRoundOp))
7279 .addImm(Val: ImmVal & 0xf);
7280
7281 // If we also have the denorm mode, get just the denorm mode bits.
7282 ImmVal >>= 4;
7283 }
7284
7285 if (SetDenormOp) {
7286 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: SetDenormOp))
7287 .addImm(Val: ImmVal & 0xf);
7288 }
7289
7290 MI.eraseFromParent();
7291 return BB;
7292 }
7293 }
7294 }
7295
7296 // If only FP bits are touched, used the no side effects pseudo.
7297 if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
7298 AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
7299 MI.setDesc(TII->get(Opcode: AMDGPU::S_SETREG_B32_mode));
7300
7301 return BB;
7302 }
7303 case AMDGPU::S_INVERSE_BALLOT_U32:
7304 case AMDGPU::S_INVERSE_BALLOT_U64:
7305 // These opcodes only exist to let SIFixSGPRCopies insert a readfirstlane if
7306 // necessary. After that they are equivalent to a COPY.
7307 MI.setDesc(TII->get(Opcode: AMDGPU::COPY));
7308 return BB;
7309 case AMDGPU::ENDPGM_TRAP: {
7310 if (BB->succ_empty() && std::next(x: MI.getIterator()) == BB->end()) {
7311 MI.setDesc(TII->get(Opcode: AMDGPU::S_ENDPGM));
7312 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
7313 return BB;
7314 }
7315
7316 // We need a block split to make the real endpgm a terminator. We also don't
7317 // want to break phis in successor blocks, so we can't just delete to the
7318 // end of the block.
7319
7320 MachineBasicBlock *SplitBB = BB->splitAt(SplitInst&: MI, UpdateLiveIns: false /*UpdateLiveIns*/);
7321 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
7322 MF->push_back(MBB: TrapBB);
7323 // clang-format off
7324 BuildMI(BB&: *TrapBB, I: TrapBB->end(), MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ENDPGM))
7325 .addImm(Val: 0);
7326 BuildMI(BB&: *BB, I: &MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CBRANCH_EXECNZ))
7327 .addMBB(MBB: TrapBB);
7328 // clang-format on
7329
7330 BB->addSuccessor(Succ: TrapBB);
7331 MI.eraseFromParent();
7332 return SplitBB;
7333 }
7334 case AMDGPU::SIMULATED_TRAP: {
7335 assert(Subtarget->hasPrivEnabledTrap2NopBug());
7336 MachineBasicBlock *SplitBB =
7337 TII->insertSimulatedTrap(MRI, MBB&: *BB, MI, DL: MI.getDebugLoc());
7338 MI.eraseFromParent();
7339 return SplitBB;
7340 }
7341 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
7342 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
7343 assert(MFI->isWholeWaveFunction());
7344
7345 // During ISel, it's difficult to propagate the original EXEC mask to use as
7346 // an input to SI_WHOLE_WAVE_FUNC_RETURN. Set it up here instead.
7347 MachineInstr *Setup = TII->getWholeWaveFunctionSetup(MF&: *BB->getParent());
7348 assert(Setup && "Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
7349 Register OriginalExec = Setup->getOperand(i: 0).getReg();
7350 MF->getRegInfo().clearKillFlags(Reg: OriginalExec);
7351 MI.getOperand(i: 0).setReg(OriginalExec);
7352 return BB;
7353 }
7354 default:
7355 if (TII->isImage(MI) || TII->isMUBUF(MI)) {
7356 if (!MI.mayStore())
7357 AddMemOpInit(MI);
7358 return BB;
7359 }
7360 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, MBB: BB);
7361 }
7362}
7363
7364bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
7365 // This currently forces unfolding various combinations of fsub into fma with
7366 // free fneg'd operands. As long as we have fast FMA (controlled by
7367 // isFMAFasterThanFMulAndFAdd), we should perform these.
7368
7369 // When fma is quarter rate, for f64 where add / sub are at best half rate,
7370 // most of these combines appear to be cycle neutral but save on instruction
7371 // count / code size.
7372 return true;
7373}
7374
7375bool SITargetLowering::enableAggressiveFMAFusion(LLT Ty) const { return true; }
7376
7377EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
7378 EVT VT) const {
7379 if (!VT.isVector()) {
7380 return MVT::i1;
7381 }
7382 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i1, NumElements: VT.getVectorNumElements());
7383}
7384
7385MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
7386 // TODO: Should i16 be used always if legal? For now it would force VALU
7387 // shifts.
7388 return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
7389}
7390
7391LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
7392 return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
7393 ? Ty.changeElementSize(NewEltSize: 16)
7394 : Ty.changeElementSize(NewEltSize: 32);
7395}
7396
7397// Answering this is somewhat tricky and depends on the specific device which
7398// have different rates for fma or all f64 operations.
7399//
7400// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
7401// regardless of which device (although the number of cycles differs between
7402// devices), so it is always profitable for f64.
7403//
7404// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
7405// only on full rate devices. Normally, we should prefer selecting v_mad_f32
7406// which we can always do even without fused FP ops since it returns the same
7407// result as the separate operations and since it is always full
7408// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
7409// however does not support denormals, so we do report fma as faster if we have
7410// a fast fma device and require denormals.
7411//
7412bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7413 EVT VT) const {
7414 VT = VT.getScalarType();
7415
7416 switch (VT.getSimpleVT().SimpleTy) {
7417 case MVT::f32: {
7418 // If mad is not available this depends only on if f32 fma is full rate.
7419 if (!Subtarget->hasMadMacF32Insts())
7420 return Subtarget->hasFastFMAF32();
7421
7422 // Otherwise f32 mad is always full rate and returns the same result as
7423 // the separate operations so should be preferred over fma.
7424 // However does not support denormals.
7425 if (!denormalModeIsFlushAllF32(MF))
7426 return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
7427
7428 // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
7429 return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
7430 }
7431 case MVT::f64:
7432 return true;
7433 case MVT::f16:
7434 case MVT::bf16:
7435 return Subtarget->has16BitInsts() && !denormalModeIsFlushAllF64F16(MF);
7436 default:
7437 break;
7438 }
7439
7440 return false;
7441}
7442
7443bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
7444 LLT Ty) const {
7445 switch (Ty.getScalarSizeInBits()) {
7446 case 16:
7447 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f16);
7448 case 32:
7449 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f32);
7450 case 64:
7451 return isFMAFasterThanFMulAndFAdd(MF, VT: MVT::f64);
7452 default:
7453 break;
7454 }
7455
7456 return false;
7457}
7458
7459bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
7460 if (!Ty.isScalar())
7461 return false;
7462
7463 if (Ty.getScalarSizeInBits() == 16)
7464 return Subtarget->hasMadF16() && denormalModeIsFlushAllF64F16(MF: *MI.getMF());
7465 if (Ty.getScalarSizeInBits() == 32)
7466 return Subtarget->hasMadMacF32Insts() &&
7467 denormalModeIsFlushAllF32(MF: *MI.getMF());
7468
7469 return false;
7470}
7471
7472bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
7473 const SDNode *N) const {
7474 // TODO: Check future ftz flag
7475 // v_mad_f32/v_mac_f32 do not support denormals.
7476 EVT VT = N->getValueType(ResNo: 0);
7477 if (VT == MVT::f32)
7478 return Subtarget->hasMadMacF32Insts() &&
7479 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
7480 if (VT == MVT::f16) {
7481 return Subtarget->hasMadF16() &&
7482 denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction());
7483 }
7484
7485 return false;
7486}
7487
7488//===----------------------------------------------------------------------===//
7489// Custom DAG Lowering Operations
7490//===----------------------------------------------------------------------===//
7491
7492// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7493// wider vector type is legal.
7494SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
7495 SelectionDAG &DAG) const {
7496 unsigned Opc = Op.getOpcode();
7497 EVT VT = Op.getValueType();
7498 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7499
7500 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7501 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
7502
7503 SDLoc SL(Op);
7504
7505 // Forward any trailing scalar operands unchanged to both halves.
7506 SmallVector<SDValue, 2> LoOps = {Lo};
7507 SmallVector<SDValue, 2> HiOps = {Hi};
7508 auto TrailingOps = drop_begin(RangeOrContainer: Op->ops());
7509 LoOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7510 HiOps.append(in_start: TrailingOps.begin(), in_end: TrailingOps.end());
7511
7512 SDValue OpLo = DAG.getNode(Opcode: Opc, DL: SL, VT: LoVT, Ops: LoOps, Flags: Op->getFlags());
7513 SDValue OpHi = DAG.getNode(Opcode: Opc, DL: SL, VT: HiVT, Ops: HiOps, Flags: Op->getFlags());
7514
7515 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7516}
7517
7518// Enable lowering of ROTR for vxi32 types. This is a workaround for a
7519// regression whereby extra unnecessary instructions were added to codegen
7520// for rotr operations, casued by legalising v2i32 or. This resulted in extra
7521// instructions to extract the result from the vector.
7522SDValue SITargetLowering::lowerROTR(SDValue Op, SelectionDAG &DAG) const {
7523 [[maybe_unused]] EVT VT = Op.getValueType();
7524
7525 assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
7526 VT == MVT::v16i32) &&
7527 "Unexpected ValueType.");
7528
7529 return DAG.UnrollVectorOp(N: Op.getNode());
7530}
7531
7532// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
7533// wider vector type is legal.
7534SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
7535 SelectionDAG &DAG) const {
7536 unsigned Opc = Op.getOpcode();
7537 EVT VT = Op.getValueType();
7538 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7539
7540 auto [Lo0, Hi0] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
7541 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7542
7543 SDLoc SL(Op);
7544
7545 SDValue OpLo =
7546 DAG.getNode(Opcode: Opc, DL: SL, VT: Lo0.getValueType(), N1: Lo0, N2: Lo1, Flags: Op->getFlags());
7547 SDValue OpHi =
7548 DAG.getNode(Opcode: Opc, DL: SL, VT: Hi0.getValueType(), N1: Hi0, N2: Hi1, Flags: Op->getFlags());
7549
7550 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7551}
7552
7553SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
7554 SelectionDAG &DAG) const {
7555 unsigned Opc = Op.getOpcode();
7556 EVT VT = Op.getValueType();
7557 assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
7558
7559 SDValue Op0 = Op.getOperand(i: 0);
7560 auto [Lo0, Hi0] = Op0.getValueType().isVector()
7561 ? DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0)
7562 : std::pair(Op0, Op0);
7563
7564 auto [Lo1, Hi1] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 1);
7565 auto [Lo2, Hi2] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 2);
7566
7567 SDLoc SL(Op);
7568 auto ResVT = DAG.GetSplitDestVTs(VT);
7569
7570 SDValue OpLo =
7571 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.first, N1: Lo0, N2: Lo1, N3: Lo2, Flags: Op->getFlags());
7572 SDValue OpHi =
7573 DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT.second, N1: Hi0, N2: Hi1, N3: Hi2, Flags: Op->getFlags());
7574
7575 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(Op), VT, N1: OpLo, N2: OpHi);
7576}
7577
7578SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
7579 switch (Op.getOpcode()) {
7580 default:
7581 return AMDGPUTargetLowering::LowerOperation(Op, DAG);
7582 case ISD::BRCOND:
7583 return LowerBRCOND(Op, DAG);
7584 case ISD::RETURNADDR:
7585 return LowerRETURNADDR(Op, DAG);
7586 case ISD::SPONENTRY:
7587 return LowerSPONENTRY(Op, DAG);
7588 case ISD::LOAD: {
7589 SDValue Result = LowerLOAD(Op, DAG);
7590 assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
7591 "Load should return a value and a chain");
7592 return Result;
7593 }
7594 case ISD::FSQRT: {
7595 EVT VT = Op.getValueType();
7596 if (VT == MVT::f32)
7597 return lowerFSQRTF32(Op, DAG);
7598 if (VT == MVT::f64)
7599 return lowerFSQRTF64(Op, DAG);
7600 return SDValue();
7601 }
7602 case ISD::FSIN:
7603 case ISD::FCOS:
7604 return LowerTrig(Op, DAG);
7605 case ISD::SELECT:
7606 return LowerSELECT(Op, DAG);
7607 case ISD::FDIV:
7608 return LowerFDIV(Op, DAG);
7609 case ISD::FFREXP:
7610 return LowerFFREXP(Op, DAG);
7611 case ISD::ATOMIC_CMP_SWAP:
7612 return LowerATOMIC_CMP_SWAP(Op, DAG);
7613 case ISD::STORE:
7614 return LowerSTORE(Op, DAG);
7615 case ISD::GlobalAddress: {
7616 MachineFunction &MF = DAG.getMachineFunction();
7617 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
7618 return LowerGlobalAddress(MFI, Op, DAG);
7619 }
7620 case ISD::BlockAddress:
7621 return LowerBlockAddress(Op, DAG);
7622 case ISD::ExternalSymbol:
7623 return LowerExternalSymbol(Op, DAG);
7624 case ISD::INTRINSIC_WO_CHAIN:
7625 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7626 case ISD::INTRINSIC_W_CHAIN:
7627 return LowerINTRINSIC_W_CHAIN(Op, DAG);
7628 case ISD::INTRINSIC_VOID:
7629 return LowerINTRINSIC_VOID(Op, DAG);
7630 case ISD::ADDRSPACECAST:
7631 return lowerADDRSPACECAST(Op, DAG);
7632 case ISD::INSERT_SUBVECTOR:
7633 return lowerINSERT_SUBVECTOR(Op, DAG);
7634 case ISD::INSERT_VECTOR_ELT:
7635 return lowerINSERT_VECTOR_ELT(Op, DAG);
7636 case ISD::EXTRACT_VECTOR_ELT:
7637 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
7638 case ISD::VECTOR_SHUFFLE:
7639 return lowerVECTOR_SHUFFLE(Op, DAG);
7640 case ISD::SCALAR_TO_VECTOR:
7641 return lowerSCALAR_TO_VECTOR(Op, DAG);
7642 case ISD::BUILD_VECTOR:
7643 return lowerBUILD_VECTOR(Op, DAG);
7644 case ISD::FP_ROUND:
7645 case ISD::STRICT_FP_ROUND:
7646 return lowerFP_ROUND(Op, DAG);
7647 case ISD::TRAP:
7648 return lowerTRAP(Op, DAG);
7649 case ISD::DEBUGTRAP:
7650 return lowerDEBUGTRAP(Op, DAG);
7651 case ISD::ABS:
7652 case ISD::FABS:
7653 case ISD::FNEG:
7654 case ISD::FCANONICALIZE:
7655 case ISD::BSWAP:
7656 return splitUnaryVectorOp(Op, DAG);
7657 case ISD::FP_TO_SINT_SAT:
7658 case ISD::FP_TO_UINT_SAT:
7659 if (Op.getValueType().isVector() && Op.getValueType() != MVT::v2i16 &&
7660 Op.getOperand(i: 0).getValueType().getScalarType() == MVT::f32)
7661 return splitUnaryVectorOp(Op, DAG);
7662 return LowerFP_TO_INT_SAT(Op, DAG);
7663 case ISD::FMINNUM:
7664 case ISD::FMAXNUM:
7665 return lowerFMINNUM_FMAXNUM(Op, DAG);
7666 case ISD::FMINIMUMNUM:
7667 case ISD::FMAXIMUMNUM:
7668 return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
7669 case ISD::FMINIMUM:
7670 case ISD::FMAXIMUM:
7671 return lowerFMINIMUM_FMAXIMUM(Op, DAG);
7672 case ISD::FLDEXP:
7673 case ISD::STRICT_FLDEXP:
7674 return lowerFLDEXP(Op, DAG);
7675 case ISD::FMA:
7676 return splitTernaryVectorOp(Op, DAG);
7677 case ISD::FP_TO_SINT:
7678 case ISD::FP_TO_UINT:
7679 if (Subtarget->hasVCvtPkIU16F32() && Op.getValueType() == MVT::i16 &&
7680 Op.getOperand(i: 0).getValueType() == MVT::f32) {
7681 // Make f32->i16 legal so we can select V_CVT_PK_[IU]16_F32.
7682 return Op;
7683 }
7684 return LowerFP_TO_INT(Op, DAG);
7685 case ISD::SHL:
7686 case ISD::SRA:
7687 case ISD::SRL:
7688 case ISD::ADD:
7689 case ISD::SUB:
7690 case ISD::SMIN:
7691 case ISD::SMAX:
7692 case ISD::UMIN:
7693 case ISD::UMAX:
7694 case ISD::FMUL:
7695 case ISD::FMINNUM_IEEE:
7696 case ISD::FMAXNUM_IEEE:
7697 case ISD::UADDSAT:
7698 case ISD::USUBSAT:
7699 case ISD::SADDSAT:
7700 case ISD::SSUBSAT:
7701 return splitBinaryVectorOp(Op, DAG);
7702 case ISD::FADD:
7703 if (Op.getValueType() == MVT::bf16)
7704 return lowerScalarBF16FAdd(Op, DAG);
7705 return splitBinaryVectorOp(Op, DAG);
7706 case ISD::FCOPYSIGN:
7707 return lowerFCOPYSIGN(Op, DAG);
7708 case ISD::MUL:
7709 return lowerMUL(Op, DAG);
7710 case ISD::SMULO:
7711 case ISD::UMULO:
7712 return lowerXMULO(Op, DAG);
7713 case ISD::SMUL_LOHI:
7714 case ISD::UMUL_LOHI:
7715 return lowerXMUL_LOHI(Op, DAG);
7716 case ISD::DYNAMIC_STACKALLOC:
7717 return LowerDYNAMIC_STACKALLOC(Op, DAG);
7718 case ISD::STACKSAVE:
7719 return LowerSTACKSAVE(Op, DAG);
7720 case ISD::GET_ROUNDING:
7721 return lowerGET_ROUNDING(Op, DAG);
7722 case ISD::SET_ROUNDING:
7723 return lowerSET_ROUNDING(Op, DAG);
7724 case ISD::PREFETCH:
7725 return lowerPREFETCH(Op, DAG);
7726 case ISD::FP_EXTEND:
7727 case ISD::STRICT_FP_EXTEND:
7728 return lowerFP_EXTEND(Op, DAG);
7729 case ISD::GET_FPENV:
7730 return lowerGET_FPENV(Op, DAG);
7731 case ISD::SET_FPENV:
7732 return lowerSET_FPENV(Op, DAG);
7733 case ISD::ROTR:
7734 return lowerROTR(Op, DAG);
7735 case ISD::INLINEASM:
7736 return LowerINLINEASM(Op, DAG);
7737 }
7738 return SDValue();
7739}
7740
7741// Used for D16: Casts the result of an instruction into the right vector,
7742// packs values if loads return unpacked values.
7743static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
7744 const SDLoc &DL, SelectionDAG &DAG,
7745 bool Unpacked) {
7746 if (!LoadVT.isVector())
7747 return Result;
7748
7749 // Cast back to the original packed type or to a larger type that is a
7750 // multiple of 32 bit for D16. Widening the return type is a required for
7751 // legalization.
7752 EVT FittingLoadVT = LoadVT;
7753 if ((LoadVT.getVectorNumElements() % 2) == 1) {
7754 FittingLoadVT =
7755 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7756 NumElements: LoadVT.getVectorNumElements() + 1);
7757 }
7758
7759 if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
7760 // Truncate to v2i16/v4i16.
7761 EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
7762
7763 // Workaround legalizer not scalarizing truncate after vector op
7764 // legalization but not creating intermediate vector trunc.
7765 SmallVector<SDValue, 4> Elts;
7766 DAG.ExtractVectorElements(Op: Result, Args&: Elts);
7767 for (SDValue &Elt : Elts)
7768 Elt = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Elt);
7769
7770 // Pad illegal v1i16/v3fi6 to v4i16
7771 if ((LoadVT.getVectorNumElements() % 2) == 1)
7772 Elts.push_back(Elt: DAG.getPOISON(VT: MVT::i16));
7773
7774 Result = DAG.getBuildVector(VT: IntLoadVT, DL, Ops: Elts);
7775
7776 // Bitcast to original type (v2f16/v4f16).
7777 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7778 }
7779
7780 // Cast back to the original packed type.
7781 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FittingLoadVT, Operand: Result);
7782}
7783
7784SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
7785 SelectionDAG &DAG,
7786 ArrayRef<SDValue> Ops,
7787 bool IsIntrinsic) const {
7788 SDLoc DL(M);
7789
7790 bool Unpacked = Subtarget->hasUnpackedD16VMem();
7791 EVT LoadVT = M->getValueType(ResNo: 0);
7792
7793 EVT EquivLoadVT = LoadVT;
7794 if (LoadVT.isVector()) {
7795 if (Unpacked) {
7796 EquivLoadVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
7797 NumElements: LoadVT.getVectorNumElements());
7798 } else if ((LoadVT.getVectorNumElements() % 2) == 1) {
7799 // Widen v3f16 to legal type
7800 EquivLoadVT =
7801 EVT::getVectorVT(Context&: *DAG.getContext(), VT: LoadVT.getVectorElementType(),
7802 NumElements: LoadVT.getVectorNumElements() + 1);
7803 }
7804 }
7805
7806 // Change from v4f16/v2f16 to EquivLoadVT.
7807 SDVTList VTList = DAG.getVTList(VT1: EquivLoadVT, VT2: MVT::Other);
7808
7809 SDValue Load = DAG.getMemIntrinsicNode(
7810 Opcode: IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, dl: DL, VTList, Ops,
7811 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
7812
7813 SDValue Adjusted = adjustLoadValueTypeImpl(Result: Load, LoadVT, DL, DAG, Unpacked);
7814
7815 return DAG.getMergeValues(Ops: {Adjusted, Load.getValue(R: 1)}, dl: DL);
7816}
7817
7818SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
7819 SelectionDAG &DAG,
7820 ArrayRef<SDValue> Ops) const {
7821 SDLoc DL(M);
7822 EVT LoadVT = M->getValueType(ResNo: 0);
7823 EVT EltType = LoadVT.getScalarType();
7824 EVT IntVT = LoadVT.changeTypeToInteger();
7825
7826 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
7827
7828 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
7829 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
7830 DAG.getMachineFunction().getFunction(),
7831 "unsupported sub-dword format buffer load", DL.getDebugLoc()));
7832 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: LoadVT), M->getOperand(Num: 0)}, dl: DL);
7833 }
7834
7835 assert(M->getNumValues() == 2 || M->getNumValues() == 3);
7836 bool IsTFE = M->getNumValues() == 3;
7837
7838 unsigned Opc = IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
7839 : AMDGPUISD::BUFFER_LOAD_FORMAT)
7840 : IsTFE ? AMDGPUISD::BUFFER_LOAD_TFE
7841 : AMDGPUISD::BUFFER_LOAD;
7842
7843 if (IsD16) {
7844 return adjustLoadValueType(Opcode: AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops);
7845 }
7846
7847 // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
7848 if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
7849 return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, MMO: M->getMemOperand(),
7850 IsTFE);
7851
7852 if (isTypeLegal(VT: LoadVT)) {
7853 return getMemIntrinsicNode(Opcode: Opc, DL, VTList: M->getVTList(), Ops, MemVT: IntVT,
7854 MMO: M->getMemOperand(), DAG);
7855 }
7856
7857 EVT CastVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: LoadVT);
7858 SDVTList VTList = DAG.getVTList(VT1: CastVT, VT2: MVT::Other);
7859 SDValue MemNode = getMemIntrinsicNode(Opcode: Opc, DL, VTList, Ops, MemVT: CastVT,
7860 MMO: M->getMemOperand(), DAG);
7861 return DAG.getMergeValues(
7862 Ops: {DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: MemNode), MemNode.getValue(R: 1)},
7863 dl: DL);
7864}
7865
7866static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, SDNode *N,
7867 SelectionDAG &DAG) {
7868 EVT VT = N->getValueType(ResNo: 0);
7869 unsigned CondCode = N->getConstantOperandVal(Num: 3);
7870 if (!ICmpInst::isIntPredicate(P: static_cast<ICmpInst::Predicate>(CondCode)))
7871 return DAG.getPOISON(VT);
7872
7873 ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode);
7874
7875 SDValue LHS = N->getOperand(Num: 1);
7876 SDValue RHS = N->getOperand(Num: 2);
7877
7878 SDLoc DL(N);
7879
7880 EVT CmpVT = LHS.getValueType();
7881 if (CmpVT == MVT::i16 && !TLI.isTypeLegal(VT: MVT::i16)) {
7882 unsigned PromoteOp =
7883 ICmpInst::isSigned(Pred: IcInput) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
7884 LHS = DAG.getNode(Opcode: PromoteOp, DL, VT: MVT::i32, Operand: LHS);
7885 RHS = DAG.getNode(Opcode: PromoteOp, DL, VT: MVT::i32, Operand: RHS);
7886 }
7887
7888 ISD::CondCode CCOpcode = getICmpCondCode(Pred: IcInput);
7889
7890 unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
7891 EVT CCVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WavefrontSize);
7892
7893 SDValue SetCC = DAG.getNode(Opcode: AMDGPUISD::SETCC, DL, VT: CCVT, N1: LHS, N2: RHS,
7894 N3: DAG.getCondCode(Cond: CCOpcode));
7895 if (VT.bitsEq(VT: CCVT))
7896 return SetCC;
7897 return DAG.getZExtOrTrunc(Op: SetCC, DL, VT);
7898}
7899
7900static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI, SDNode *N,
7901 SelectionDAG &DAG) {
7902 EVT VT = N->getValueType(ResNo: 0);
7903
7904 unsigned CondCode = N->getConstantOperandVal(Num: 3);
7905 if (!FCmpInst::isFPPredicate(P: static_cast<FCmpInst::Predicate>(CondCode)))
7906 return DAG.getPOISON(VT);
7907
7908 SDValue Src0 = N->getOperand(Num: 1);
7909 SDValue Src1 = N->getOperand(Num: 2);
7910 EVT CmpVT = Src0.getValueType();
7911 SDLoc SL(N);
7912
7913 if (CmpVT == MVT::f16 && !TLI.isTypeLegal(VT: CmpVT)) {
7914 Src0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src0);
7915 Src1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src1);
7916 }
7917
7918 FCmpInst::Predicate IcInput = static_cast<FCmpInst::Predicate>(CondCode);
7919 ISD::CondCode CCOpcode = getFCmpCondCode(Pred: IcInput);
7920 unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
7921 EVT CCVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WavefrontSize);
7922 SDValue SetCC = DAG.getNode(Opcode: AMDGPUISD::SETCC, DL: SL, VT: CCVT, N1: Src0, N2: Src1,
7923 N3: DAG.getCondCode(Cond: CCOpcode));
7924 if (VT.bitsEq(VT: CCVT))
7925 return SetCC;
7926 return DAG.getZExtOrTrunc(Op: SetCC, DL: SL, VT);
7927}
7928
7929static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
7930 SelectionDAG &DAG) {
7931 EVT VT = N->getValueType(ResNo: 0);
7932 SDValue Src = N->getOperand(Num: 1);
7933 SDLoc SL(N);
7934
7935 if (Src.getOpcode() == ISD::SETCC) {
7936 SDValue Op0 = Src.getOperand(i: 0);
7937 SDValue Op1 = Src.getOperand(i: 1);
7938 // Need to expand bfloat to float for comparison (setcc).
7939 if (Op0.getValueType() == MVT::bf16) {
7940 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op0);
7941 Op1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op1);
7942 }
7943 // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
7944 return DAG.getNode(Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: Op0, N2: Op1, N3: Src.getOperand(i: 2));
7945 }
7946 if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Val&: Src)) {
7947 // (ballot 0) -> 0
7948 if (Arg->isZero())
7949 return DAG.getConstant(Val: 0, DL: SL, VT);
7950
7951 // (ballot 1) -> EXEC/EXEC_LO
7952 if (Arg->isOne()) {
7953 Register Exec;
7954 if (VT.getScalarSizeInBits() == 32)
7955 Exec = AMDGPU::EXEC_LO;
7956 else if (VT.getScalarSizeInBits() == 64)
7957 Exec = AMDGPU::EXEC;
7958 else
7959 return SDValue();
7960
7961 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: Exec, VT);
7962 }
7963 }
7964
7965 // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
7966 // ISD::SETNE)
7967 return DAG.getNode(
7968 Opcode: AMDGPUISD::SETCC, DL: SL, VT, N1: DAG.getZExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32),
7969 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), N3: DAG.getCondCode(Cond: ISD::SETNE));
7970}
7971
7972static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
7973 EVT VT);
7974
7975static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
7976 SelectionDAG &DAG) {
7977 EVT VT = N->getValueType(ResNo: 0);
7978 unsigned ValSize = VT.getSizeInBits();
7979 unsigned IID = N->getConstantOperandVal(Num: 0);
7980 bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
7981 IID == Intrinsic::amdgcn_permlanex16;
7982 bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
7983 IID == Intrinsic::amdgcn_set_inactive_chain_arg;
7984 bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
7985 IID == Intrinsic::amdgcn_permlane_up ||
7986 IID == Intrinsic::amdgcn_permlane_down ||
7987 IID == Intrinsic::amdgcn_permlane_xor;
7988 SDLoc SL(N);
7989 MVT IntVT = MVT::getIntegerVT(BitWidth: ValSize);
7990 const GCNSubtarget *ST = TLI.getSubtarget();
7991
7992 unsigned SplitSize = 32;
7993 if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
7994 ST->hasDPALU_DPP() &&
7995 AMDGPU::isLegalDPALU_DPPControl(ST: *ST, DC: N->getConstantOperandVal(Num: 3)))
7996 SplitSize = 64;
7997
7998 auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
7999 SDValue Src2, MVT ValT) -> SDValue {
8000 SmallVector<SDValue, 8> Operands;
8001 switch (IID) {
8002 case Intrinsic::amdgcn_permlane16:
8003 case Intrinsic::amdgcn_permlanex16:
8004 case Intrinsic::amdgcn_update_dpp:
8005 Operands.push_back(Elt: N->getOperand(Num: 6));
8006 Operands.push_back(Elt: N->getOperand(Num: 5));
8007 Operands.push_back(Elt: N->getOperand(Num: 4));
8008 [[fallthrough]];
8009 case Intrinsic::amdgcn_writelane:
8010 case Intrinsic::amdgcn_permlane_bcast:
8011 case Intrinsic::amdgcn_permlane_up:
8012 case Intrinsic::amdgcn_permlane_down:
8013 case Intrinsic::amdgcn_permlane_xor:
8014 Operands.push_back(Elt: Src2);
8015 [[fallthrough]];
8016 case Intrinsic::amdgcn_readlane:
8017 case Intrinsic::amdgcn_set_inactive:
8018 case Intrinsic::amdgcn_set_inactive_chain_arg:
8019 case Intrinsic::amdgcn_mov_dpp8:
8020 Operands.push_back(Elt: Src1);
8021 [[fallthrough]];
8022 case Intrinsic::amdgcn_readfirstlane:
8023 case Intrinsic::amdgcn_permlane64:
8024 Operands.push_back(Elt: Src0);
8025 break;
8026 default:
8027 llvm_unreachable("unhandled lane op");
8028 }
8029
8030 Operands.push_back(Elt: DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32));
8031 std::reverse(first: Operands.begin(), last: Operands.end());
8032
8033 if (SDNode *GL = N->getGluedNode()) {
8034 assert(GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8035 GL = GL->getOperand(Num: 0).getNode();
8036 Operands.push_back(Elt: DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8037 Operand: SDValue(GL, 0)));
8038 }
8039
8040 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: ValT, Ops: Operands);
8041 };
8042
8043 SDValue Src0 = N->getOperand(Num: 1);
8044 SDValue Src1, Src2;
8045 if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
8046 IID == Intrinsic::amdgcn_mov_dpp8 ||
8047 IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
8048 IsPermlaneShuffle) {
8049 Src1 = N->getOperand(Num: 2);
8050 if (IID == Intrinsic::amdgcn_writelane ||
8051 IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
8052 IsPermlaneShuffle)
8053 Src2 = N->getOperand(Num: 3);
8054 }
8055
8056 if (ValSize == SplitSize) {
8057 // Already legal
8058 return SDValue();
8059 }
8060
8061 if (ValSize < 32) {
8062 bool IsFloat = VT.isFloatingPoint();
8063 Src0 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src0) : Src0,
8064 DL: SL, VT: MVT::i32);
8065
8066 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
8067 Src1 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src1) : Src1,
8068 DL: SL, VT: MVT::i32);
8069 }
8070
8071 if (IID == Intrinsic::amdgcn_writelane) {
8072 Src2 = DAG.getAnyExtOrTrunc(Op: IsFloat ? DAG.getBitcast(VT: IntVT, V: Src2) : Src2,
8073 DL: SL, VT: MVT::i32);
8074 }
8075
8076 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
8077 SDValue Trunc = DAG.getAnyExtOrTrunc(Op: LaneOp, DL: SL, VT: IntVT);
8078 return IsFloat ? DAG.getBitcast(VT, V: Trunc) : Trunc;
8079 }
8080
8081 if (ValSize % SplitSize != 0)
8082 return SDValue();
8083
8084 auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
8085 EVT VT = N->getValueType(ResNo: 0);
8086 unsigned NE = VT.getVectorNumElements();
8087 EVT EltVT = VT.getVectorElementType();
8088 SmallVector<SDValue, 8> Scalars;
8089 unsigned NumOperands = N->getNumOperands();
8090 SmallVector<SDValue, 4> Operands(NumOperands);
8091 SDNode *GL = N->getGluedNode();
8092
8093 // only handle convergencectrl_glue
8094 assert(!GL || GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
8095
8096 for (unsigned i = 0; i != NE; ++i) {
8097 for (unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
8098 ++j) {
8099 SDValue Operand = N->getOperand(Num: j);
8100 EVT OperandVT = Operand.getValueType();
8101 if (OperandVT.isVector()) {
8102 // A vector operand; extract a single element.
8103 EVT OperandEltVT = OperandVT.getVectorElementType();
8104 Operands[j] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: OperandEltVT,
8105 N1: Operand, N2: DAG.getVectorIdxConstant(Val: i, DL: SL));
8106 } else {
8107 // A scalar operand; just use it as is.
8108 Operands[j] = Operand;
8109 }
8110 }
8111
8112 if (GL)
8113 Operands[NumOperands - 1] =
8114 DAG.getNode(Opcode: ISD::CONVERGENCECTRL_GLUE, DL: SL, VT: MVT::Glue,
8115 Operand: SDValue(GL->getOperand(Num: 0).getNode(), 0));
8116
8117 Scalars.push_back(Elt: DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: EltVT, Ops: Operands));
8118 }
8119
8120 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NE);
8121 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops: Scalars);
8122 };
8123
8124 if (VT.isVector()) {
8125 switch (MVT::SimpleValueType EltTy =
8126 VT.getVectorElementType().getSimpleVT().SimpleTy) {
8127 case MVT::i32:
8128 case MVT::f32:
8129 if (SplitSize == 32) {
8130 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT());
8131 return unrollLaneOp(LaneOp.getNode());
8132 }
8133 [[fallthrough]];
8134 case MVT::i16:
8135 case MVT::f16:
8136 case MVT::bf16: {
8137 unsigned SubVecNumElt =
8138 SplitSize / VT.getVectorElementType().getSizeInBits();
8139 MVT SubVecVT = MVT::getVectorVT(VT: EltTy, NumElements: SubVecNumElt);
8140 SmallVector<SDValue, 4> Pieces;
8141 SDValue Src0SubVec, Src1SubVec, Src2SubVec;
8142 for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
8143 Src0SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src0,
8144 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8145
8146 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
8147 IsPermLane16) {
8148 Src1SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src1,
8149 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8150
8151 Pieces.push_back(
8152 Elt: createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
8153 } else if (IID == Intrinsic::amdgcn_writelane) {
8154 Src2SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: SubVecVT, N1: Src2,
8155 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
8156 Pieces.push_back(
8157 Elt: createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
8158 } else {
8159 Pieces.push_back(Elt: createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
8160 }
8161
8162 EltIdx += SubVecNumElt;
8163 }
8164 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, Ops: Pieces);
8165 }
8166 default:
8167 // Handle all other cases by bitcasting to i32 vectors
8168 break;
8169 }
8170 }
8171
8172 MVT VecVT =
8173 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SplitSize), NumElements: ValSize / SplitSize);
8174 Src0 = DAG.getBitcast(VT: VecVT, V: Src0);
8175
8176 if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
8177 Src1 = DAG.getBitcast(VT: VecVT, V: Src1);
8178
8179 if (IID == Intrinsic::amdgcn_writelane)
8180 Src2 = DAG.getBitcast(VT: VecVT, V: Src2);
8181
8182 SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
8183 SDValue UnrolledLaneOp = unrollLaneOp(LaneOp.getNode());
8184 return DAG.getBitcast(VT, V: UnrolledLaneOp);
8185}
8186
8187static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N,
8188 SelectionDAG &DAG) {
8189 EVT VT = N->getValueType(ResNo: 0);
8190
8191 if (VT.getSizeInBits() != 32)
8192 return SDValue();
8193
8194 SDLoc SL(N);
8195
8196 SDValue Value = N->getOperand(Num: 1);
8197 SDValue Index = N->getOperand(Num: 2);
8198
8199 // ds_bpermute requires index to be multiplied by 4
8200 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: 2, VT: MVT::i32, DL: SL);
8201 SDValue ShiftedIndex =
8202 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: Index.getValueType(), N1: Index, N2: ShiftAmount);
8203
8204 // Intrinsics will require i32 to operate on
8205 SDValue ValueI32 = DAG.getBitcast(VT: MVT::i32, V: Value);
8206
8207 auto MakeIntrinsic = [&DAG, &SL](unsigned IID, MVT RetVT,
8208 SmallVector<SDValue> IntrinArgs) -> SDValue {
8209 SmallVector<SDValue> Operands(1);
8210 Operands[0] = DAG.getTargetConstant(Val: IID, DL: SL, VT: MVT::i32);
8211 Operands.append(RHS: IntrinArgs);
8212 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: RetVT, Ops: Operands);
8213 };
8214
8215 // If we can bpermute across the whole wave, then just do that
8216 if (TLI.getSubtarget()->supportsWaveWideBPermute()) {
8217 SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8218 {ShiftedIndex, ValueI32});
8219 return DAG.getBitcast(VT, V: BPermute);
8220 }
8221
8222 assert(TLI.getSubtarget()->isWave64());
8223
8224 // Otherwise, we need to make use of whole wave mode
8225 SDValue PoisonVal = DAG.getPOISON(VT: ValueI32->getValueType(ResNo: 0));
8226
8227 // Set inactive lanes to poison
8228 SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8229 {ValueI32, PoisonVal});
8230 SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
8231 {ShiftedIndex, PoisonVal});
8232
8233 SDValue Swapped =
8234 MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
8235
8236 // Get permutation of each half, then we'll select which one to use
8237 SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
8238 {WWMIndex, WWMValue});
8239 SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
8240 MVT::i32, {WWMIndex, Swapped});
8241 SDValue BPermOtherHalfWWM =
8242 MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
8243
8244 // Select which side to take the permute from
8245 SDValue ThreadIDMask = DAG.getAllOnesConstant(DL: SL, VT: MVT::i32);
8246 // We can get away with only using mbcnt_lo here since we're only
8247 // trying to detect which side of 32 each lane is on, and mbcnt_lo
8248 // returns 32 for lanes 32-63.
8249 SDValue ThreadID =
8250 MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
8251 {ThreadIDMask, DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32)});
8252
8253 SDValue SameOrOtherHalf =
8254 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32,
8255 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: ThreadID, N2: Index),
8256 N2: DAG.getTargetConstant(Val: 32, DL: SL, VT: MVT::i32));
8257 SDValue UseSameHalf =
8258 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SameOrOtherHalf,
8259 RHS: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond: ISD::SETEQ);
8260 SDValue Result = DAG.getSelect(DL: SL, VT: MVT::i32, Cond: UseSameHalf, LHS: BPermSameHalf,
8261 RHS: BPermOtherHalfWWM);
8262 return DAG.getBitcast(VT, V: Result);
8263}
8264
8265void SITargetLowering::ReplaceNodeResults(SDNode *N,
8266 SmallVectorImpl<SDValue> &Results,
8267 SelectionDAG &DAG) const {
8268 switch (N->getOpcode()) {
8269 case ISD::INSERT_VECTOR_ELT: {
8270 if (SDValue Res = lowerINSERT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8271 Results.push_back(Elt: Res);
8272 return;
8273 }
8274 case ISD::EXTRACT_VECTOR_ELT: {
8275 if (SDValue Res = lowerEXTRACT_VECTOR_ELT(Op: SDValue(N, 0), DAG))
8276 Results.push_back(Elt: Res);
8277 return;
8278 }
8279 case ISD::INTRINSIC_WO_CHAIN: {
8280 unsigned IID = N->getConstantOperandVal(Num: 0);
8281 switch (IID) {
8282 case Intrinsic::amdgcn_wave_reduce_min:
8283 case Intrinsic::amdgcn_wave_reduce_umin:
8284 case Intrinsic::amdgcn_wave_reduce_max:
8285 case Intrinsic::amdgcn_wave_reduce_umax:
8286 case Intrinsic::amdgcn_wave_reduce_add:
8287 case Intrinsic::amdgcn_wave_reduce_sub:
8288 case Intrinsic::amdgcn_wave_reduce_and:
8289 case Intrinsic::amdgcn_wave_reduce_or:
8290 case Intrinsic::amdgcn_wave_reduce_xor: {
8291 EVT VT = N->getValueType(ResNo: 0);
8292 if (isTypeLegal(VT))
8293 return;
8294 SDLoc SL(N);
8295 bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
8296 IID == Intrinsic::amdgcn_wave_reduce_max ||
8297 IID == Intrinsic::amdgcn_wave_reduce_add ||
8298 IID == Intrinsic::amdgcn_wave_reduce_sub;
8299 unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8300 SDValue ExtSrc = DAG.getNode(Opcode: ExtOpc, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 1));
8301 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
8302 N1: N->getOperand(Num: 0), N2: ExtSrc, N3: N->getOperand(Num: 2));
8303 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Result));
8304 return;
8305 }
8306 case Intrinsic::amdgcn_make_buffer_rsrc:
8307 Results.push_back(Elt: lowerPointerAsRsrcIntrin(Op: N, DAG));
8308 return;
8309 case Intrinsic::amdgcn_cvt_pkrtz: {
8310 SDValue Src0 = N->getOperand(Num: 1);
8311 SDValue Src1 = N->getOperand(Num: 2);
8312 SDLoc SL(N);
8313 SDValue Cvt =
8314 DAG.getNode(Opcode: AMDGPUISD::CVT_PKRTZ_F16_F32, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8315 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Cvt));
8316 return;
8317 }
8318 case Intrinsic::amdgcn_cvt_pknorm_i16:
8319 case Intrinsic::amdgcn_cvt_pknorm_u16:
8320 case Intrinsic::amdgcn_cvt_pk_i16:
8321 case Intrinsic::amdgcn_cvt_pk_u16: {
8322 SDValue Src0 = N->getOperand(Num: 1);
8323 SDValue Src1 = N->getOperand(Num: 2);
8324 SDLoc SL(N);
8325 unsigned Opcode;
8326
8327 if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
8328 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
8329 else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
8330 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
8331 else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
8332 Opcode = AMDGPUISD::CVT_PK_I16_I32;
8333 else
8334 Opcode = AMDGPUISD::CVT_PK_U16_U32;
8335
8336 EVT VT = N->getValueType(ResNo: 0);
8337 if (isTypeLegal(VT))
8338 Results.push_back(Elt: DAG.getNode(Opcode, DL: SL, VT, N1: Src0, N2: Src1));
8339 else {
8340 SDValue Cvt = DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Src0, N2: Src1);
8341 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: Cvt));
8342 }
8343 return;
8344 }
8345 case Intrinsic::amdgcn_s_buffer_load: {
8346 // Lower llvm.amdgcn.s.buffer.load.(i8, u8) intrinsics. First, we generate
8347 // s_buffer_load_u8 for signed and unsigned load instructions. Next, DAG
8348 // combiner tries to merge the s_buffer_load_u8 with a sext instruction
8349 // (performSignExtendInRegCombine()) and it replaces s_buffer_load_u8 with
8350 // s_buffer_load_i8.
8351 if (!Subtarget->hasScalarSubwordLoads())
8352 return;
8353 SDValue Op = SDValue(N, 0);
8354 SDValue Rsrc = Op.getOperand(i: 1);
8355 SDValue Offset = Op.getOperand(i: 2);
8356 SDValue CachePolicy = Op.getOperand(i: 3);
8357 EVT VT = Op.getValueType();
8358 assert(VT == MVT::i8 && "Expected 8-bit s_buffer_load intrinsics.\n");
8359 SDLoc DL(Op);
8360 MachineFunction &MF = DAG.getMachineFunction();
8361 const DataLayout &DataLayout = DAG.getDataLayout();
8362 Align Alignment =
8363 DataLayout.getABITypeAlign(Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
8364 MachineMemOperand *MMO = MF.getMachineMemOperand(
8365 PtrInfo: MachinePointerInfo(),
8366 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
8367 MachineMemOperand::MOInvariant,
8368 Size: VT.getStoreSize(), BaseAlignment: Alignment);
8369 SDValue LoadVal;
8370 if (!Offset->isDivergent()) {
8371 SDValue Ops[] = {Rsrc, // source register
8372 Offset, CachePolicy};
8373 SDValue BufferLoad =
8374 DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD_UBYTE, dl: DL,
8375 VTList: DAG.getVTList(VT: MVT::i32), Ops, MemVT: VT, MMO);
8376 LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: BufferLoad);
8377 } else {
8378 SDValue Ops[] = {
8379 DAG.getEntryNode(), // Chain
8380 Rsrc, // rsrc
8381 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
8382 {}, // voffset
8383 {}, // soffset
8384 {}, // offset
8385 CachePolicy, // cachepolicy
8386 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
8387 };
8388 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
8389 LoadVal = handleByteShortBufferLoads(DAG, LoadVT: VT, DL, Ops, MMO);
8390 }
8391 Results.push_back(Elt: LoadVal);
8392 return;
8393 }
8394 case Intrinsic::amdgcn_dead: {
8395 for (unsigned I = 0, E = N->getNumValues(); I < E; ++I)
8396 Results.push_back(Elt: DAG.getPOISON(VT: N->getValueType(ResNo: I)));
8397 return;
8398 }
8399 }
8400 break;
8401 }
8402 case ISD::INTRINSIC_W_CHAIN: {
8403 if (SDValue Res = LowerINTRINSIC_W_CHAIN(Op: SDValue(N, 0), DAG)) {
8404 if (Res.getOpcode() == ISD::MERGE_VALUES) {
8405 // FIXME: Hacky
8406 for (unsigned I = 0; I < Res.getNumOperands(); I++) {
8407 Results.push_back(Elt: Res.getOperand(i: I));
8408 }
8409 } else {
8410 Results.push_back(Elt: Res);
8411 Results.push_back(Elt: Res.getValue(R: 1));
8412 }
8413 return;
8414 }
8415
8416 break;
8417 }
8418 case ISD::SELECT: {
8419 SDLoc SL(N);
8420 EVT VT = N->getValueType(ResNo: 0);
8421 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT);
8422 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 1));
8423 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: N->getOperand(Num: 2));
8424
8425 EVT SelectVT = NewVT;
8426 if (NewVT.bitsLT(VT: MVT::i32)) {
8427 LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: LHS);
8428 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: RHS);
8429 SelectVT = MVT::i32;
8430 }
8431
8432 SDValue NewSelect =
8433 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: SelectVT, N1: N->getOperand(Num: 0), N2: LHS, N3: RHS);
8434
8435 if (NewVT != SelectVT)
8436 NewSelect = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: NewVT, Operand: NewSelect);
8437 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewSelect));
8438 return;
8439 }
8440 case ISD::FNEG: {
8441 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8442 break;
8443
8444 SDLoc SL(N);
8445 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8446
8447 SDValue Op = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: BC,
8448 N2: DAG.getConstant(Val: 0x80008000, DL: SL, VT: MVT::i32));
8449 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8450 return;
8451 }
8452 case ISD::FABS: {
8453 if (N->getValueType(ResNo: 0) != MVT::v2f16)
8454 break;
8455
8456 SDLoc SL(N);
8457 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: N->getOperand(Num: 0));
8458
8459 SDValue Op = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: BC,
8460 N2: DAG.getConstant(Val: 0x7fff7fff, DL: SL, VT: MVT::i32));
8461 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2f16, Operand: Op));
8462 return;
8463 }
8464 case ISD::FSQRT: {
8465 if (N->getValueType(ResNo: 0) != MVT::f16)
8466 break;
8467 Results.push_back(Elt: lowerFSQRTF16(Op: SDValue(N, 0), DAG));
8468 break;
8469 }
8470 default:
8471 AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
8472 break;
8473 }
8474}
8475
8476/// Helper function for LowerBRCOND
8477static SDNode *findUser(SDValue Value, unsigned Opcode) {
8478
8479 for (SDUse &U : Value->uses()) {
8480 if (U.get() != Value)
8481 continue;
8482
8483 if (U.getUser()->getOpcode() == Opcode)
8484 return U.getUser();
8485 }
8486 return nullptr;
8487}
8488
8489unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
8490 if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
8491 switch (Intr->getConstantOperandVal(Num: 1)) {
8492 case Intrinsic::amdgcn_if:
8493 return AMDGPUISD::IF;
8494 case Intrinsic::amdgcn_else:
8495 return AMDGPUISD::ELSE;
8496 case Intrinsic::amdgcn_loop:
8497 return AMDGPUISD::LOOP;
8498 case Intrinsic::amdgcn_end_cf:
8499 llvm_unreachable("should not occur");
8500 default:
8501 return 0;
8502 }
8503 }
8504
8505 // break, if_break, else_break are all only used as inputs to loop, not
8506 // directly as branch conditions.
8507 return 0;
8508}
8509
8510bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
8511 const Triple &TT = getTargetMachine().getTargetTriple();
8512 return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
8513 GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
8514 AMDGPU::shouldEmitConstantsToTextSection(TT);
8515}
8516
8517bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
8518 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
8519 return false;
8520
8521 // FIXME: Either avoid relying on address space here or change the default
8522 // address space for functions to avoid the explicit check.
8523 return (GV->getValueType()->isFunctionTy() ||
8524 !isNonGlobalAddrSpace(AS: GV->getAddressSpace())) &&
8525 !shouldEmitFixup(GV) && !getTargetMachine().shouldAssumeDSOLocal(GV);
8526}
8527
8528bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
8529 return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
8530}
8531
8532bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const {
8533 if (!GV->hasExternalLinkage())
8534 return true;
8535
8536 // With object linking, external LDS declarations need relocations so the
8537 // linker can assign their offsets.
8538 if (AMDGPUTargetMachine::EnableObjectLinking) {
8539 if (const auto *GVar = dyn_cast<GlobalVariable>(Val: GV)) {
8540 if (GVar->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
8541 assert(GVar->isDeclaration() && "AS3 GVs should be declaration here "
8542 "when object linking is enabled");
8543 return false;
8544 }
8545 }
8546 }
8547
8548 const auto OS = getTargetMachine().getTargetTriple().getOS();
8549 return OS == Triple::AMDHSA || OS == Triple::AMDPAL;
8550}
8551
8552/// This transforms the control flow intrinsics to get the branch destination as
8553/// last parameter, also switches branch target with BR if the need arise
8554SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, SelectionDAG &DAG) const {
8555 SDLoc DL(BRCOND);
8556
8557 SDNode *Intr = BRCOND.getOperand(i: 1).getNode();
8558 SDValue Target = BRCOND.getOperand(i: 2);
8559 SDNode *BR = nullptr;
8560 SDNode *SetCC = nullptr;
8561
8562 switch (Intr->getOpcode()) {
8563 case ISD::SETCC: {
8564 // As long as we negate the condition everything is fine
8565 SetCC = Intr;
8566 Intr = SetCC->getOperand(Num: 0).getNode();
8567 break;
8568 }
8569 case ISD::XOR: {
8570 // Similar to SETCC, if we have (xor c, -1), we will be fine.
8571 SDValue LHS = Intr->getOperand(Num: 0);
8572 SDValue RHS = Intr->getOperand(Num: 1);
8573 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS); C && C->getZExtValue()) {
8574 Intr = LHS.getNode();
8575 break;
8576 }
8577 [[fallthrough]];
8578 }
8579 default: {
8580 // Get the target from BR if we don't negate the condition
8581 BR = findUser(Value: BRCOND, Opcode: ISD::BR);
8582 assert(BR && "brcond missing unconditional branch user");
8583 Target = BR->getOperand(Num: 1);
8584 }
8585 }
8586
8587 unsigned CFNode = isCFIntrinsic(Intr);
8588 if (CFNode == 0) {
8589 // This is a uniform branch so we don't need to legalize.
8590 return BRCOND;
8591 }
8592
8593 bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
8594 Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
8595
8596 assert(!SetCC ||
8597 (SetCC->getConstantOperandVal(1) == 1 &&
8598 cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
8599 ISD::SETNE));
8600
8601 // operands of the new intrinsic call
8602 SmallVector<SDValue, 4> Ops;
8603 if (HaveChain)
8604 Ops.push_back(Elt: BRCOND.getOperand(i: 0));
8605
8606 Ops.append(in_start: Intr->op_begin() + (HaveChain ? 2 : 1), in_end: Intr->op_end());
8607 Ops.push_back(Elt: Target);
8608
8609 ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
8610
8611 // build the new intrinsic call
8612 SDNode *Result = DAG.getNode(Opcode: CFNode, DL, VTList: DAG.getVTList(VTs: Res), Ops).getNode();
8613
8614 if (!HaveChain) {
8615 SDValue Ops[] = {SDValue(Result, 0), BRCOND.getOperand(i: 0)};
8616
8617 Result = DAG.getMergeValues(Ops, dl: DL).getNode();
8618 }
8619
8620 if (BR) {
8621 // Give the branch instruction our target
8622 SDValue Ops[] = {BR->getOperand(Num: 0), BRCOND.getOperand(i: 2)};
8623 SDValue NewBR = DAG.getNode(Opcode: ISD::BR, DL, VTList: BR->getVTList(), Ops);
8624 DAG.ReplaceAllUsesWith(From: BR, To: NewBR.getNode());
8625 }
8626
8627 SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
8628
8629 // Copy the intrinsic results to registers
8630 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) {
8631 SDNode *CopyToReg = findUser(Value: SDValue(Intr, i), Opcode: ISD::CopyToReg);
8632 if (!CopyToReg)
8633 continue;
8634
8635 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: CopyToReg->getOperand(Num: 1),
8636 N: SDValue(Result, i - 1), Glue: SDValue());
8637
8638 DAG.ReplaceAllUsesWith(From: SDValue(CopyToReg, 0), To: CopyToReg->getOperand(Num: 0));
8639 }
8640
8641 // Remove the old intrinsic from the chain
8642 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Intr, Intr->getNumValues() - 1),
8643 To: Intr->getOperand(Num: 0));
8644
8645 return Chain;
8646}
8647
8648SDValue SITargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const {
8649 MVT VT = Op.getSimpleValueType();
8650 SDLoc DL(Op);
8651 // Checking the depth
8652 if (Op.getConstantOperandVal(i: 0) != 0)
8653 return DAG.getConstant(Val: 0, DL, VT);
8654
8655 MachineFunction &MF = DAG.getMachineFunction();
8656 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8657 // Check for kernel and shader functions
8658 if (Info->isEntryFunction())
8659 return DAG.getConstant(Val: 0, DL, VT);
8660
8661 MachineFrameInfo &MFI = MF.getFrameInfo();
8662 // There is a call to @llvm.returnaddress in this function
8663 MFI.setReturnAddressIsTaken(true);
8664
8665 const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
8666 // Get the return address reg and mark it as an implicit live-in
8667 Register Reg = MF.addLiveIn(PReg: TRI->getReturnAddressReg(MF),
8668 RC: getRegClassFor(VT, isDivergent: Op.getNode()->isDivergent()));
8669
8670 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT);
8671}
8672
8673SDValue SITargetLowering::LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const {
8674 MachineFunction &MF = DAG.getMachineFunction();
8675 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
8676
8677 // For functions that set up their own stack, select the GET_STACK_BASE
8678 // pseudo.
8679 if (MFI->isBottomOfStack())
8680 return Op;
8681
8682 // For everything else, create a dummy stack object.
8683 int FI = MF.getFrameInfo().CreateFixedObject(Size: 1, SPOffset: 0, /*IsImmutable=*/false);
8684 return DAG.getFrameIndex(FI, VT: Op.getValueType());
8685}
8686
8687SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op,
8688 const SDLoc &DL, EVT VT) const {
8689 return Op.getValueType().bitsLE(VT)
8690 ? DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
8691 : DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
8692 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8693}
8694
8695SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op,
8696 SelectionDAG &DAG) const {
8697 EVT DstVT = Op.getValueType();
8698 unsigned NumElts = DstVT.getVectorNumElements();
8699 assert(NumElts > 2 && isPowerOf2_32(NumElts));
8700
8701 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
8702
8703 SDLoc DL(Op);
8704 unsigned Opc = Op.getOpcode();
8705 SDValue Flags = Op.getOperand(i: 1);
8706 EVT HalfDstVT =
8707 EVT::getVectorVT(Context&: *DAG.getContext(), VT: DstVT.getScalarType(), NumElements: NumElts / 2);
8708 SDValue OpLo = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Lo, N2: Flags);
8709 SDValue OpHi = DAG.getNode(Opcode: Opc, DL, VT: HalfDstVT, N1: Hi, N2: Flags);
8710
8711 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, N1: OpLo, N2: OpHi);
8712}
8713
8714SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
8715 bool IsStrict = Op->isStrictFPOpcode();
8716 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8717 EVT SrcVT = Src.getValueType();
8718 EVT DstVT = Op.getValueType();
8719
8720 if (DstVT.isVectorOf(EltVT: MVT::f16)) {
8721 assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32");
8722 if (SrcVT.getScalarType() != MVT::f32)
8723 return SDValue();
8724 return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG);
8725 }
8726
8727 if (SrcVT.getScalarType() != MVT::f64)
8728 return Op;
8729
8730 SDLoc DL(Op);
8731 if (DstVT == MVT::f16) {
8732 // TODO: Handle strictfp
8733 if (Op.getOpcode() != ISD::FP_ROUND)
8734 return Op;
8735
8736 if (!Subtarget->has16BitInsts()) {
8737 SDValue FpToFp16 = DAG.getNode(Opcode: ISD::FP_TO_FP16, DL, VT: MVT::i32, Operand: Src);
8738 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8739 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8740 }
8741 if (Op->getFlags().hasApproximateFuncs()) {
8742 SDValue Flags = Op.getOperand(i: 1);
8743 SDValue Src32 = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f32, N1: Src, N2: Flags);
8744 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Src32, N2: Flags);
8745 }
8746 SDValue FpToFp16 = LowerF64ToF16Safe(Src, DL, DAG);
8747 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToFp16);
8748 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f16, Operand: Trunc);
8749 }
8750
8751 assert(DstVT.getScalarType() == MVT::bf16 &&
8752 "custom lower FP_ROUND for f16 or bf16");
8753 assert(Subtarget->hasBF16ConversionInsts() && "f32 -> bf16 is legal");
8754
8755 // Round-inexact-to-odd f64 to f32, then do the final rounding using the
8756 // hardware f32 -> bf16 instruction.
8757 EVT F32VT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
8758 SDValue Rod = expandRoundInexactToOdd(ResultVT: F32VT, Op: Src, DL, DAG);
8759 if (IsStrict) {
8760 return DAG.getNode(
8761 Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {DstVT, MVT::Other},
8762 Ops: {Op.getOperand(i: 0), Rod, DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)});
8763 }
8764 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: DstVT, N1: Rod,
8765 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8766}
8767
8768SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
8769 SelectionDAG &DAG) const {
8770 assert(Subtarget->hasBF16PackedInsts());
8771
8772 SDLoc DL(Op);
8773
8774 auto WidenOperand = [&](SDValue Src) {
8775 if (Src.getOpcode() == ISD::FNEG) {
8776 SDValue WideSrc = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16,
8777 Operand: Src.getOperand(i: 0));
8778 return DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::v2bf16, Operand: WideSrc, Flags: Src->getFlags());
8779 }
8780
8781 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2bf16, Operand: Src);
8782 };
8783
8784 SDValue LHS = WidenOperand(Op.getOperand(i: 0));
8785 SDValue RHS = WidenOperand(Op.getOperand(i: 1));
8786 SDValue Add =
8787 DAG.getNode(Opcode: ISD::FADD, DL, VT: MVT::v2bf16, N1: LHS, N2: RHS, Flags: Op->getFlags());
8788
8789 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::bf16, N1: Add,
8790 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
8791}
8792
8793SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
8794 SelectionDAG &DAG) const {
8795 EVT VT = Op.getValueType();
8796 const MachineFunction &MF = DAG.getMachineFunction();
8797 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8798 bool IsIEEEMode = Info->getMode().IEEE;
8799
8800 // FIXME: Assert during selection that this is only selected for
8801 // ieee_mode. Currently a combine can produce the ieee version for non-ieee
8802 // mode functions, but this happens to be OK since it's only done in cases
8803 // where there is known no sNaN.
8804 if (IsIEEEMode)
8805 return expandFMINNUM_FMAXNUM(N: Op.getNode(), DAG);
8806
8807 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8808 VT == MVT::v16bf16)
8809 return splitBinaryVectorOp(Op, DAG);
8810 return Op;
8811}
8812
8813SDValue
8814SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op,
8815 SelectionDAG &DAG) const {
8816 EVT VT = Op.getValueType();
8817 const MachineFunction &MF = DAG.getMachineFunction();
8818 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
8819 bool IsIEEEMode = Info->getMode().IEEE;
8820
8821 if (IsIEEEMode)
8822 return expandFMINIMUMNUM_FMAXIMUMNUM(N: Op.getNode(), DAG);
8823
8824 if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 ||
8825 VT == MVT::v16bf16)
8826 return splitBinaryVectorOp(Op, DAG);
8827 return Op;
8828}
8829
8830SDValue SITargetLowering::lowerFMINIMUM_FMAXIMUM(SDValue Op,
8831 SelectionDAG &DAG) const {
8832 EVT VT = Op.getValueType();
8833 if (VT.isVector())
8834 return splitBinaryVectorOp(Op, DAG);
8835
8836 assert(!Subtarget->hasIEEEMinimumMaximumInsts() &&
8837 !Subtarget->hasMinimum3Maximum3F16() &&
8838 Subtarget->hasMinimum3Maximum3PKF16() && VT == MVT::f16 &&
8839 "should not need to widen f16 minimum/maximum to v2f16");
8840
8841 // Widen f16 operation to v2f16
8842
8843 // fminimum f16:x, f16:y ->
8844 // extract_vector_elt (fminimum (v2f16 (scalar_to_vector x))
8845 // (v2f16 (scalar_to_vector y))), 0
8846 SDLoc SL(Op);
8847 SDValue WideSrc0 =
8848 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SL, VT: MVT::v2f16, Operand: Op.getOperand(i: 0));
8849 SDValue WideSrc1 =
8850 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SL, VT: MVT::v2f16, Operand: Op.getOperand(i: 1));
8851
8852 SDValue Widened =
8853 DAG.getNode(Opcode: Op.getOpcode(), DL: SL, VT: MVT::v2f16, N1: WideSrc0, N2: WideSrc1);
8854
8855 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::f16, N1: Widened,
8856 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
8857}
8858
8859SDValue SITargetLowering::lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const {
8860 bool IsStrict = Op.getOpcode() == ISD::STRICT_FLDEXP;
8861 EVT VT = Op.getValueType();
8862 assert(VT == MVT::f16);
8863
8864 SDValue Exp = Op.getOperand(i: IsStrict ? 2 : 1);
8865 EVT ExpVT = Exp.getValueType();
8866 if (ExpVT == MVT::i16)
8867 return Op;
8868
8869 SDLoc DL(Op);
8870
8871 // Correct the exponent type for f16 to i16.
8872 // Clamp the range of the exponent to the instruction's range.
8873
8874 // TODO: This should be a generic narrowing legalization, and can easily be
8875 // for GlobalISel.
8876
8877 SDValue MinExp = DAG.getSignedConstant(Val: minIntN(N: 16), DL, VT: ExpVT);
8878 SDValue ClampMin = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ExpVT, N1: Exp, N2: MinExp);
8879
8880 SDValue MaxExp = DAG.getSignedConstant(Val: maxIntN(N: 16), DL, VT: ExpVT);
8881 SDValue Clamp = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ExpVT, N1: ClampMin, N2: MaxExp);
8882
8883 SDValue TruncExp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Clamp);
8884
8885 if (IsStrict) {
8886 return DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL, ResultTys: {VT, MVT::Other},
8887 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), TruncExp});
8888 }
8889
8890 return DAG.getNode(Opcode: ISD::FLDEXP, DL, VT, N1: Op.getOperand(i: 0), N2: TruncExp);
8891}
8892
8893static unsigned getExtOpcodeForPromotedOp(SDValue Op) {
8894 switch (Op->getOpcode()) {
8895 case ISD::ABS:
8896 case ISD::SRA:
8897 case ISD::SMIN:
8898 case ISD::SMAX:
8899 return ISD::SIGN_EXTEND;
8900 case ISD::SRL:
8901 case ISD::UMIN:
8902 case ISD::UMAX:
8903 case ISD::USUBSAT:
8904 return ISD::ZERO_EXTEND;
8905 case ISD::ADD:
8906 case ISD::SUB:
8907 case ISD::AND:
8908 case ISD::OR:
8909 case ISD::XOR:
8910 case ISD::SHL:
8911 case ISD::SELECT:
8912 case ISD::MUL:
8913 // operation result won't be influenced by garbage high bits.
8914 // TODO: are all of those cases correct, and are there more?
8915 return ISD::ANY_EXTEND;
8916 case ISD::SETCC: {
8917 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
8918 return ISD::isSignedIntSetCC(Code: CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8919 }
8920 default:
8921 llvm_unreachable("unexpected opcode!");
8922 }
8923}
8924
8925SDValue
8926SITargetLowering::promoteUniformUnaryOpToI32(SDValue Op,
8927 DAGCombinerInfo &DCI) const {
8928 EVT OpTy = Op.getValueType();
8929 SelectionDAG &DAG = DCI.DAG;
8930 EVT ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
8931
8932 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
8933 return SDValue();
8934
8935 SDLoc DL(Op);
8936 SDValue Input = Op.getOperand(i: 0);
8937 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
8938 Input = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: Input);
8939
8940 SDValue NewVal = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: ExtTy, Operand: Input);
8941
8942 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: OpTy, Operand: NewVal);
8943}
8944
8945SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op,
8946 DAGCombinerInfo &DCI) const {
8947 const unsigned Opc = Op.getOpcode();
8948 assert(Opc == ISD::ADD || Opc == ISD::SUB || Opc == ISD::SHL ||
8949 Opc == ISD::SRL || Opc == ISD::SRA || Opc == ISD::AND ||
8950 Opc == ISD::OR || Opc == ISD::XOR || Opc == ISD::MUL ||
8951 Opc == ISD::SETCC || Opc == ISD::SELECT || Opc == ISD::SMIN ||
8952 Opc == ISD::SMAX || Opc == ISD::UMIN || Opc == ISD::UMAX ||
8953 Opc == ISD::USUBSAT);
8954
8955 EVT OpTy = (Opc != ISD::SETCC) ? Op.getValueType()
8956 : Op->getOperand(Num: 0).getValueType();
8957 auto &DAG = DCI.DAG;
8958 auto ExtTy = OpTy.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
8959
8960 if (DCI.isBeforeLegalizeOps() ||
8961 isNarrowingProfitable(N: Op.getNode(), SrcVT: ExtTy, DestVT: OpTy))
8962 return SDValue();
8963
8964 SDLoc DL(Op);
8965 SDValue LHS;
8966 SDValue RHS;
8967 if (Opc == ISD::SELECT) {
8968 LHS = Op->getOperand(Num: 1);
8969 RHS = Op->getOperand(Num: 2);
8970 } else {
8971 LHS = Op->getOperand(Num: 0);
8972 RHS = Op->getOperand(Num: 1);
8973 }
8974
8975 const unsigned ExtOp = getExtOpcodeForPromotedOp(Op);
8976 LHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {LHS});
8977
8978 // Special case: for shifts, the RHS always needs a zext.
8979 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
8980 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtTy, Operand: {RHS});
8981 else
8982 RHS = DAG.getNode(Opcode: ExtOp, DL, VT: ExtTy, Operand: {RHS});
8983
8984 // setcc always return i1/i1 vec so no need to truncate after.
8985 if (Opc == ISD::SETCC) {
8986 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
8987 return DAG.getSetCC(DL, VT: Op.getValueType(), LHS, RHS, Cond: CC);
8988 }
8989
8990 // For other ops, we extend the operation's return type as well so we need to
8991 // truncate back to the original type.
8992 SDValue NewVal;
8993 if (Opc == ISD::SELECT)
8994 NewVal = DAG.getNode(Opcode: ISD::SELECT, DL, VT: ExtTy, Ops: {Op->getOperand(Num: 0), LHS, RHS});
8995 else
8996 NewVal = DAG.getNode(Opcode: Opc, DL, VT: ExtTy, Ops: {LHS, RHS});
8997
8998 return DAG.getZExtOrTrunc(Op: NewVal, DL, VT: OpTy);
8999}
9000
9001SDValue SITargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
9002 SDValue Mag = Op.getOperand(i: 0);
9003 EVT MagVT = Mag.getValueType();
9004
9005 if (MagVT.getVectorNumElements() > 2)
9006 return splitBinaryVectorOp(Op, DAG);
9007
9008 SDValue Sign = Op.getOperand(i: 1);
9009 EVT SignVT = Sign.getValueType();
9010
9011 if (MagVT == SignVT)
9012 return Op;
9013
9014 // fcopysign v2f16:mag, v2f32:sign ->
9015 // fcopysign v2f16:mag,
9016 // bitcast (trunc (srl (bitcast sign to v2i32), 16) to v2i16)
9017
9018 SDLoc SL(Op);
9019 SDValue SignAsInt32 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Sign);
9020 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: 16, VT: MVT::v2i32, DL: SL);
9021 SDValue SignShifted =
9022 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::v2i32, N1: SignAsInt32, N2: ShiftAmt);
9023 SDValue SignAsInt16 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::v2i16, Operand: SignShifted);
9024
9025 SDValue SignAsHalf16 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MagVT, Operand: SignAsInt16);
9026
9027 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MagVT, N1: Mag, N2: SignAsHalf16);
9028}
9029
9030// Custom lowering for vector multiplications and s_mul_u64.
9031SDValue SITargetLowering::lowerMUL(SDValue Op, SelectionDAG &DAG) const {
9032 EVT VT = Op.getValueType();
9033
9034 // Split vector operands.
9035 if (VT.isVector())
9036 return splitBinaryVectorOp(Op, DAG);
9037
9038 assert(VT == MVT::i64 && "The following code is a special for s_mul_u64");
9039
9040 // There are four ways to lower s_mul_u64:
9041 //
9042 // 1. If all the operands are uniform, then we lower it as it is.
9043 //
9044 // 2. If the operands are divergent, then we have to split s_mul_u64 in 32-bit
9045 // multiplications because there is not a vector equivalent of s_mul_u64.
9046 //
9047 // 3. If the cost model decides that it is more efficient to use vector
9048 // registers, then we have to split s_mul_u64 in 32-bit multiplications.
9049 // This happens in splitScalarSMULU64() in SIInstrInfo.cpp .
9050 //
9051 // 4. If the cost model decides to use vector registers and both of the
9052 // operands are zero-extended/sign-extended from 32-bits, then we split the
9053 // s_mul_u64 in two 32-bit multiplications. The problem is that it is not
9054 // possible to check if the operands are zero-extended or sign-extended in
9055 // SIInstrInfo.cpp. For this reason, here, we replace s_mul_u64 with
9056 // s_mul_u64_u32_pseudo if both operands are zero-extended and we replace
9057 // s_mul_u64 with s_mul_i64_i32_pseudo if both operands are sign-extended.
9058 // If the cost model decides that we have to use vector registers, then
9059 // splitScalarSMulPseudo() (in SIInstrInfo.cpp) split s_mul_u64_u32/
9060 // s_mul_i64_i32_pseudo in two vector multiplications. If the cost model
9061 // decides that we should use scalar registers, then s_mul_u64_u32_pseudo/
9062 // s_mul_i64_i32_pseudo is lowered as s_mul_u64 in expandPostRAPseudo() in
9063 // SIInstrInfo.cpp .
9064
9065 if (Op->isDivergent())
9066 return SDValue();
9067
9068 SDValue Op0 = Op.getOperand(i: 0);
9069 SDValue Op1 = Op.getOperand(i: 1);
9070 // If all the operands are zero-enteted to 32-bits, then we replace s_mul_u64
9071 // with s_mul_u64_u32_pseudo. If all the operands are sign-extended to
9072 // 32-bits, then we replace s_mul_u64 with s_mul_i64_i32_pseudo.
9073 KnownBits Op0KnownBits = DAG.computeKnownBits(Op: Op0);
9074 unsigned Op0LeadingZeros = Op0KnownBits.countMinLeadingZeros();
9075 KnownBits Op1KnownBits = DAG.computeKnownBits(Op: Op1);
9076 unsigned Op1LeadingZeros = Op1KnownBits.countMinLeadingZeros();
9077 SDLoc SL(Op);
9078 if (Op0LeadingZeros >= 32 && Op1LeadingZeros >= 32)
9079 return SDValue(
9080 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_U64_U32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9081 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op0);
9082 unsigned Op1SignBits = DAG.ComputeNumSignBits(Op: Op1);
9083 if (Op0SignBits >= 33 && Op1SignBits >= 33)
9084 return SDValue(
9085 DAG.getMachineNode(Opcode: AMDGPU::S_MUL_I64_I32_PSEUDO, dl: SL, VT, Op1: Op0, Op2: Op1), 0);
9086 // If all the operands are uniform, then we lower s_mul_u64 as it is.
9087 return Op;
9088}
9089
9090SDValue SITargetLowering::lowerXMULO(SDValue Op, SelectionDAG &DAG) const {
9091 EVT VT = Op.getValueType();
9092 SDLoc SL(Op);
9093 SDValue LHS = Op.getOperand(i: 0);
9094 SDValue RHS = Op.getOperand(i: 1);
9095 bool isSigned = Op.getOpcode() == ISD::SMULO;
9096
9097 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
9098 const APInt &C = RHSC->getAPIntValue();
9099 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
9100 if (C.isPowerOf2()) {
9101 // smulo(x, signed_min) is same as umulo(x, signed_min).
9102 bool UseArithShift = isSigned && !C.isMinSignedValue();
9103 SDValue ShiftAmt = DAG.getConstant(Val: C.logBase2(), DL: SL, VT: MVT::i32);
9104 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: LHS, N2: ShiftAmt);
9105 SDValue Overflow =
9106 DAG.getSetCC(DL: SL, VT: MVT::i1,
9107 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL, DL: SL, VT,
9108 N1: Result, N2: ShiftAmt),
9109 RHS: LHS, Cond: ISD::SETNE);
9110 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9111 }
9112 }
9113
9114 SDValue Result = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT, N1: LHS, N2: RHS);
9115 SDValue Top =
9116 DAG.getNode(Opcode: isSigned ? ISD::MULHS : ISD::MULHU, DL: SL, VT, N1: LHS, N2: RHS);
9117
9118 SDValue Sign = isSigned
9119 ? DAG.getNode(Opcode: ISD::SRA, DL: SL, VT, N1: Result,
9120 N2: DAG.getConstant(Val: VT.getScalarSizeInBits() - 1,
9121 DL: SL, VT: MVT::i32))
9122 : DAG.getConstant(Val: 0, DL: SL, VT);
9123 SDValue Overflow = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Top, RHS: Sign, Cond: ISD::SETNE);
9124
9125 return DAG.getMergeValues(Ops: {Result, Overflow}, dl: SL);
9126}
9127
9128SDValue SITargetLowering::lowerXMUL_LOHI(SDValue Op, SelectionDAG &DAG) const {
9129 if (Op->isDivergent()) {
9130 // Select to V_MAD_[IU]64_[IU]32.
9131 return Op;
9132 }
9133 if (Subtarget->hasSMulHi()) {
9134 // Expand to S_MUL_I32 + S_MUL_HI_[IU]32.
9135 return SDValue();
9136 }
9137 // The multiply is uniform but we would have to use V_MUL_HI_[IU]32 to
9138 // calculate the high part, so we might as well do the whole thing with
9139 // V_MAD_[IU]64_[IU]32.
9140 return Op;
9141}
9142
9143SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
9144 if (!Subtarget->hasTrapHandler() ||
9145 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
9146 return lowerTrapEndpgm(Op, DAG);
9147
9148 return Subtarget->supportsGetDoorbellID() ? lowerTrapHsa(Op, DAG)
9149 : lowerTrapHsaQueuePtr(Op, DAG);
9150}
9151
9152SDValue SITargetLowering::lowerTrapEndpgm(SDValue Op, SelectionDAG &DAG) const {
9153 SDLoc SL(Op);
9154 SDValue Chain = Op.getOperand(i: 0);
9155 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9156}
9157
9158SDValue
9159SITargetLowering::loadImplicitKernelArgument(SelectionDAG &DAG, MVT VT,
9160 const SDLoc &DL, Align Alignment,
9161 ImplicitParameter Param) const {
9162 MachineFunction &MF = DAG.getMachineFunction();
9163 uint64_t Offset = getImplicitParameterOffset(MF, Param);
9164 SDValue Ptr = lowerKernArgParameterPtr(DAG, SL: DL, Chain: DAG.getEntryNode(), Offset);
9165 MachinePointerInfo PtrInfo =
9166 getKernargSegmentPtrInfo(MF&: DAG.getMachineFunction());
9167 return DAG.getLoad(
9168 VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: PtrInfo.getWithOffset(O: Offset), Alignment,
9169 MMOFlags: MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
9170}
9171
9172SDValue SITargetLowering::lowerTrapHsaQueuePtr(SDValue Op,
9173 SelectionDAG &DAG) const {
9174 SDLoc SL(Op);
9175 SDValue Chain = Op.getOperand(i: 0);
9176
9177 SDValue QueuePtr;
9178 // For code object version 5, QueuePtr is passed through implicit kernarg.
9179 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9180 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9181 QueuePtr =
9182 loadImplicitKernelArgument(DAG, VT: MVT::i64, DL: SL, Alignment: Align(8), Param: QUEUE_PTR);
9183 } else {
9184 MachineFunction &MF = DAG.getMachineFunction();
9185 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9186 Register UserSGPR = Info->getQueuePtrUserSGPR();
9187
9188 if (UserSGPR == AMDGPU::NoRegister) {
9189 // We probably are in a function incorrectly marked with
9190 // amdgpu-no-queue-ptr. This is undefined. We don't want to delete the
9191 // trap, so just use a null pointer.
9192 QueuePtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9193 } else {
9194 QueuePtr = CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR,
9195 VT: MVT::i64);
9196 }
9197 }
9198
9199 SDValue SGPR01 = DAG.getRegister(Reg: AMDGPU::SGPR0_SGPR1, VT: MVT::i64);
9200 SDValue ToReg = DAG.getCopyToReg(Chain, dl: SL, Reg: SGPR01, N: QueuePtr, Glue: SDValue());
9201
9202 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9203 SDValue Ops[] = {ToReg, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16), SGPR01,
9204 ToReg.getValue(R: 1)};
9205 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9206}
9207
9208SDValue SITargetLowering::lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const {
9209 SDLoc SL(Op);
9210 SDValue Chain = Op.getOperand(i: 0);
9211
9212 // We need to simulate the 's_trap 2' instruction on targets that run in
9213 // PRIV=1 (where it is treated as a nop).
9214 if (Subtarget->hasPrivEnabledTrap2NopBug())
9215 return DAG.getNode(Opcode: AMDGPUISD::SIMULATED_TRAP, DL: SL, VT: MVT::Other, Operand: Chain);
9216
9217 uint64_t TrapID = static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSATrap);
9218 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9219 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9220}
9221
9222SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
9223 SDLoc SL(Op);
9224 SDValue Chain = Op.getOperand(i: 0);
9225 MachineFunction &MF = DAG.getMachineFunction();
9226
9227 if (!Subtarget->hasTrapHandler() ||
9228 Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
9229 LLVMContext &Ctx = MF.getFunction().getContext();
9230 Ctx.diagnose(DI: DiagnosticInfoUnsupported(MF.getFunction(),
9231 "debugtrap handler not supported",
9232 Op.getDebugLoc(), DS_Warning));
9233 return Chain;
9234 }
9235
9236 uint64_t TrapID =
9237 static_cast<uint64_t>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap);
9238 SDValue Ops[] = {Chain, DAG.getTargetConstant(Val: TrapID, DL: SL, VT: MVT::i16)};
9239 return DAG.getNode(Opcode: AMDGPUISD::TRAP, DL: SL, VT: MVT::Other, Ops);
9240}
9241
9242/// When a divergent value (in VGPR) is passed to an inline asm with an SGPR
9243/// constraint ('s'), we need to insert v_readfirstlane to move the value from
9244/// VGPR to SGPR. This is done by modifying the CopyToReg nodes in the glue
9245/// chain that feed into the INLINEASM node.
9246SDValue SITargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
9247 unsigned NumOps = Op.getNumOperands();
9248
9249 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
9250 SmallSet<Register, 8> SGPRInputRegs;
9251
9252 unsigned NumVals = 0;
9253 for (unsigned I = InlineAsm::Op_FirstOperand; I < NumOps - 1;
9254 I += 1 + NumVals) {
9255 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i: I));
9256 NumVals = Flags.getNumOperandRegisters();
9257
9258 unsigned RCID;
9259 bool IsSGPRInput = Flags.getKind() == InlineAsm::Kind::RegUse &&
9260 NumVals > 0 && Flags.hasRegClassConstraint(RC&: RCID) &&
9261 TRI->isSGPRClass(RC: TRI->getRegClass(i: RCID));
9262
9263 for (unsigned J = 0; J < NumVals; ++J) {
9264 SDValue Val = Op.getOperand(i: I + 1 + J);
9265 if (const RegisterSDNode *RegNode =
9266 dyn_cast<RegisterSDNode>(Val: Val.getNode())) {
9267 Register Reg = RegNode->getReg();
9268 if (IsSGPRInput || (Reg.isPhysical() && TRI->isSGPRPhysReg(Reg)))
9269 SGPRInputRegs.insert(V: Reg);
9270 }
9271 }
9272 }
9273
9274 if (SGPRInputRegs.empty())
9275 return Op;
9276
9277 // Walk the glue chain and insert readfirstlane for divergent SGPR inputs.
9278 SDLoc DL(Op);
9279 SDNode *N = Op.getOperand(i: NumOps - 1).getNode();
9280
9281 while (N && N->getOpcode() == ISD::CopyToReg) {
9282 Register Reg = cast<RegisterSDNode>(Val: N->getOperand(Num: 1))->getReg();
9283 SDValue SrcVal = N->getOperand(Num: 2);
9284
9285 // Insert readfirstlane if copying a divergent value to an SGPR input.
9286 if (SrcVal->isDivergent() && SGPRInputRegs.count(V: Reg)) {
9287 SDValue ReadFirstLaneID =
9288 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
9289 SDValue ReadFirstLane =
9290 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: SrcVal.getValueType(),
9291 N1: ReadFirstLaneID, N2: SrcVal);
9292
9293 SmallVector<SDValue, 4> Ops = {N->getOperand(Num: 0), N->getOperand(Num: 1),
9294 ReadFirstLane};
9295 if (N->getNumOperands() > 3)
9296 Ops.push_back(Elt: N->getOperand(Num: 3)); // Glue input
9297
9298 DAG.UpdateNodeOperands(N, Ops);
9299 }
9300
9301 // Follow glue chain to next CopyToReg.
9302 SDNode *Next = nullptr;
9303 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I) {
9304 if (N->getOperand(Num: I).getValueType() == MVT::Glue) {
9305 Next = N->getOperand(Num: I).getNode();
9306 break;
9307 }
9308 }
9309 N = Next;
9310 }
9311
9312 return Op;
9313}
9314
9315SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
9316 SelectionDAG &DAG) const {
9317 if (Subtarget->hasApertureRegs()) {
9318 const unsigned ApertureRegNo = (AS == AMDGPUAS::LOCAL_ADDRESS)
9319 ? AMDGPU::SRC_SHARED_BASE
9320 : AMDGPU::SRC_PRIVATE_BASE;
9321 assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
9322 !Subtarget->hasGloballyAddressableScratch()) &&
9323 "Cannot use src_private_base with globally addressable scratch!");
9324 // Note: this feature (register) is broken. When used as a 32-bit operand,
9325 // it returns a wrong value (all zeroes?). The real value is in the upper 32
9326 // bits.
9327 //
9328 // To work around the issue, emit a 64 bit copy from this register
9329 // then extract the high bits. Note that this shouldn't even result in a
9330 // shift being emitted and simply become a pair of registers (e.g.):
9331 // s_mov_b64 s[6:7], src_shared_base
9332 // v_mov_b32_e32 v1, s7
9333 SDValue Copy =
9334 DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: ApertureRegNo, VT: MVT::v2i32);
9335 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Copy, Idx: 1);
9336 }
9337
9338 // For code object version 5, private_base and shared_base are passed through
9339 // implicit kernargs.
9340 const Module *M = DAG.getMachineFunction().getFunction().getParent();
9341 if (AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5) {
9342 ImplicitParameter Param =
9343 (AS == AMDGPUAS::LOCAL_ADDRESS) ? SHARED_BASE : PRIVATE_BASE;
9344 return loadImplicitKernelArgument(DAG, VT: MVT::i32, DL, Alignment: Align(4), Param);
9345 }
9346
9347 MachineFunction &MF = DAG.getMachineFunction();
9348 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
9349 Register UserSGPR = Info->getQueuePtrUserSGPR();
9350 if (UserSGPR == AMDGPU::NoRegister) {
9351 // We probably are in a function incorrectly marked with
9352 // amdgpu-no-queue-ptr. This is undefined.
9353 return DAG.getPOISON(VT: MVT::i32);
9354 }
9355
9356 SDValue QueuePtr =
9357 CreateLiveInRegister(DAG, RC: &AMDGPU::SReg_64RegClass, Reg: UserSGPR, VT: MVT::i64);
9358
9359 // Offset into amd_queue_t for group_segment_aperture_base_hi /
9360 // private_segment_aperture_base_hi.
9361 uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
9362
9363 SDValue Ptr =
9364 DAG.getObjectPtrOffset(SL: DL, Ptr: QueuePtr, Offset: TypeSize::getFixed(ExactSize: StructOffset));
9365
9366 // TODO: Use custom target PseudoSourceValue.
9367 // TODO: We should use the value from the IR intrinsic call, but it might not
9368 // be available and how do we get it?
9369 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
9370 return DAG.getLoad(VT: MVT::i32, dl: DL, Chain: QueuePtr.getValue(R: 1), Ptr, PtrInfo,
9371 Alignment: commonAlignment(A: Align(64), Offset: StructOffset),
9372 MMOFlags: MachineMemOperand::MODereferenceable |
9373 MachineMemOperand::MOInvariant);
9374}
9375
9376/// Return true if the value is a known valid address, such that a null check is
9377/// not necessary.
9378static bool isKnownNonNull(SDValue Val, SelectionDAG &DAG,
9379 const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
9380 if (isa<FrameIndexSDNode, GlobalAddressSDNode, BasicBlockSDNode>(Val))
9381 return true;
9382
9383 if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val))
9384 return ConstVal->getSExtValue() != AMDGPU::getNullPointerValue(AS: AddrSpace);
9385
9386 // TODO: Search through arithmetic, handle arguments and loads
9387 // marked nonnull.
9388 return false;
9389}
9390
9391SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
9392 SelectionDAG &DAG) const {
9393 SDLoc SL(Op);
9394
9395 const AMDGPUTargetMachine &TM =
9396 static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
9397
9398 unsigned DestAS, SrcAS;
9399 SDValue Src;
9400 bool IsNonNull = false;
9401 if (const auto *ASC = dyn_cast<AddrSpaceCastSDNode>(Val&: Op)) {
9402 SrcAS = ASC->getSrcAddressSpace();
9403 Src = ASC->getOperand(Num: 0);
9404 DestAS = ASC->getDestAddressSpace();
9405 } else {
9406 assert(Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
9407 Op.getConstantOperandVal(0) ==
9408 Intrinsic::amdgcn_addrspacecast_nonnull);
9409 Src = Op->getOperand(Num: 1);
9410 SrcAS = Op->getConstantOperandVal(Num: 2);
9411 DestAS = Op->getConstantOperandVal(Num: 3);
9412 IsNonNull = true;
9413 }
9414
9415 SDValue FlatNullPtr = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i64);
9416
9417 // flat -> local/private
9418 if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
9419 if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
9420 DestAS == AMDGPUAS::PRIVATE_ADDRESS) {
9421 SDValue Ptr = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9422
9423 if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
9424 Subtarget->hasGloballyAddressableScratch()) {
9425 // flat -> private with globally addressable scratch: subtract
9426 // src_flat_scratch_base_lo.
9427 SDValue FlatScratchBaseLo(
9428 DAG.getMachineNode(
9429 Opcode: AMDGPU::S_MOV_B32, dl: SL, VT: MVT::i32,
9430 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, VT: MVT::i32)),
9431 0);
9432 Ptr = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: Ptr, N2: FlatScratchBaseLo);
9433 }
9434
9435 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9436 return Ptr;
9437
9438 unsigned NullVal = AMDGPU::getNullPointerValue(AS: DestAS);
9439 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9440 SDValue NonNull = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: FlatNullPtr, Cond: ISD::SETNE);
9441
9442 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i32, N1: NonNull, N2: Ptr,
9443 N3: SegmentNullPtr);
9444 }
9445 }
9446
9447 // local/private -> flat
9448 if (DestAS == AMDGPUAS::FLAT_ADDRESS) {
9449 if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
9450 SrcAS == AMDGPUAS::PRIVATE_ADDRESS) {
9451 SDValue CvtPtr;
9452 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
9453 Subtarget->hasGloballyAddressableScratch()) {
9454 // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
9455 // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
9456 SDValue AllOnes = DAG.getSignedTargetConstant(Val: -1, DL: SL, VT: MVT::i32);
9457 SDValue ThreadID = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
9458 ThreadID = DAG.getNode(
9459 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9460 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_lo, DL: SL, VT: MVT::i32),
9461 N2: AllOnes, N3: ThreadID);
9462 if (Subtarget->isWave64())
9463 ThreadID = DAG.getNode(
9464 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
9465 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_mbcnt_hi, DL: SL, VT: MVT::i32),
9466 N2: AllOnes, N3: ThreadID);
9467 SDValue ShAmt = DAG.getShiftAmountConstant(
9468 Val: 57 - 32 - Subtarget->getWavefrontSizeLog2(), VT: MVT::i32, DL: SL);
9469 SDValue SrcHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ThreadID, N2: ShAmt);
9470 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: SrcHi);
9471 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9472 // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
9473 // 64-bit hi:lo value.
9474 SDValue FlatScratchBase = {
9475 DAG.getMachineNode(
9476 Opcode: AMDGPU::S_MOV_B64, dl: SL, VT: MVT::i64,
9477 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE, VT: MVT::i64)),
9478 0};
9479 CvtPtr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: CvtPtr, N2: FlatScratchBase);
9480 } else {
9481 SDValue Aperture = getSegmentAperture(AS: SrcAS, DL: SL, DAG);
9482 CvtPtr = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Aperture);
9483 CvtPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: CvtPtr);
9484 }
9485
9486 if (IsNonNull || isKnownNonNull(Val: Op, DAG, TM, AddrSpace: SrcAS))
9487 return CvtPtr;
9488
9489 unsigned NullVal = AMDGPU::getNullPointerValue(AS: SrcAS);
9490 SDValue SegmentNullPtr = DAG.getConstant(Val: NullVal, DL: SL, VT: MVT::i32);
9491
9492 SDValue NonNull =
9493 DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: Src, RHS: SegmentNullPtr, Cond: ISD::SETNE);
9494
9495 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: NonNull, N2: CvtPtr,
9496 N3: FlatNullPtr);
9497 }
9498 }
9499
9500 if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9501 Op.getValueType() == MVT::i64) {
9502 const SIMachineFunctionInfo *Info =
9503 DAG.getMachineFunction().getInfo<SIMachineFunctionInfo>();
9504 if (Info->get32BitAddressHighBits() == 0)
9505 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: Src);
9506
9507 SDValue Hi = DAG.getConstant(Val: Info->get32BitAddressHighBits(), DL: SL, VT: MVT::i32);
9508 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: Src, N2: Hi);
9509 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
9510 }
9511
9512 if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
9513 Src.getValueType() == MVT::i64)
9514 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Src);
9515
9516 // global <-> flat are no-ops and never emitted.
9517
9518 // Invalid casts are poison.
9519 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
9520}
9521
9522// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
9523// the small vector and inserting them into the big vector. That is better than
9524// the default expansion of doing it via a stack slot. Even though the use of
9525// the stack slot would be optimized away afterwards, the stack slot itself
9526// remains.
9527SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
9528 SelectionDAG &DAG) const {
9529 SDValue Vec = Op.getOperand(i: 0);
9530 SDValue Ins = Op.getOperand(i: 1);
9531 SDValue Idx = Op.getOperand(i: 2);
9532 EVT VecVT = Vec.getValueType();
9533 EVT InsVT = Ins.getValueType();
9534 EVT EltVT = VecVT.getVectorElementType();
9535 unsigned InsNumElts = InsVT.getVectorNumElements();
9536 unsigned IdxVal = Idx->getAsZExtVal();
9537 SDLoc SL(Op);
9538
9539 if (EltVT.getScalarSizeInBits() == 16 && IdxVal % 2 == 0) {
9540 // Insert 32-bit registers at a time.
9541 assert(InsNumElts % 2 == 0 && "expect legal vector types");
9542
9543 unsigned VecNumElts = VecVT.getVectorNumElements();
9544 EVT NewVecVT =
9545 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: VecNumElts / 2);
9546 EVT NewInsVT = InsNumElts == 2 ? MVT::i32
9547 : EVT::getVectorVT(Context&: *DAG.getContext(),
9548 VT: MVT::i32, NumElements: InsNumElts / 2);
9549
9550 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVecVT, Operand: Vec);
9551 Ins = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewInsVT, Operand: Ins);
9552
9553 for (unsigned I = 0; I != InsNumElts / 2; ++I) {
9554 SDValue Elt;
9555 if (InsNumElts == 2) {
9556 Elt = Ins;
9557 } else {
9558 Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Ins,
9559 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9560 }
9561 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NewVecVT, N1: Vec, N2: Elt,
9562 N3: DAG.getConstant(Val: IdxVal / 2 + I, DL: SL, VT: MVT::i32));
9563 }
9564
9565 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Vec);
9566 }
9567
9568 for (unsigned I = 0; I != InsNumElts; ++I) {
9569 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Ins,
9570 N2: DAG.getConstant(Val: I, DL: SL, VT: MVT::i32));
9571 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: VecVT, N1: Vec, N2: Elt,
9572 N3: DAG.getConstant(Val: IdxVal + I, DL: SL, VT: MVT::i32));
9573 }
9574 return Vec;
9575}
9576
9577SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
9578 SelectionDAG &DAG) const {
9579 SDValue Vec = Op.getOperand(i: 0);
9580 SDValue InsVal = Op.getOperand(i: 1);
9581 SDValue Idx = Op.getOperand(i: 2);
9582 EVT VecVT = Vec.getValueType();
9583 EVT EltVT = VecVT.getVectorElementType();
9584 unsigned VecSize = VecVT.getSizeInBits();
9585 unsigned EltSize = EltVT.getSizeInBits();
9586 SDLoc SL(Op);
9587
9588 // Specially handle the case of v4i16 with static indexing.
9589 unsigned NumElts = VecVT.getVectorNumElements();
9590 auto *KIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
9591 if (NumElts == 4 && EltSize == 16 && KIdx) {
9592 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Vec);
9593
9594 SDValue LoHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9595 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
9596 SDValue HiHalf = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: BCVec,
9597 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
9598
9599 SDValue LoVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: LoHalf);
9600 SDValue HiVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i16, Operand: HiHalf);
9601
9602 unsigned Idx = KIdx->getZExtValue();
9603 bool InsertLo = Idx < 2;
9604 SDValue InsHalf = DAG.getNode(
9605 Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: MVT::v2i16, N1: InsertLo ? LoVec : HiVec,
9606 N2: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: InsVal),
9607 N3: DAG.getConstant(Val: InsertLo ? Idx : (Idx - 2), DL: SL, VT: MVT::i32));
9608
9609 InsHalf = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: InsHalf);
9610
9611 SDValue Concat =
9612 InsertLo ? DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {InsHalf, HiHalf})
9613 : DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoHalf, InsHalf});
9614
9615 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: Concat);
9616 }
9617
9618 // Static indexing does not lower to stack access, and hence there is no need
9619 // for special custom lowering to avoid stack access.
9620 if (isa<ConstantSDNode>(Val: Idx))
9621 return SDValue();
9622
9623 // Avoid stack access for dynamic indexing by custom lowering to
9624 // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
9625
9626 assert(VecSize <= 64 && "Expected target vector size to be <= 64 bits");
9627
9628 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9629
9630 // Convert vector index to bit-index and get the required bit mask.
9631 assert(isPowerOf2_32(EltSize));
9632 const auto EltMask = maskTrailingOnes<uint64_t>(N: EltSize);
9633 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9634 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9635 SDValue BFM = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT,
9636 N1: DAG.getConstant(Val: EltMask, DL: SL, VT: IntVT), N2: ScaledIdx);
9637
9638 // 1. Create a congruent vector with the target value in each element.
9639 SDValue ExtVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT,
9640 Operand: DAG.getSplatBuildVector(VT: VecVT, DL: SL, Op: InsVal));
9641
9642 // 2. Mask off all other indices except the required index within (1).
9643 SDValue LHS = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: BFM, N2: ExtVal);
9644
9645 // 3. Mask off the required index within the target vector.
9646 SDValue BCVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9647 SDValue RHS =
9648 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IntVT, N1: DAG.getNOT(DL: SL, Val: BFM, VT: IntVT), N2: BCVec);
9649
9650 // 4. Get (2) and (3) ORed into the target vector.
9651 SDValue BFI =
9652 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: LHS, N2: RHS, Flags: SDNodeFlags::Disjoint);
9653
9654 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecVT, Operand: BFI);
9655}
9656
9657SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
9658 SelectionDAG &DAG) const {
9659 SDLoc SL(Op);
9660
9661 EVT ResultVT = Op.getValueType();
9662 SDValue Vec = Op.getOperand(i: 0);
9663 SDValue Idx = Op.getOperand(i: 1);
9664 EVT VecVT = Vec.getValueType();
9665 unsigned VecSize = VecVT.getSizeInBits();
9666 EVT EltVT = VecVT.getVectorElementType();
9667
9668 DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
9669
9670 // Make sure we do any optimizations that will make it easier to fold
9671 // source modifiers before obscuring it with bit operations.
9672
9673 // XXX - Why doesn't this get called when vector_shuffle is expanded?
9674 if (SDValue Combined = performExtractVectorEltCombine(N: Op.getNode(), DCI))
9675 return Combined;
9676
9677 if (VecSize == 128 || VecSize == 256 || VecSize == 512) {
9678 SDValue Lo, Hi;
9679 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
9680
9681 if (VecSize == 128) {
9682 SDValue V2 = DAG.getBitcast(VT: MVT::v2i64, V: Vec);
9683 Lo = DAG.getBitcast(VT: LoVT,
9684 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9685 N2: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32)));
9686 Hi = DAG.getBitcast(VT: HiVT,
9687 V: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9688 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32)));
9689 } else if (VecSize == 256) {
9690 SDValue V2 = DAG.getBitcast(VT: MVT::v4i64, V: Vec);
9691 SDValue Parts[4];
9692 for (unsigned P = 0; P < 4; ++P) {
9693 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9694 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9695 }
9696
9697 Lo = DAG.getBitcast(VT: LoVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9698 N1: Parts[0], N2: Parts[1]));
9699 Hi = DAG.getBitcast(VT: HiVT, V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i64,
9700 N1: Parts[2], N2: Parts[3]));
9701 } else {
9702 assert(VecSize == 512);
9703
9704 SDValue V2 = DAG.getBitcast(VT: MVT::v8i64, V: Vec);
9705 SDValue Parts[8];
9706 for (unsigned P = 0; P < 8; ++P) {
9707 Parts[P] = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i64, N1: V2,
9708 N2: DAG.getConstant(Val: P, DL: SL, VT: MVT::i32));
9709 }
9710
9711 Lo = DAG.getBitcast(VT: LoVT,
9712 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9713 N1: Parts[0], N2: Parts[1], N3: Parts[2], N4: Parts[3]));
9714 Hi = DAG.getBitcast(VT: HiVT,
9715 V: DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v4i64,
9716 N1: Parts[4], N2: Parts[5], N3: Parts[6], N4: Parts[7]));
9717 }
9718
9719 EVT IdxVT = Idx.getValueType();
9720 unsigned NElem = VecVT.getVectorNumElements();
9721 assert(isPowerOf2_32(NElem));
9722 SDValue IdxMask = DAG.getConstant(Val: NElem / 2 - 1, DL: SL, VT: IdxVT);
9723 SDValue NewIdx = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: IdxVT, N1: Idx, N2: IdxMask);
9724 SDValue Half = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IdxMask, True: Hi, False: Lo, Cond: ISD::SETUGT);
9725 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Half, N2: NewIdx);
9726 }
9727
9728 assert(VecSize <= 64);
9729
9730 MVT IntVT = MVT::getIntegerVT(BitWidth: VecSize);
9731
9732 // If Vec is just a SCALAR_TO_VECTOR, then use the scalar integer directly.
9733 SDValue VecBC = peekThroughBitcasts(V: Vec);
9734 if (VecBC.getOpcode() == ISD::SCALAR_TO_VECTOR) {
9735 SDValue Src = VecBC.getOperand(i: 0);
9736 Src = DAG.getBitcast(VT: Src.getValueType().changeTypeToInteger(), V: Src);
9737 Vec = DAG.getAnyExtOrTrunc(Op: Src, DL: SL, VT: IntVT);
9738 }
9739
9740 unsigned EltSize = EltVT.getSizeInBits();
9741 assert(isPowerOf2_32(EltSize));
9742
9743 SDValue ScaleFactor = DAG.getConstant(Val: Log2_32(Value: EltSize), DL: SL, VT: MVT::i32);
9744
9745 // Convert vector index to bit-index (* EltSize)
9746 SDValue ScaledIdx = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Idx, N2: ScaleFactor);
9747
9748 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: Vec);
9749 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: IntVT, N1: BC, N2: ScaledIdx);
9750
9751 if (ResultVT == MVT::f16 || ResultVT == MVT::bf16) {
9752 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i16, Operand: Elt);
9753 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: ResultVT, Operand: Result);
9754 }
9755
9756 return DAG.getAnyExtOrTrunc(Op: Elt, DL: SL, VT: ResultVT);
9757}
9758
9759static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
9760 assert(Elt % 2 == 0);
9761 return Mask[Elt + 1] == Mask[Elt] + 1 && (Mask[Elt] % 2 == 0);
9762}
9763
9764static bool elementPairIsOddToEven(ArrayRef<int> Mask, int Elt) {
9765 assert(Elt % 2 == 0);
9766 return Mask[Elt] >= 0 && Mask[Elt + 1] >= 0 && (Mask[Elt] & 1) &&
9767 !(Mask[Elt + 1] & 1);
9768}
9769
9770SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
9771 SelectionDAG &DAG) const {
9772 SDLoc SL(Op);
9773 EVT ResultVT = Op.getValueType();
9774 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
9775 MVT EltVT = ResultVT.getVectorElementType().getSimpleVT();
9776 const int NewSrcNumElts = 2;
9777 MVT PackVT = MVT::getVectorVT(VT: EltVT, NumElements: NewSrcNumElts);
9778 int SrcNumElts = Op.getOperand(i: 0).getValueType().getVectorNumElements();
9779
9780 // Break up the shuffle into registers sized pieces.
9781 //
9782 // We're trying to form sub-shuffles that the register allocation pipeline
9783 // won't be able to figure out, like how to use v_pk_mov_b32 to do a register
9784 // blend or 16-bit op_sel. It should be able to figure out how to reassemble a
9785 // pair of copies into a consecutive register copy, so use the ordinary
9786 // extract_vector_elt lowering unless we can use the shuffle.
9787 //
9788 // TODO: This is a bit of hack, and we should probably always use
9789 // extract_subvector for the largest possible subvector we can (or at least
9790 // use it for PackVT aligned pieces). However we have worse support for
9791 // combines on them don't directly treat extract_subvector / insert_subvector
9792 // as legal. The DAG scheduler also ends up doing a worse job with the
9793 // extract_subvectors.
9794 const bool ShouldUseConsecutiveExtract = EltVT.getSizeInBits() == 16;
9795
9796 // vector_shuffle <0,1,6,7> lhs, rhs
9797 // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
9798 //
9799 // vector_shuffle <6,7,2,3> lhs, rhs
9800 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
9801 //
9802 // vector_shuffle <6,7,0,1> lhs, rhs
9803 // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
9804
9805 // Avoid scalarizing when both halves are reading from consecutive elements.
9806
9807 // If we're treating 2 element shuffles as legal, also create odd-to-even
9808 // shuffles of neighboring pairs.
9809 //
9810 // vector_shuffle <3,2,7,6> lhs, rhs
9811 // -> concat_vectors vector_shuffle <1, 0> (extract_subvector lhs, 0)
9812 // vector_shuffle <1, 0> (extract_subvector rhs, 2)
9813
9814 SmallVector<SDValue, 16> Pieces;
9815 for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; I += 2) {
9816 if (ShouldUseConsecutiveExtract &&
9817 elementPairIsContiguous(Mask: SVN->getMask(), Elt: I)) {
9818 const int Idx = SVN->getMaskElt(Idx: I);
9819 int VecIdx = Idx < SrcNumElts ? 0 : 1;
9820 int EltIdx = Idx < SrcNumElts ? Idx : Idx - SrcNumElts;
9821 SDValue SubVec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT,
9822 N1: SVN->getOperand(Num: VecIdx),
9823 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
9824 Pieces.push_back(Elt: SubVec);
9825 } else if (elementPairIsOddToEven(Mask: SVN->getMask(), Elt: I) &&
9826 isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: PackVT)) {
9827 int Idx0 = SVN->getMaskElt(Idx: I);
9828 int Idx1 = SVN->getMaskElt(Idx: I + 1);
9829
9830 SDValue SrcOp0 = SVN->getOperand(Num: 0);
9831 SDValue SrcOp1 = SrcOp0;
9832 if (Idx0 >= SrcNumElts) {
9833 SrcOp0 = SVN->getOperand(Num: 1);
9834 Idx0 -= SrcNumElts;
9835 }
9836
9837 if (Idx1 >= SrcNumElts) {
9838 SrcOp1 = SVN->getOperand(Num: 1);
9839 Idx1 -= SrcNumElts;
9840 }
9841
9842 int AlignedIdx0 = Idx0 & ~(NewSrcNumElts - 1);
9843 int AlignedIdx1 = Idx1 & ~(NewSrcNumElts - 1);
9844
9845 // Extract nearest even aligned piece.
9846 SDValue SubVec0 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp0,
9847 N2: DAG.getConstant(Val: AlignedIdx0, DL: SL, VT: MVT::i32));
9848 SDValue SubVec1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT: PackVT, N1: SrcOp1,
9849 N2: DAG.getConstant(Val: AlignedIdx1, DL: SL, VT: MVT::i32));
9850
9851 int NewMaskIdx0 = Idx0 - AlignedIdx0;
9852 int NewMaskIdx1 = Idx1 - AlignedIdx1;
9853
9854 SDValue Result0 = SubVec0;
9855 SDValue Result1 = SubVec0;
9856
9857 if (SubVec0 != SubVec1) {
9858 NewMaskIdx1 += NewSrcNumElts;
9859 Result1 = SubVec1;
9860 } else {
9861 Result1 = DAG.getPOISON(VT: PackVT);
9862 }
9863
9864 SDValue Shuf = DAG.getVectorShuffle(VT: PackVT, dl: SL, N1: Result0, N2: Result1,
9865 Mask: {NewMaskIdx0, NewMaskIdx1});
9866 Pieces.push_back(Elt: Shuf);
9867 } else {
9868 const int Idx0 = SVN->getMaskElt(Idx: I);
9869 const int Idx1 = SVN->getMaskElt(Idx: I + 1);
9870 int VecIdx0 = Idx0 < SrcNumElts ? 0 : 1;
9871 int VecIdx1 = Idx1 < SrcNumElts ? 0 : 1;
9872 int EltIdx0 = Idx0 < SrcNumElts ? Idx0 : Idx0 - SrcNumElts;
9873 int EltIdx1 = Idx1 < SrcNumElts ? Idx1 : Idx1 - SrcNumElts;
9874
9875 SDValue Vec0 = SVN->getOperand(Num: VecIdx0);
9876 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec0,
9877 N2: DAG.getSignedConstant(Val: EltIdx0, DL: SL, VT: MVT::i32));
9878
9879 SDValue Vec1 = SVN->getOperand(Num: VecIdx1);
9880 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec1,
9881 N2: DAG.getSignedConstant(Val: EltIdx1, DL: SL, VT: MVT::i32));
9882 Pieces.push_back(Elt: DAG.getBuildVector(VT: PackVT, DL: SL, Ops: {Elt0, Elt1}));
9883 }
9884 }
9885
9886 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: ResultVT, Ops: Pieces);
9887}
9888
9889SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
9890 SelectionDAG &DAG) const {
9891 SDValue SVal = Op.getOperand(i: 0);
9892 EVT ResultVT = Op.getValueType();
9893 EVT SValVT = SVal.getValueType();
9894 SDValue UndefVal = DAG.getPOISON(VT: SValVT);
9895 SDLoc SL(Op);
9896
9897 SmallVector<SDValue, 8> VElts;
9898 VElts.push_back(Elt: SVal);
9899 for (int I = 1, E = ResultVT.getVectorNumElements(); I < E; ++I)
9900 VElts.push_back(Elt: UndefVal);
9901
9902 return DAG.getBuildVector(VT: ResultVT, DL: SL, Ops: VElts);
9903}
9904
9905SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
9906 SelectionDAG &DAG) const {
9907 SDLoc SL(Op);
9908 EVT VT = Op.getValueType();
9909
9910 if (VT == MVT::v2f16 || VT == MVT::v2i16 || VT == MVT::v2bf16) {
9911 assert(!Subtarget->hasVOP3PInsts() && "this should be legal");
9912
9913 SDValue Lo = Op.getOperand(i: 0);
9914 SDValue Hi = Op.getOperand(i: 1);
9915
9916 // Avoid adding defined bits with the zero_extend.
9917 if (Hi.isUndef()) {
9918 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
9919 SDValue ExtLo = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
9920 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ExtLo);
9921 }
9922
9923 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Hi);
9924 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Hi);
9925
9926 SDValue ShlHi = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: Hi,
9927 N2: DAG.getConstant(Val: 16, DL: SL, VT: MVT::i32));
9928 if (Lo.isUndef())
9929 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: ShlHi);
9930
9931 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Lo);
9932 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Lo);
9933
9934 SDValue Or =
9935 DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Lo, N2: ShlHi, Flags: SDNodeFlags::Disjoint);
9936 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Or);
9937 }
9938
9939 // Split into 2-element chunks.
9940 const unsigned NumParts = VT.getVectorNumElements() / 2;
9941 EVT PartVT = MVT::getVectorVT(VT: VT.getVectorElementType().getSimpleVT(), NumElements: 2);
9942 MVT PartIntVT = MVT::getIntegerVT(BitWidth: PartVT.getSizeInBits());
9943
9944 SmallVector<SDValue> Casts;
9945 for (unsigned P = 0; P < NumParts; ++P) {
9946 SDValue Vec = DAG.getBuildVector(
9947 VT: PartVT, DL: SL, Ops: {Op.getOperand(i: P * 2), Op.getOperand(i: P * 2 + 1)});
9948 Casts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: PartIntVT, Operand: Vec));
9949 }
9950
9951 SDValue Blend =
9952 DAG.getBuildVector(VT: MVT::getVectorVT(VT: PartIntVT, NumElements: NumParts), DL: SL, Ops: Casts);
9953 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Blend);
9954}
9955
9956bool SITargetLowering::isOffsetFoldingLegal(
9957 const GlobalAddressSDNode *GA) const {
9958 // Named barriers have fixed, non-relocated LDS addresses, so a constant
9959 // offset into an array of them can be folded into the address.
9960 if (GA->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
9961 const auto *GV = dyn_cast<GlobalVariable>(Val: GA->getGlobal());
9962 return GV && AMDGPU::isNamedBarrier(GV: *GV);
9963 }
9964
9965 // OSes that use ELF REL relocations (instead of RELA) can only store a
9966 // 32-bit addend in the instruction, so it is not safe to allow offset folding
9967 // which can create arbitrary 64-bit addends. (This is only a problem for
9968 // R_AMDGPU_*32_HI relocations since other relocation types are unaffected by
9969 // the high 32 bits of the addend.)
9970 //
9971 // This should be kept in sync with how HasRelocationAddend is initialized in
9972 // the constructor of ELFAMDGPUAsmBackend.
9973 if (!Subtarget->isAmdHsaOS())
9974 return false;
9975
9976 // We can fold offsets for anything that doesn't require a GOT relocation.
9977 return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
9978 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
9979 GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
9980 !shouldEmitGOTReloc(GV: GA->getGlobal());
9981}
9982
9983static SDValue
9984buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
9985 const SDLoc &DL, int64_t Offset, EVT PtrVT,
9986 unsigned GAFlags = SIInstrInfo::MO_NONE) {
9987 assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
9988 // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
9989 // lowered to the following code sequence:
9990 //
9991 // For constant address space:
9992 // s_getpc_b64 s[0:1]
9993 // s_add_u32 s0, s0, $symbol
9994 // s_addc_u32 s1, s1, 0
9995 //
9996 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
9997 // a fixup or relocation is emitted to replace $symbol with a literal
9998 // constant, which is a pc-relative offset from the encoding of the $symbol
9999 // operand to the global variable.
10000 //
10001 // For global address space:
10002 // s_getpc_b64 s[0:1]
10003 // s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
10004 // s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
10005 //
10006 // s_getpc_b64 returns the address of the s_add_u32 instruction and then
10007 // fixups or relocations are emitted to replace $symbol@*@lo and
10008 // $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
10009 // which is a 64-bit pc-relative offset from the encoding of the $symbol
10010 // operand to the global variable.
10011 if (((const GCNSubtarget &)DAG.getSubtarget()).has64BitLiterals()) {
10012 assert(GAFlags != SIInstrInfo::MO_NONE);
10013
10014 SDValue Ptr =
10015 DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i64, offset: Offset, TargetFlags: GAFlags + 2);
10016 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET64, DL, VT: PtrVT, Operand: Ptr);
10017 }
10018
10019 SDValue PtrLo = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags);
10020 SDValue PtrHi;
10021 if (GAFlags == SIInstrInfo::MO_NONE)
10022 PtrHi = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
10023 else
10024 PtrHi = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: Offset, TargetFlags: GAFlags + 1);
10025 return DAG.getNode(Opcode: AMDGPUISD::PC_ADD_REL_OFFSET, DL, VT: PtrVT, N1: PtrLo, N2: PtrHi);
10026}
10027
10028SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
10029 SDValue Op,
10030 SelectionDAG &DAG) const {
10031 GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Val&: Op);
10032 SDLoc DL(GSD);
10033 EVT PtrVT = Op.getValueType();
10034
10035 const GlobalValue *GV = GSD->getGlobal();
10036 if ((GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
10037 shouldUseLDSConstAddress(GV)) ||
10038 GSD->getAddressSpace() == AMDGPUAS::REGION_ADDRESS ||
10039 GSD->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) {
10040 if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
10041 GV->hasExternalLinkage()) {
10042 const GlobalVariable &GVar = *cast<GlobalVariable>(Val: GV);
10043 // HIP uses an unsized array `extern __shared__ T s[]` or similar
10044 // zero-sized type in other languages to declare the dynamic shared
10045 // memory which size is not known at the compile time. They will be
10046 // allocated by the runtime and placed directly after the static
10047 // allocated ones. They all share the same offset.
10048 if (GVar.getGlobalSize(DL: GVar.getDataLayout()) == 0) {
10049 assert(PtrVT == MVT::i32 && "32-bit pointer is expected.");
10050 // Adjust alignment for that dynamic shared memory array.
10051 Function &F = DAG.getMachineFunction().getFunction();
10052 MFI->setDynLDSAlign(F, GV: GVar);
10053 MFI->setUsesDynamicLDS(true);
10054 return SDValue(
10055 DAG.getMachineNode(Opcode: AMDGPU::GET_GROUPSTATICSIZE, dl: DL, VT: PtrVT), 0);
10056 }
10057 }
10058 return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
10059 }
10060
10061 if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
10062 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: GSD->getOffset(),
10063 TargetFlags: SIInstrInfo::MO_ABS32_LO);
10064 return DAG.getNode(Opcode: AMDGPUISD::LDS, DL, VT: MVT::i32, Operand: GA);
10065 }
10066
10067 if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS()) {
10068 if (Subtarget->has64BitLiterals()) {
10069 SDValue Addr = DAG.getTargetGlobalAddress(
10070 GV, DL, VT: MVT::i64, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS64);
10071 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B64, dl: DL, VT: MVT::i64, Op1: Addr),
10072 0);
10073 }
10074
10075 SDValue AddrLo = DAG.getTargetGlobalAddress(
10076 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_LO);
10077 AddrLo = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrLo), 0};
10078
10079 SDValue AddrHi = DAG.getTargetGlobalAddress(
10080 GV, DL, VT: MVT::i32, offset: GSD->getOffset(), TargetFlags: SIInstrInfo::MO_ABS32_HI);
10081 AddrHi = {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: AddrHi), 0};
10082
10083 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: AddrLo, N2: AddrHi);
10084 }
10085
10086 if (shouldEmitFixup(GV))
10087 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT);
10088
10089 if (shouldEmitPCReloc(GV))
10090 return buildPCRelGlobalAddress(DAG, GV, DL, Offset: GSD->getOffset(), PtrVT,
10091 GAFlags: SIInstrInfo::MO_REL32);
10092
10093 SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, Offset: 0, PtrVT,
10094 GAFlags: SIInstrInfo::MO_GOTPCREL32);
10095 PointerType *PtrTy =
10096 PointerType::get(C&: *DAG.getContext(), AddressSpace: AMDGPUAS::CONSTANT_ADDRESS);
10097 const DataLayout &DataLayout = DAG.getDataLayout();
10098 Align Alignment = DataLayout.getABITypeAlign(Ty: PtrTy);
10099 MachinePointerInfo PtrInfo =
10100 MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction());
10101
10102 return DAG.getLoad(VT: PtrVT, dl: DL, Chain: DAG.getEntryNode(), Ptr: GOTAddr, PtrInfo, Alignment,
10103 MMOFlags: MachineMemOperand::MODereferenceable |
10104 MachineMemOperand::MOInvariant);
10105}
10106
10107SDValue SITargetLowering::LowerExternalSymbol(SDValue Op,
10108 SelectionDAG &DAG) const {
10109 // TODO: Handle this. It should be mostly the same as LowerGlobalAddress.
10110 const Function &Fn = DAG.getMachineFunction().getFunction();
10111 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10112 Fn, "unsupported external symbol", Op.getDebugLoc()));
10113 return DAG.getPOISON(VT: Op.getValueType());
10114}
10115
10116SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
10117 const SDLoc &DL, SDValue V) const {
10118 // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
10119 // the destination register.
10120 //
10121 // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
10122 // so we will end up with redundant moves to m0.
10123 //
10124 // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
10125
10126 // A Null SDValue creates a glue result.
10127 SDNode *M0 = DAG.getMachineNode(Opcode: AMDGPU::SI_INIT_M0, dl: DL, VT1: MVT::Other, VT2: MVT::Glue,
10128 Op1: V, Op2: Chain);
10129 return SDValue(M0, 0);
10130}
10131
10132SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op,
10133 MVT VT,
10134 unsigned Offset) const {
10135 SDLoc SL(Op);
10136 SDValue Param = lowerKernargMemParameter(
10137 DAG, VT: MVT::i32, MemVT: MVT::i32, SL, Chain: DAG.getEntryNode(), Offset, Alignment: Align(4), Signed: false);
10138 // The local size values will have the hi 16-bits as zero.
10139 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Param,
10140 N2: DAG.getValueType(VT));
10141}
10142
10143static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10144 EVT VT) {
10145 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10146 DAG.getMachineFunction().getFunction(),
10147 "non-hsa intrinsic with hsa target", DL.getDebugLoc()));
10148 return DAG.getPOISON(VT);
10149}
10150
10151static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
10152 EVT VT) {
10153 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10154 DAG.getMachineFunction().getFunction(),
10155 "intrinsic not supported on subtarget", DL.getDebugLoc()));
10156 return DAG.getPOISON(VT);
10157}
10158
10159static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
10160 ArrayRef<SDValue> Elts) {
10161 assert(!Elts.empty());
10162 MVT Type;
10163 unsigned NumElts = Elts.size();
10164
10165 if (NumElts <= 12) {
10166 Type = MVT::getVectorVT(VT: MVT::f32, NumElements: NumElts);
10167 } else {
10168 assert(Elts.size() <= 16);
10169 Type = MVT::v16f32;
10170 NumElts = 16;
10171 }
10172
10173 SmallVector<SDValue, 16> VecElts(NumElts);
10174 for (unsigned i = 0; i < Elts.size(); ++i) {
10175 SDValue Elt = Elts[i];
10176 if (Elt.getValueType() != MVT::f32)
10177 Elt = DAG.getBitcast(VT: MVT::f32, V: Elt);
10178 VecElts[i] = Elt;
10179 }
10180 for (unsigned i = Elts.size(); i < NumElts; ++i)
10181 VecElts[i] = DAG.getPOISON(VT: MVT::f32);
10182
10183 if (NumElts == 1)
10184 return VecElts[0];
10185 return DAG.getBuildVector(VT: Type, DL, Ops: VecElts);
10186}
10187
10188static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT,
10189 SDValue Src, int ExtraElts) {
10190 EVT SrcVT = Src.getValueType();
10191
10192 SmallVector<SDValue, 8> Elts;
10193
10194 if (SrcVT.isVector())
10195 DAG.ExtractVectorElements(Op: Src, Args&: Elts);
10196 else
10197 Elts.push_back(Elt: Src);
10198
10199 SDValue Undef = DAG.getPOISON(VT: SrcVT.getScalarType());
10200 while (ExtraElts--)
10201 Elts.push_back(Elt: Undef);
10202
10203 return DAG.getBuildVector(VT: CastVT, DL, Ops: Elts);
10204}
10205
10206// Re-construct the required return value for a image load intrinsic.
10207// This is more complicated due to the optional use TexFailCtrl which means the
10208// required return type is an aggregate
10209static SDValue constructRetValue(SelectionDAG &DAG, MachineSDNode *Result,
10210 ArrayRef<EVT> ResultTypes, bool IsTexFail,
10211 bool Unpacked, bool IsD16, int DMaskPop,
10212 int NumVDataDwords, bool IsAtomicPacked16Bit,
10213 const SDLoc &DL) {
10214 // Determine the required return type. This is the same regardless of
10215 // IsTexFail flag
10216 EVT ReqRetVT = ResultTypes[0];
10217 int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1;
10218 int NumDataDwords = ((IsD16 && !Unpacked) || IsAtomicPacked16Bit)
10219 ? (ReqRetNumElts + 1) / 2
10220 : ReqRetNumElts;
10221
10222 int MaskPopDwords = (!IsD16 || Unpacked) ? DMaskPop : (DMaskPop + 1) / 2;
10223
10224 MVT DataDwordVT =
10225 NumDataDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: NumDataDwords);
10226
10227 MVT MaskPopVT =
10228 MaskPopDwords == 1 ? MVT::i32 : MVT::getVectorVT(VT: MVT::i32, NumElements: MaskPopDwords);
10229
10230 SDValue Data(Result, 0);
10231 SDValue TexFail;
10232
10233 if (DMaskPop > 0 && Data.getValueType() != MaskPopVT) {
10234 SDValue ZeroIdx = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
10235 if (MaskPopVT.isVector()) {
10236 Data = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: MaskPopVT,
10237 N1: SDValue(Result, 0), N2: ZeroIdx);
10238 } else {
10239 Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MaskPopVT,
10240 N1: SDValue(Result, 0), N2: ZeroIdx);
10241 }
10242 }
10243
10244 if (DataDwordVT.isVector() && !IsAtomicPacked16Bit)
10245 Data = padEltsToUndef(DAG, DL, CastVT: DataDwordVT, Src: Data,
10246 ExtraElts: NumDataDwords - MaskPopDwords);
10247
10248 if (IsD16)
10249 Data = adjustLoadValueTypeImpl(Result: Data, LoadVT: ReqRetVT, DL, DAG, Unpacked);
10250
10251 EVT LegalReqRetVT = ReqRetVT;
10252 if (!ReqRetVT.isVector()) {
10253 if (!Data.getValueType().isInteger())
10254 Data = DAG.getNode(Opcode: ISD::BITCAST, DL,
10255 VT: Data.getValueType().changeTypeToInteger(), Operand: Data);
10256 Data = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ReqRetVT.changeTypeToInteger(), Operand: Data);
10257 } else {
10258 // We need to widen the return vector to a legal type
10259 if ((ReqRetVT.getVectorNumElements() % 2) == 1 &&
10260 ReqRetVT.getVectorElementType().getSizeInBits() == 16) {
10261 LegalReqRetVT =
10262 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ReqRetVT.getVectorElementType(),
10263 NumElements: ReqRetVT.getVectorNumElements() + 1);
10264 }
10265 }
10266 Data = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LegalReqRetVT, Operand: Data);
10267
10268 if (IsTexFail) {
10269 TexFail =
10270 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: SDValue(Result, 0),
10271 N2: DAG.getConstant(Val: MaskPopDwords, DL, VT: MVT::i32));
10272
10273 return DAG.getMergeValues(Ops: {Data, TexFail, SDValue(Result, 1)}, dl: DL);
10274 }
10275
10276 if (Result->getNumValues() == 1)
10277 return Data;
10278
10279 return DAG.getMergeValues(Ops: {Data, SDValue(Result, 1)}, dl: DL);
10280}
10281
10282static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
10283 SDValue *LWE, bool &IsTexFail) {
10284 auto *TexFailCtrlConst = cast<ConstantSDNode>(Val: TexFailCtrl.getNode());
10285
10286 uint64_t Value = TexFailCtrlConst->getZExtValue();
10287 if (Value) {
10288 IsTexFail = true;
10289 }
10290
10291 SDLoc DL(TexFailCtrlConst);
10292 *TFE = DAG.getTargetConstant(Val: (Value & 0x1) ? 1 : 0, DL, VT: MVT::i32);
10293 Value &= ~(uint64_t)0x1;
10294 *LWE = DAG.getTargetConstant(Val: (Value & 0x2) ? 1 : 0, DL, VT: MVT::i32);
10295 Value &= ~(uint64_t)0x2;
10296
10297 return Value == 0;
10298}
10299
10300static void packImage16bitOpsToDwords(SelectionDAG &DAG, SDValue Op,
10301 MVT PackVectorVT,
10302 SmallVectorImpl<SDValue> &PackedAddrs,
10303 unsigned DimIdx, unsigned EndIdx,
10304 unsigned NumGradients) {
10305 SDLoc DL(Op);
10306 for (unsigned I = DimIdx; I < EndIdx; I++) {
10307 SDValue Addr = Op.getOperand(i: I);
10308
10309 // Gradients are packed with undef for each coordinate.
10310 // In <hi 16 bit>,<lo 16 bit> notation, the registers look like this:
10311 // 1D: undef,dx/dh; undef,dx/dv
10312 // 2D: dy/dh,dx/dh; dy/dv,dx/dv
10313 // 3D: dy/dh,dx/dh; undef,dz/dh; dy/dv,dx/dv; undef,dz/dv
10314 if (((I + 1) >= EndIdx) ||
10315 ((NumGradients / 2) % 2 == 1 && (I == DimIdx + (NumGradients / 2) - 1 ||
10316 I == DimIdx + NumGradients - 1))) {
10317 if (Addr.getValueType() != MVT::i16)
10318 Addr = DAG.getBitcast(VT: MVT::i16, V: Addr);
10319 Addr = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Addr);
10320 } else {
10321 Addr = DAG.getBuildVector(VT: PackVectorVT, DL, Ops: {Addr, Op.getOperand(i: I + 1)});
10322 I++;
10323 }
10324 Addr = DAG.getBitcast(VT: MVT::f32, V: Addr);
10325 PackedAddrs.push_back(Elt: Addr);
10326 }
10327}
10328
10329SDValue SITargetLowering::lowerImage(SDValue Op,
10330 const AMDGPU::ImageDimIntrinsicInfo *Intr,
10331 SelectionDAG &DAG, bool WithChain) const {
10332 SDLoc DL(Op);
10333 MachineFunction &MF = DAG.getMachineFunction();
10334 const GCNSubtarget *ST = &MF.getSubtarget<GCNSubtarget>();
10335 unsigned IntrOpcode = Intr->BaseOpcode;
10336 // For image atomic: use no-return opcode if result is unused.
10337 if (Intr->AtomicNoRetBaseOpcode != Intr->BaseOpcode &&
10338 !Op.getNode()->hasAnyUseOfValue(Value: 0))
10339 IntrOpcode = Intr->AtomicNoRetBaseOpcode;
10340 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
10341 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: IntrOpcode);
10342 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(DimEnum: Intr->Dim);
10343 bool IsGFX10Plus = AMDGPU::isGFX10Plus(STI: *Subtarget);
10344 bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
10345 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
10346 bool IsGFX13 = AMDGPU::isGFX13(STI: *Subtarget);
10347
10348 SmallVector<EVT, 3> ResultTypes(Op->values());
10349 SmallVector<EVT, 3> OrigResultTypes(Op->values());
10350 if (BaseOpcode->NoReturn && BaseOpcode->Atomic)
10351 ResultTypes.erase(CI: &ResultTypes[0]);
10352
10353 bool IsD16 = false;
10354 bool IsG16 = false;
10355 bool IsA16 = false;
10356 SDValue VData;
10357 int NumVDataDwords = 0;
10358 bool AdjustRetType = false;
10359 bool IsAtomicPacked16Bit = false;
10360
10361 // Offset of intrinsic arguments
10362 const unsigned ArgOffset = WithChain ? 2 : 1;
10363
10364 unsigned DMask;
10365 unsigned DMaskLanes = 0;
10366
10367 if (BaseOpcode->Atomic) {
10368 VData = Op.getOperand(i: 2);
10369
10370 IsAtomicPacked16Bit =
10371 (IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
10372 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16_NORTN ||
10373 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16 ||
10374 IntrOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16_NORTN);
10375
10376 bool Is64Bit = VData.getValueSizeInBits() == 64;
10377 if (BaseOpcode->AtomicX2) {
10378 SDValue VData2 = Op.getOperand(i: 3);
10379 VData = DAG.getBuildVector(VT: Is64Bit ? MVT::v2i64 : MVT::v2i32, DL,
10380 Ops: {VData, VData2});
10381 if (Is64Bit)
10382 VData = DAG.getBitcast(VT: MVT::v4i32, V: VData);
10383
10384 if (!BaseOpcode->NoReturn)
10385 ResultTypes[0] = Is64Bit ? MVT::v2i64 : MVT::v2i32;
10386
10387 DMask = Is64Bit ? 0xf : 0x3;
10388 NumVDataDwords = Is64Bit ? 4 : 2;
10389 } else {
10390 DMask = Is64Bit ? 0x3 : 0x1;
10391 NumVDataDwords = Is64Bit ? 2 : 1;
10392 }
10393 } else {
10394 DMask = Op->getConstantOperandVal(Num: ArgOffset + Intr->DMaskIndex);
10395 DMaskLanes = BaseOpcode->Gather4 ? 4 : llvm::popcount(Value: DMask);
10396
10397 if (BaseOpcode->Store) {
10398 VData = Op.getOperand(i: 2);
10399
10400 MVT StoreVT = VData.getSimpleValueType();
10401 if (StoreVT.getScalarType() == MVT::f16) {
10402 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10403 return Op; // D16 is unsupported for this instruction
10404
10405 IsD16 = true;
10406 VData = handleD16VData(VData, DAG, ImageStore: true);
10407 }
10408
10409 NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
10410 } else if (!BaseOpcode->NoReturn) {
10411 // Work out the num dwords based on the dmask popcount and underlying type
10412 // and whether packing is supported.
10413 MVT LoadVT = ResultTypes[0].getSimpleVT();
10414 if (LoadVT.getScalarType() == MVT::f16) {
10415 if (!Subtarget->hasD16Images() || !BaseOpcode->HasD16)
10416 return Op; // D16 is unsupported for this instruction
10417
10418 IsD16 = true;
10419 }
10420
10421 // Confirm that the return type is large enough for the dmask specified
10422 if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) ||
10423 (!LoadVT.isVector() && DMaskLanes > 1))
10424 return Op;
10425
10426 // The sq block of gfx8 and gfx9 do not estimate register use correctly
10427 // for d16 image_gather4, image_gather4_l, and image_gather4_lz
10428 // instructions.
10429 if (IsD16 && !Subtarget->hasUnpackedD16VMem() &&
10430 !(BaseOpcode->Gather4 && Subtarget->hasImageGather4D16Bug()))
10431 NumVDataDwords = (DMaskLanes + 1) / 2;
10432 else
10433 NumVDataDwords = DMaskLanes;
10434
10435 AdjustRetType = true;
10436 }
10437 }
10438
10439 unsigned VAddrEnd = ArgOffset + Intr->VAddrEnd;
10440 SmallVector<SDValue, 4> VAddrs;
10441
10442 // Check for 16 bit addresses or derivatives and pack if true.
10443 MVT VAddrVT =
10444 Op.getOperand(i: ArgOffset + Intr->GradientStart).getSimpleValueType();
10445 MVT VAddrScalarVT = VAddrVT.getScalarType();
10446 MVT GradPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10447 IsG16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10448
10449 VAddrVT = Op.getOperand(i: ArgOffset + Intr->CoordStart).getSimpleValueType();
10450 VAddrScalarVT = VAddrVT.getScalarType();
10451 MVT AddrPackVectorVT = VAddrScalarVT == MVT::f16 ? MVT::v2f16 : MVT::v2i16;
10452 IsA16 = VAddrScalarVT == MVT::f16 || VAddrScalarVT == MVT::i16;
10453
10454 // Push back extra arguments.
10455 for (unsigned I = Intr->VAddrStart; I < Intr->GradientStart; I++) {
10456 if (IsA16 && (Op.getOperand(i: ArgOffset + I).getValueType() == MVT::f16)) {
10457 assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
10458 // Special handling of bias when A16 is on. Bias is of type half but
10459 // occupies full 32-bit.
10460 SDValue Bias = DAG.getBuildVector(
10461 VT: MVT::v2f16, DL,
10462 Ops: {Op.getOperand(i: ArgOffset + I), DAG.getPOISON(VT: MVT::f16)});
10463 VAddrs.push_back(Elt: Bias);
10464 } else {
10465 assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
10466 "Bias needs to be converted to 16 bit in A16 mode");
10467 VAddrs.push_back(Elt: Op.getOperand(i: ArgOffset + I));
10468 }
10469 }
10470
10471 if (BaseOpcode->Gradients && !ST->hasG16() && (IsA16 != IsG16)) {
10472 // 16 bit gradients are supported, but are tied to the A16 control
10473 // so both gradients and addresses must be 16 bit
10474 LLVM_DEBUG(
10475 dbgs() << "Failed to lower image intrinsic: 16 bit addresses "
10476 "require 16 bit args for both gradients and addresses");
10477 return Op;
10478 }
10479
10480 if (IsA16) {
10481 if (!ST->hasA16()) {
10482 LLVM_DEBUG(dbgs() << "Failed to lower image intrinsic: Target does not "
10483 "support 16 bit addresses\n");
10484 return Op;
10485 }
10486 }
10487
10488 // We've dealt with incorrect input so we know that if IsA16, IsG16
10489 // are set then we have to compress/pack operands (either address,
10490 // gradient or both)
10491 // In the case where a16 and gradients are tied (no G16 support) then we
10492 // have already verified that both IsA16 and IsG16 are true
10493 if (BaseOpcode->Gradients && IsG16 && ST->hasG16()) {
10494 // Activate g16
10495 const AMDGPU::MIMGG16MappingInfo *G16MappingInfo =
10496 AMDGPU::getMIMGG16MappingInfo(G: Intr->BaseOpcode);
10497 IntrOpcode = G16MappingInfo->G16; // set new opcode to variant with _g16
10498 }
10499
10500 // Add gradients (packed or unpacked)
10501 if (IsG16) {
10502 // Pack the gradients
10503 // const int PackEndIdx = IsA16 ? VAddrEnd : (ArgOffset + Intr->CoordStart);
10504 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: GradPackVectorVT, PackedAddrs&: VAddrs,
10505 DimIdx: ArgOffset + Intr->GradientStart,
10506 EndIdx: ArgOffset + Intr->CoordStart, NumGradients: Intr->NumGradients);
10507 } else {
10508 for (unsigned I = ArgOffset + Intr->GradientStart;
10509 I < ArgOffset + Intr->CoordStart; I++)
10510 VAddrs.push_back(Elt: Op.getOperand(i: I));
10511 }
10512
10513 // Add addresses (packed or unpacked)
10514 if (IsA16) {
10515 packImage16bitOpsToDwords(DAG, Op, PackVectorVT: AddrPackVectorVT, PackedAddrs&: VAddrs,
10516 DimIdx: ArgOffset + Intr->CoordStart, EndIdx: VAddrEnd,
10517 NumGradients: 0 /* No gradients */);
10518 } else {
10519 // Add uncompressed address
10520 for (unsigned I = ArgOffset + Intr->CoordStart; I < VAddrEnd; I++)
10521 VAddrs.push_back(Elt: Op.getOperand(i: I));
10522 }
10523
10524 // If the register allocator cannot place the address registers contiguously
10525 // without introducing moves, then using the non-sequential address encoding
10526 // is always preferable, since it saves VALU instructions and is usually a
10527 // wash in terms of code size or even better.
10528 //
10529 // However, we currently have no way of hinting to the register allocator that
10530 // MIMG addresses should be placed contiguously when it is possible to do so,
10531 // so force non-NSA for the common 2-address case as a heuristic.
10532 //
10533 // SIShrinkInstructions will convert NSA encodings to non-NSA after register
10534 // allocation when possible.
10535 //
10536 // Partial NSA is allowed on GFX11+ where the final register is a contiguous
10537 // set of the remaining addresses.
10538 const unsigned NSAMaxSize = ST->getNSAMaxSize(HasSampler: BaseOpcode->Sampler);
10539 const bool HasPartialNSAEncoding = ST->hasPartialNSAEncoding();
10540 const bool UseNSA = ST->hasNSAEncoding() &&
10541 VAddrs.size() >= ST->getNSAThreshold(MF) &&
10542 (VAddrs.size() <= NSAMaxSize || HasPartialNSAEncoding);
10543 const bool UsePartialNSA =
10544 UseNSA && HasPartialNSAEncoding && VAddrs.size() > NSAMaxSize;
10545
10546 SDValue VAddr;
10547 if (UsePartialNSA) {
10548 VAddr = getBuildDwordsVector(DAG, DL,
10549 Elts: ArrayRef(VAddrs).drop_front(N: NSAMaxSize - 1));
10550 } else if (!UseNSA) {
10551 VAddr = getBuildDwordsVector(DAG, DL, Elts: VAddrs);
10552 }
10553
10554 SDValue True = DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1);
10555 SDValue False = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1);
10556 SDValue Unorm;
10557 if (!BaseOpcode->Sampler) {
10558 Unorm = True;
10559 } else {
10560 uint64_t UnormConst =
10561 Op.getConstantOperandVal(i: ArgOffset + Intr->UnormIndex);
10562
10563 Unorm = UnormConst ? True : False;
10564 }
10565
10566 SDValue TFE;
10567 SDValue LWE;
10568 SDValue TexFail = Op.getOperand(i: ArgOffset + Intr->TexFailCtrlIndex);
10569 bool IsTexFail = false;
10570 if (!parseTexFail(TexFailCtrl: TexFail, DAG, TFE: &TFE, LWE: &LWE, IsTexFail))
10571 return Op;
10572
10573 if (IsTexFail) {
10574 if (!DMaskLanes) {
10575 // Expecting to get an error flag since TFC is on - and dmask is 0
10576 // Force dmask to be at least 1 otherwise the instruction will fail
10577 DMask = 0x1;
10578 DMaskLanes = 1;
10579 NumVDataDwords = 1;
10580 }
10581 NumVDataDwords += 1;
10582 AdjustRetType = true;
10583 }
10584
10585 // Has something earlier tagged that the return type needs adjusting
10586 // This happens if the instruction is a load or has set TexFailCtrl flags
10587 if (AdjustRetType) {
10588 // NumVDataDwords reflects the true number of dwords required in the return
10589 // type
10590 if (DMaskLanes == 0 && !BaseOpcode->Store) {
10591 // This is a no-op load. This can be eliminated
10592 SDValue Undef = DAG.getPOISON(VT: Op.getValueType());
10593 if (isa<MemSDNode>(Val: Op))
10594 return DAG.getMergeValues(Ops: {Undef, Op.getOperand(i: 0)}, dl: DL);
10595 return Undef;
10596 }
10597
10598 EVT NewVT = NumVDataDwords > 1 ? EVT::getVectorVT(Context&: *DAG.getContext(),
10599 VT: MVT::i32, NumElements: NumVDataDwords)
10600 : MVT::i32;
10601
10602 ResultTypes[0] = NewVT;
10603 if (ResultTypes.size() == 3) {
10604 // Original result was aggregate type used for TexFailCtrl results
10605 // The actual instruction returns as a vector type which has now been
10606 // created. Remove the aggregate result.
10607 ResultTypes.erase(CI: &ResultTypes[1]);
10608 }
10609 }
10610
10611 unsigned CPol = Op.getConstantOperandVal(i: ArgOffset + Intr->CachePolicyIndex);
10612 // Keep GLC only when the atomic's result is actually used.
10613 if (BaseOpcode->Atomic && !BaseOpcode->NoReturn)
10614 CPol |= AMDGPU::CPol::GLC;
10615 if (CPol & ~((IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12) |
10616 AMDGPU::CPol::VOLATILE))
10617 return Op;
10618
10619 SmallVector<SDValue, 26> Ops;
10620 if (BaseOpcode->Store || BaseOpcode->Atomic)
10621 Ops.push_back(Elt: VData); // vdata
10622 if (UsePartialNSA) {
10623 append_range(C&: Ops, R: ArrayRef(VAddrs).take_front(N: NSAMaxSize - 1));
10624 Ops.push_back(Elt: VAddr);
10625 } else if (UseNSA)
10626 append_range(C&: Ops, R&: VAddrs);
10627 else
10628 Ops.push_back(Elt: VAddr);
10629 SDValue Rsrc = Op.getOperand(i: ArgOffset + Intr->RsrcIndex);
10630 EVT RsrcVT = Rsrc.getValueType();
10631 if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32)
10632 return Op;
10633 Ops.push_back(Elt: Rsrc);
10634 if (BaseOpcode->Sampler) {
10635 SDValue Samp = Op.getOperand(i: ArgOffset + Intr->SampIndex);
10636 if (Samp.getValueType() != MVT::v4i32)
10637 return Op;
10638 Ops.push_back(Elt: Samp);
10639 }
10640 Ops.push_back(Elt: DAG.getTargetConstant(Val: DMask, DL, VT: MVT::i32));
10641 if (IsGFX10Plus)
10642 Ops.push_back(Elt: DAG.getTargetConstant(Val: DimInfo->Encoding, DL, VT: MVT::i32));
10643 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10644 Ops.push_back(Elt: Unorm);
10645 Ops.push_back(Elt: DAG.getTargetConstant(Val: CPol, DL, VT: MVT::i32));
10646 Ops.push_back(Elt: IsA16 && // r128, a16 for gfx9
10647 ST->hasFeature(Feature: AMDGPU::FeatureR128A16)
10648 ? True
10649 : False);
10650 if (IsGFX10Plus)
10651 Ops.push_back(Elt: IsA16 ? True : False);
10652
10653 if (!Subtarget->hasGFX90AInsts())
10654 Ops.push_back(Elt: TFE); // tfe
10655 else if (TFE->getAsZExtVal()) {
10656 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10657 DAG.getMachineFunction().getFunction(),
10658 "TFE is not supported on this GPU", DL.getDebugLoc()));
10659 }
10660
10661 if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA)
10662 Ops.push_back(Elt: LWE); // lwe
10663 if (!IsGFX10Plus)
10664 Ops.push_back(Elt: DimInfo->DA ? True : False);
10665 if (BaseOpcode->HasD16)
10666 Ops.push_back(Elt: IsD16 ? True : False);
10667 if (isa<MemSDNode>(Val: Op))
10668 Ops.push_back(Elt: Op.getOperand(i: 0)); // chain
10669
10670 int NumVAddrDwords =
10671 UseNSA ? VAddrs.size() : VAddr.getValueType().getSizeInBits() / 32;
10672 int Opcode = -1;
10673
10674 if (IsGFX13) {
10675 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx13,
10676 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10677 } else if (IsGFX12Plus) {
10678 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx12,
10679 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10680 } else if (IsGFX11Plus) {
10681 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10682 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx11NSA
10683 : AMDGPU::MIMGEncGfx11Default,
10684 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10685 } else if (IsGFX10Plus) {
10686 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode,
10687 MIMGEncoding: UseNSA ? AMDGPU::MIMGEncGfx10NSA
10688 : AMDGPU::MIMGEncGfx10Default,
10689 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10690 } else {
10691 if (Subtarget->hasGFX90AInsts()) {
10692 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx90a,
10693 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10694 if (Opcode == -1) {
10695 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10696 DAG.getMachineFunction().getFunction(),
10697 "requested image instruction is not supported on this GPU",
10698 DL.getDebugLoc()));
10699
10700 unsigned Idx = 0;
10701 SmallVector<SDValue, 3> RetValues(OrigResultTypes.size());
10702 for (EVT VT : OrigResultTypes) {
10703 if (VT == MVT::Other)
10704 RetValues[Idx++] = Op.getOperand(i: 0); // Chain
10705 else
10706 RetValues[Idx++] = DAG.getPOISON(VT);
10707 }
10708
10709 return DAG.getMergeValues(Ops: RetValues, dl: DL);
10710 }
10711 }
10712 if (Opcode == -1 &&
10713 Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10714 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx8,
10715 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10716 if (Opcode == -1)
10717 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: IntrOpcode, MIMGEncoding: AMDGPU::MIMGEncGfx6,
10718 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
10719 }
10720 if (Opcode == -1)
10721 return Op;
10722
10723 MachineSDNode *NewNode = DAG.getMachineNode(Opcode, dl: DL, ResultTys: ResultTypes, Ops);
10724 if (auto *MemOp = dyn_cast<MemSDNode>(Val&: Op)) {
10725 MachineMemOperand *MemRef = MemOp->getMemOperand();
10726 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
10727 }
10728
10729 if (BaseOpcode->NoReturn) {
10730 if (BaseOpcode->Atomic)
10731 return DAG.getMergeValues(
10732 Ops: {DAG.getPOISON(VT: OrigResultTypes[0]), SDValue(NewNode, 0)}, dl: DL);
10733
10734 return SDValue(NewNode, 0);
10735 }
10736
10737 if (BaseOpcode->AtomicX2) {
10738 SmallVector<SDValue, 1> Elt;
10739 DAG.ExtractVectorElements(Op: SDValue(NewNode, 0), Args&: Elt, Start: 0, Count: 1);
10740 return DAG.getMergeValues(Ops: {Elt[0], SDValue(NewNode, 1)}, dl: DL);
10741 }
10742
10743 return constructRetValue(DAG, Result: NewNode, ResultTypes: OrigResultTypes, IsTexFail,
10744 Unpacked: Subtarget->hasUnpackedD16VMem(), IsD16, DMaskPop: DMaskLanes,
10745 NumVDataDwords, IsAtomicPacked16Bit, DL);
10746}
10747
10748SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
10749 SDValue Offset, SDValue CachePolicy,
10750 SelectionDAG &DAG) const {
10751 MachineFunction &MF = DAG.getMachineFunction();
10752
10753 const DataLayout &DataLayout = DAG.getDataLayout();
10754 Align Alignment =
10755 DataLayout.getABITypeAlign(Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
10756
10757 MachineMemOperand *MMO = MF.getMachineMemOperand(
10758 PtrInfo: MachinePointerInfo(),
10759 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
10760 MachineMemOperand::MOInvariant,
10761 Size: VT.getStoreSize(), BaseAlignment: Alignment);
10762
10763 if (!Offset->isDivergent()) {
10764 SDValue Ops[] = {Rsrc, Offset, CachePolicy};
10765
10766 // Lower llvm.amdgcn.s.buffer.load.{i16, u16} intrinsics. Initially, the
10767 // s_buffer_load_u16 instruction is emitted for both signed and unsigned
10768 // loads. Later, DAG combiner tries to combine s_buffer_load_u16 with sext
10769 // and generates s_buffer_load_i16 (performSignExtendInRegCombine).
10770 if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) {
10771 SDValue BufferLoad =
10772 DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD_USHORT, dl: DL,
10773 VTList: DAG.getVTList(VT: MVT::i32), Ops, MemVT: VT, MMO);
10774 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: BufferLoad);
10775 }
10776
10777 // Widen vec3 load to vec4.
10778 if (VT.isVector() && VT.getVectorNumElements() == 3 &&
10779 !Subtarget->hasScalarDwordx3Loads()) {
10780 EVT WidenedVT =
10781 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
10782 auto WidenedOp = DAG.getMemIntrinsicNode(
10783 Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL, VTList: DAG.getVTList(VT: WidenedVT), Ops, MemVT: WidenedVT,
10784 MMO: MF.getMachineMemOperand(MMO, Offset: 0, Size: WidenedVT.getStoreSize()));
10785 auto Subvector = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WidenedOp,
10786 N2: DAG.getVectorIdxConstant(Val: 0, DL));
10787 return Subvector;
10788 }
10789
10790 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_LOAD, dl: DL,
10791 VTList: DAG.getVTList(VT), Ops, MemVT: VT, MMO);
10792 }
10793
10794 // We have a divergent offset. Emit a MUBUF buffer load instead. We can
10795 // assume that the buffer is unswizzled.
10796 SDValue Ops[] = {
10797 DAG.getEntryNode(), // Chain
10798 Rsrc, // rsrc
10799 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
10800 {}, // voffset
10801 {}, // soffset
10802 {}, // offset
10803 CachePolicy, // cachepolicy
10804 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
10805 };
10806 if (VT == MVT::i16 && Subtarget->hasScalarSubwordLoads()) {
10807 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3], Alignment: Align(4));
10808 return handleByteShortBufferLoads(DAG, LoadVT: VT, DL, Ops, MMO);
10809 }
10810
10811 SmallVector<SDValue, 4> Loads;
10812 unsigned NumLoads = 1;
10813 MVT LoadVT = VT.getSimpleVT();
10814 unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1;
10815 assert((LoadVT.getScalarType() == MVT::i32 ||
10816 LoadVT.getScalarType() == MVT::f32));
10817
10818 if (NumElts == 8 || NumElts == 16) {
10819 NumLoads = NumElts / 4;
10820 LoadVT = MVT::getVectorVT(VT: LoadVT.getScalarType(), NumElements: 4);
10821 }
10822
10823 SDVTList VTList = DAG.getVTList(VTs: {LoadVT, MVT::Other});
10824
10825 // Use the alignment to ensure that the required offsets will fit into the
10826 // immediate offsets.
10827 setBufferOffsets(CombinedOffset: Offset, DAG, Offsets: &Ops[3],
10828 Alignment: NumLoads > 1 ? Align(16 * NumLoads) : Align(4));
10829
10830 uint64_t InstOffset = Ops[5]->getAsZExtVal();
10831 unsigned LoadSize = LoadVT.getStoreSize();
10832 for (unsigned i = 0; i < NumLoads; ++i) {
10833 Ops[5] = DAG.getTargetConstant(Val: InstOffset + 16 * i, DL, VT: MVT::i32);
10834 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(MMO, Offset: 16 * i, Size: LoadSize);
10835 Loads.push_back(Elt: getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops,
10836 MemVT: LoadVT, MMO: LoadMMO, DAG));
10837 }
10838
10839 if (NumElts == 8 || NumElts == 16)
10840 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Loads);
10841
10842 return Loads[0];
10843}
10844
10845SDValue SITargetLowering::lowerWaveID(SelectionDAG &DAG, SDValue Op) const {
10846 // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
10847 if (!Subtarget->hasArchitectedSGPRs())
10848 return {};
10849 SDLoc SL(Op);
10850 MVT VT = MVT::i32;
10851 SDValue TTMP8 = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: AMDGPU::TTMP8, VT);
10852 return DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT, N1: TTMP8,
10853 N2: DAG.getConstant(Val: 25, DL: SL, VT), N3: DAG.getConstant(Val: 5, DL: SL, VT));
10854}
10855
10856SDValue SITargetLowering::lowerConstHwRegRead(SelectionDAG &DAG, SDValue Op,
10857 AMDGPU::Hwreg::Id HwReg,
10858 unsigned LowBit,
10859 unsigned Width) const {
10860 SDLoc SL(Op);
10861 using namespace AMDGPU::Hwreg;
10862 return {DAG.getMachineNode(
10863 Opcode: AMDGPU::S_GETREG_B32_const, dl: SL, VT: MVT::i32,
10864 Op1: DAG.getTargetConstant(Val: HwregEncoding::encode(Values: HwReg, Values: LowBit, Values: Width),
10865 DL: SL, VT: MVT::i32)),
10866 0};
10867}
10868
10869SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op,
10870 unsigned Dim,
10871 const ArgDescriptor &Arg) const {
10872 SDLoc SL(Op);
10873 MachineFunction &MF = DAG.getMachineFunction();
10874 unsigned MaxID = Subtarget->getMaxWorkitemID(Kernel: MF.getFunction(), Dimension: Dim);
10875 if (MaxID == 0)
10876 return DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
10877
10878 // It's undefined behavior if a function marked with the amdgpu-no-*
10879 // attributes uses the corresponding intrinsic.
10880 if (!Arg)
10881 return DAG.getPOISON(VT: Op->getValueType(ResNo: 0));
10882
10883 SDValue Val = loadInputValue(DAG, RC: &AMDGPU::VGPR_32RegClass, VT: MVT::i32,
10884 SL: SDLoc(DAG.getEntryNode()), Arg);
10885
10886 // Don't bother inserting AssertZext for packed IDs since we're emitting the
10887 // masking operations anyway.
10888 //
10889 // TODO: We could assert the top bit is 0 for the source copy.
10890 if (Arg.isMasked())
10891 return Val;
10892
10893 // Preserve the known bits after expansion to a copy.
10894 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_width(Value: MaxID));
10895 return DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT: MVT::i32, N1: Val,
10896 N2: DAG.getValueType(SmallVT));
10897}
10898
10899SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
10900 SelectionDAG &DAG) const {
10901 MachineFunction &MF = DAG.getMachineFunction();
10902 auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
10903
10904 EVT VT = Op.getValueType();
10905 SDLoc DL(Op);
10906 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
10907
10908 // TODO: Should this propagate fast-math-flags?
10909
10910 switch (IntrinsicID) {
10911 case Intrinsic::amdgcn_wave_reduce_min:
10912 case Intrinsic::amdgcn_wave_reduce_umin:
10913 case Intrinsic::amdgcn_wave_reduce_fmin:
10914 case Intrinsic::amdgcn_wave_reduce_max:
10915 case Intrinsic::amdgcn_wave_reduce_umax:
10916 case Intrinsic::amdgcn_wave_reduce_fmax:
10917 case Intrinsic::amdgcn_wave_reduce_add:
10918 case Intrinsic::amdgcn_wave_reduce_fadd:
10919 case Intrinsic::amdgcn_wave_reduce_sub:
10920 case Intrinsic::amdgcn_wave_reduce_fsub:
10921 case Intrinsic::amdgcn_wave_reduce_and:
10922 case Intrinsic::amdgcn_wave_reduce_or:
10923 case Intrinsic::amdgcn_wave_reduce_xor: {
10924 EVT SrcVT = Op.getOperand(i: 1).getValueType();
10925 if (SrcVT.getFixedSizeInBits() == 16) {
10926 bool IsFPOp = SrcVT.isFloatingPoint();
10927 bool NeedsSignExt = IntrinsicID == Intrinsic::amdgcn_wave_reduce_min ||
10928 IntrinsicID == Intrinsic::amdgcn_wave_reduce_max ||
10929 IntrinsicID == Intrinsic::amdgcn_wave_reduce_add ||
10930 IntrinsicID == Intrinsic::amdgcn_wave_reduce_sub;
10931 unsigned ExtOpc = IsFPOp ? ISD::FP_EXTEND
10932 : NeedsSignExt ? ISD::SIGN_EXTEND
10933 : ISD::ZERO_EXTEND;
10934 auto SrcType = IsFPOp ? MVT::f16 : MVT::i16;
10935 auto ExtType = IsFPOp ? MVT::f32 : MVT::i32;
10936 SDValue ExtendedSrc = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtType, Operand: Op.getOperand(i: 1));
10937 SDValue Strategy = Op.getOperand(i: 2);
10938 SDValue Result = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ExtType,
10939 N1: Op.getOperand(i: 0), N2: ExtendedSrc, N3: Strategy);
10940 if (IsFPOp)
10941 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: SrcType, N1: Result,
10942 N2: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32));
10943 else
10944 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcType, Operand: Result);
10945 }
10946 return SDValue();
10947 }
10948 case Intrinsic::amdgcn_implicit_buffer_ptr: {
10949 if (getSubtarget()->isAmdHsaOrMesa(F: MF.getFunction()))
10950 return emitNonHSAIntrinsicError(DAG, DL, VT);
10951 return getPreloadedValue(DAG, MFI: *MFI, VT,
10952 PVID: AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
10953 }
10954 case Intrinsic::amdgcn_dispatch_ptr:
10955 case Intrinsic::amdgcn_queue_ptr: {
10956 if (!Subtarget->isAmdHsaOrMesa(F: MF.getFunction())) {
10957 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
10958 MF.getFunction(), "unsupported hsa intrinsic without hsa target",
10959 DL.getDebugLoc()));
10960 return DAG.getPOISON(VT);
10961 }
10962
10963 auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr
10964 ? AMDGPUFunctionArgInfo::DISPATCH_PTR
10965 : AMDGPUFunctionArgInfo::QUEUE_PTR;
10966 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: RegID);
10967 }
10968 case Intrinsic::amdgcn_implicitarg_ptr: {
10969 if (MFI->isEntryFunction())
10970 return getImplicitArgPtr(DAG, SL: DL);
10971 return getPreloadedValue(DAG, MFI: *MFI, VT,
10972 PVID: AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
10973 }
10974 case Intrinsic::amdgcn_kernarg_segment_ptr: {
10975 if (!AMDGPU::isKernel(F: MF.getFunction())) {
10976 // This only makes sense to call in a kernel, so just lower to null.
10977 return DAG.getConstant(Val: 0, DL, VT);
10978 }
10979
10980 return getPreloadedValue(DAG, MFI: *MFI, VT,
10981 PVID: AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
10982 }
10983 case Intrinsic::amdgcn_dispatch_id: {
10984 return getPreloadedValue(DAG, MFI: *MFI, VT, PVID: AMDGPUFunctionArgInfo::DISPATCH_ID);
10985 }
10986 case Intrinsic::amdgcn_rcp:
10987 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT, Operand: Op.getOperand(i: 1));
10988 case Intrinsic::amdgcn_rsq:
10989 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
10990 case Intrinsic::amdgcn_rsq_legacy:
10991 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10992 return emitRemovedIntrinsicError(DAG, DL, VT);
10993 return SDValue();
10994 case Intrinsic::amdgcn_rcp_legacy:
10995 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
10996 return emitRemovedIntrinsicError(DAG, DL, VT);
10997 return DAG.getNode(Opcode: AMDGPUISD::RCP_LEGACY, DL, VT, Operand: Op.getOperand(i: 1));
10998 case Intrinsic::amdgcn_fma_legacy:
10999 case Intrinsic::amdgcn_sudot4:
11000 case Intrinsic::amdgcn_sudot8:
11001 case Intrinsic::amdgcn_tanh:
11002 return SDValue();
11003 case Intrinsic::amdgcn_rsq_clamp: {
11004 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11005 return DAG.getNode(Opcode: AMDGPUISD::RSQ_CLAMP, DL, VT, Operand: Op.getOperand(i: 1));
11006
11007 Type *Type = VT.getTypeForEVT(Context&: *DAG.getContext());
11008 APFloat Max = APFloat::getLargest(Sem: Type->getFltSemantics());
11009 APFloat Min = APFloat::getLargest(Sem: Type->getFltSemantics(), Negative: true);
11010
11011 SDValue Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: Op.getOperand(i: 1));
11012 SDValue Tmp =
11013 DAG.getNode(Opcode: ISD::FMINNUM, DL, VT, N1: Rsq, N2: DAG.getConstantFP(Val: Max, DL, VT));
11014 return DAG.getNode(Opcode: ISD::FMAXNUM, DL, VT, N1: Tmp,
11015 N2: DAG.getConstantFP(Val: Min, DL, VT));
11016 }
11017 case Intrinsic::r600_read_ngroups_x:
11018 if (Subtarget->isAmdHsaOS())
11019 return emitNonHSAIntrinsicError(DAG, DL, VT);
11020
11021 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11022 Offset: SI::KernelInputOffsets::NGROUPS_X, Alignment: Align(4),
11023 Signed: false);
11024 case Intrinsic::r600_read_ngroups_y:
11025 if (Subtarget->isAmdHsaOS())
11026 return emitNonHSAIntrinsicError(DAG, DL, VT);
11027
11028 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11029 Offset: SI::KernelInputOffsets::NGROUPS_Y, Alignment: Align(4),
11030 Signed: false);
11031 case Intrinsic::r600_read_ngroups_z:
11032 if (Subtarget->isAmdHsaOS())
11033 return emitNonHSAIntrinsicError(DAG, DL, VT);
11034
11035 return lowerKernargMemParameter(DAG, VT, MemVT: VT, SL: DL, Chain: DAG.getEntryNode(),
11036 Offset: SI::KernelInputOffsets::NGROUPS_Z, Alignment: Align(4),
11037 Signed: false);
11038 case Intrinsic::r600_read_local_size_x:
11039 if (Subtarget->isAmdHsaOS())
11040 return emitNonHSAIntrinsicError(DAG, DL, VT);
11041
11042 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11043 Offset: SI::KernelInputOffsets::LOCAL_SIZE_X);
11044 case Intrinsic::r600_read_local_size_y:
11045 if (Subtarget->isAmdHsaOS())
11046 return emitNonHSAIntrinsicError(DAG, DL, VT);
11047
11048 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11049 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Y);
11050 case Intrinsic::r600_read_local_size_z:
11051 if (Subtarget->isAmdHsaOS())
11052 return emitNonHSAIntrinsicError(DAG, DL, VT);
11053
11054 return lowerImplicitZextParam(DAG, Op, VT: MVT::i16,
11055 Offset: SI::KernelInputOffsets::LOCAL_SIZE_Z);
11056 case Intrinsic::amdgcn_workgroup_id_x:
11057 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11058 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
11059 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
11060 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
11061 case Intrinsic::amdgcn_workgroup_id_y:
11062 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11063 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
11064 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
11065 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
11066 case Intrinsic::amdgcn_workgroup_id_z:
11067 return lowerWorkGroupId(DAG, MFI: *MFI, VT,
11068 WorkGroupIdPV: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
11069 ClusterMaxIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
11070 ClusterWorkGroupIdPV: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
11071 case Intrinsic::amdgcn_cluster_id_x:
11072 return Subtarget->hasClusters()
11073 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11074 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_X)
11075 : DAG.getPOISON(VT);
11076 case Intrinsic::amdgcn_cluster_id_y:
11077 return Subtarget->hasClusters()
11078 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11079 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Y)
11080 : DAG.getPOISON(VT);
11081 case Intrinsic::amdgcn_cluster_id_z:
11082 return Subtarget->hasClusters()
11083 ? getPreloadedValue(DAG, MFI: *MFI, VT,
11084 PVID: AMDGPUFunctionArgInfo::WORKGROUP_ID_Z)
11085 : DAG.getPOISON(VT);
11086 case Intrinsic::amdgcn_cluster_workgroup_id_x:
11087 return Subtarget->hasClusters()
11088 ? getPreloadedValue(
11089 DAG, MFI: *MFI, VT,
11090 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X)
11091 : DAG.getPOISON(VT);
11092 case Intrinsic::amdgcn_cluster_workgroup_id_y:
11093 return Subtarget->hasClusters()
11094 ? getPreloadedValue(
11095 DAG, MFI: *MFI, VT,
11096 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y)
11097 : DAG.getPOISON(VT);
11098 case Intrinsic::amdgcn_cluster_workgroup_id_z:
11099 return Subtarget->hasClusters()
11100 ? getPreloadedValue(
11101 DAG, MFI: *MFI, VT,
11102 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z)
11103 : DAG.getPOISON(VT);
11104 case Intrinsic::amdgcn_cluster_workgroup_flat_id:
11105 return Subtarget->hasClusters()
11106 ? lowerConstHwRegRead(DAG, Op, HwReg: AMDGPU::Hwreg::ID_IB_STS2, LowBit: 21, Width: 4)
11107 : SDValue();
11108 case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
11109 return Subtarget->hasClusters()
11110 ? getPreloadedValue(
11111 DAG, MFI: *MFI, VT,
11112 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X)
11113 : DAG.getPOISON(VT);
11114 case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
11115 return Subtarget->hasClusters()
11116 ? getPreloadedValue(
11117 DAG, MFI: *MFI, VT,
11118 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y)
11119 : DAG.getPOISON(VT);
11120 case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
11121 return Subtarget->hasClusters()
11122 ? getPreloadedValue(
11123 DAG, MFI: *MFI, VT,
11124 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z)
11125 : DAG.getPOISON(VT);
11126 case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
11127 return Subtarget->hasClusters()
11128 ? getPreloadedValue(
11129 DAG, MFI: *MFI, VT,
11130 PVID: AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID)
11131 : DAG.getPOISON(VT);
11132 case Intrinsic::amdgcn_wave_id:
11133 return lowerWaveID(DAG, Op);
11134 case Intrinsic::amdgcn_lds_kernel_id: {
11135 if (MFI->isEntryFunction())
11136 return getLDSKernelId(DAG, SL: DL);
11137 return getPreloadedValue(DAG, MFI: *MFI, VT,
11138 PVID: AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
11139 }
11140 case Intrinsic::amdgcn_workitem_id_x:
11141 return lowerWorkitemID(DAG, Op, Dim: 0, Arg: MFI->getArgInfo().WorkItemIDX);
11142 case Intrinsic::amdgcn_workitem_id_y:
11143 return lowerWorkitemID(DAG, Op, Dim: 1, Arg: MFI->getArgInfo().WorkItemIDY);
11144 case Intrinsic::amdgcn_workitem_id_z:
11145 return lowerWorkitemID(DAG, Op, Dim: 2, Arg: MFI->getArgInfo().WorkItemIDZ);
11146 case Intrinsic::amdgcn_wavefrontsize:
11147 return DAG.getConstant(Val: MF.getSubtarget<GCNSubtarget>().getWavefrontSize(),
11148 DL: SDLoc(Op), VT: MVT::i32);
11149 case Intrinsic::amdgcn_s_buffer_load: {
11150 unsigned CPol = Op.getConstantOperandVal(i: 3);
11151 // s_buffer_load, because of how it's optimized, can't be volatile
11152 // so reject ones with the volatile bit set.
11153 if (CPol & ~((Subtarget->getGeneration() >= AMDGPUSubtarget::GFX12)
11154 ? AMDGPU::CPol::ALL
11155 : AMDGPU::CPol::ALL_pregfx12))
11156 return Op;
11157 return lowerSBuffer(VT, DL, Rsrc: Op.getOperand(i: 1), Offset: Op.getOperand(i: 2),
11158 CachePolicy: Op.getOperand(i: 3), DAG);
11159 }
11160 case Intrinsic::amdgcn_fdiv_fast:
11161 return lowerFDIV_FAST(Op, DAG);
11162 case Intrinsic::amdgcn_sin:
11163 return DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL, VT, Operand: Op.getOperand(i: 1));
11164
11165 case Intrinsic::amdgcn_cos:
11166 return DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL, VT, Operand: Op.getOperand(i: 1));
11167
11168 case Intrinsic::amdgcn_mul_u24:
11169 return DAG.getNode(Opcode: AMDGPUISD::MUL_U24, DL, VT, N1: Op.getOperand(i: 1),
11170 N2: Op.getOperand(i: 2));
11171 case Intrinsic::amdgcn_mul_i24:
11172 return DAG.getNode(Opcode: AMDGPUISD::MUL_I24, DL, VT, N1: Op.getOperand(i: 1),
11173 N2: Op.getOperand(i: 2));
11174
11175 case Intrinsic::amdgcn_log_clamp: {
11176 if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
11177 return SDValue();
11178
11179 return emitRemovedIntrinsicError(DAG, DL, VT);
11180 }
11181 case Intrinsic::amdgcn_fract:
11182 return DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: Op.getOperand(i: 1));
11183
11184 case Intrinsic::amdgcn_class:
11185 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT, N1: Op.getOperand(i: 1),
11186 N2: Op.getOperand(i: 2));
11187 case Intrinsic::amdgcn_div_fmas:
11188 return DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL, VT, N1: Op.getOperand(i: 1),
11189 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11190
11191 case Intrinsic::amdgcn_div_fixup:
11192 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL, VT, N1: Op.getOperand(i: 1),
11193 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11194
11195 case Intrinsic::amdgcn_div_scale: {
11196 const ConstantSDNode *Param = cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11197
11198 // Translate to the operands expected by the machine instruction. The
11199 // first parameter must be the same as the first instruction.
11200 SDValue Numerator = Op.getOperand(i: 1);
11201 SDValue Denominator = Op.getOperand(i: 2);
11202
11203 // Note this order is opposite of the machine instruction's operations,
11204 // which is s0.f = Quotient, s1.f = Denominator, s2.f = Numerator. The
11205 // intrinsic has the numerator as the first operand to match a normal
11206 // division operation.
11207
11208 SDValue Src0 = Param->isAllOnes() ? Numerator : Denominator;
11209
11210 return DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL, VTList: Op->getVTList(), N1: Src0,
11211 N2: Denominator, N3: Numerator);
11212 }
11213 case Intrinsic::amdgcn_icmp: {
11214 // There is a Pat that handles this variant, so return it as-is.
11215 if (Op.getOperand(i: 1).getValueType() == MVT::i1 &&
11216 Op.getConstantOperandVal(i: 2) == 0 &&
11217 Op.getConstantOperandVal(i: 3) == ICmpInst::Predicate::ICMP_NE)
11218 return Op;
11219 return lowerICMPIntrinsic(TLI: *this, N: Op.getNode(), DAG);
11220 }
11221 case Intrinsic::amdgcn_fcmp: {
11222 return lowerFCMPIntrinsic(TLI: *this, N: Op.getNode(), DAG);
11223 }
11224 case Intrinsic::amdgcn_ballot:
11225 return lowerBALLOTIntrinsic(TLI: *this, N: Op.getNode(), DAG);
11226 case Intrinsic::amdgcn_fmed3:
11227 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL, VT, N1: Op.getOperand(i: 1),
11228 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), Flags: Op->getFlags());
11229 case Intrinsic::amdgcn_fdot2:
11230 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL, VT, N1: Op.getOperand(i: 1),
11231 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3), N4: Op.getOperand(i: 4));
11232 case Intrinsic::amdgcn_fmul_legacy:
11233 return DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL, VT, N1: Op.getOperand(i: 1),
11234 N2: Op.getOperand(i: 2));
11235 case Intrinsic::amdgcn_sbfe:
11236 return DAG.getNode(Opcode: AMDGPUISD::BFE_I32, DL, VT, N1: Op.getOperand(i: 1),
11237 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11238 case Intrinsic::amdgcn_ubfe:
11239 return DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL, VT, N1: Op.getOperand(i: 1),
11240 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11241 case Intrinsic::amdgcn_cvt_pkrtz:
11242 case Intrinsic::amdgcn_cvt_pknorm_i16:
11243 case Intrinsic::amdgcn_cvt_pknorm_u16:
11244 case Intrinsic::amdgcn_cvt_pk_i16:
11245 case Intrinsic::amdgcn_cvt_pk_u16: {
11246 // FIXME: Stop adding cast if v2f16/v2i16 are legal.
11247 EVT VT = Op.getValueType();
11248 unsigned Opcode;
11249
11250 if (IntrinsicID == Intrinsic::amdgcn_cvt_pkrtz)
11251 Opcode = AMDGPUISD::CVT_PKRTZ_F16_F32;
11252 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_i16)
11253 Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
11254 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pknorm_u16)
11255 Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
11256 else if (IntrinsicID == Intrinsic::amdgcn_cvt_pk_i16)
11257 Opcode = AMDGPUISD::CVT_PK_I16_I32;
11258 else
11259 Opcode = AMDGPUISD::CVT_PK_U16_U32;
11260
11261 if (isTypeLegal(VT))
11262 return DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11263
11264 SDValue Node =
11265 DAG.getNode(Opcode, DL, VT: MVT::i32, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11266 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Node);
11267 }
11268 case Intrinsic::amdgcn_fmad_ftz:
11269 return DAG.getNode(Opcode: AMDGPUISD::FMAD_FTZ, DL, VT, N1: Op.getOperand(i: 1),
11270 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11271
11272 case Intrinsic::amdgcn_if_break:
11273 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_IF_BREAK, dl: DL, VT,
11274 Op1: Op->getOperand(Num: 1), Op2: Op->getOperand(Num: 2)),
11275 0);
11276
11277 case Intrinsic::amdgcn_groupstaticsize: {
11278 Triple::OSType OS = getTargetMachine().getTargetTriple().getOS();
11279 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL)
11280 return Op;
11281
11282 const Module *M = MF.getFunction().getParent();
11283 const GlobalValue *GV =
11284 Intrinsic::getDeclarationIfExists(M, id: Intrinsic::amdgcn_groupstaticsize);
11285 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: MVT::i32, offset: 0,
11286 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11287 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11288 }
11289 case Intrinsic::amdgcn_is_shared:
11290 case Intrinsic::amdgcn_is_private: {
11291 SDLoc SL(Op);
11292 SDValue SrcVec =
11293 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
11294 SDValue SrcHi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: SrcVec,
11295 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
11296
11297 unsigned AS = (IntrinsicID == Intrinsic::amdgcn_is_shared)
11298 ? AMDGPUAS::LOCAL_ADDRESS
11299 : AMDGPUAS::PRIVATE_ADDRESS;
11300 if (AS == AMDGPUAS::PRIVATE_ADDRESS &&
11301 Subtarget->hasGloballyAddressableScratch()) {
11302 SDValue FlatScratchBaseHi(
11303 DAG.getMachineNode(
11304 Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32,
11305 Op1: DAG.getRegister(Reg: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, VT: MVT::i32)),
11306 0);
11307 // Test bits 63..58 against the aperture address.
11308 return DAG.getSetCC(
11309 DL: SL, VT: MVT::i1,
11310 LHS: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: SrcHi, N2: FlatScratchBaseHi),
11311 RHS: DAG.getConstant(Val: 1u << 26, DL: SL, VT: MVT::i32), Cond: ISD::SETULT);
11312 }
11313
11314 SDValue Aperture = getSegmentAperture(AS, DL: SL, DAG);
11315 return DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: SrcHi, RHS: Aperture, Cond: ISD::SETEQ);
11316 }
11317 case Intrinsic::amdgcn_perm:
11318 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op.getOperand(i: 1),
11319 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11320 case Intrinsic::amdgcn_reloc_constant: {
11321 Module *M = MF.getFunction().getParent();
11322 const MDNode *Metadata = cast<MDNodeSDNode>(Val: Op.getOperand(i: 1))->getMD();
11323 auto SymbolName = cast<MDString>(Val: Metadata->getOperand(I: 0))->getString();
11324 auto *RelocSymbol = cast<GlobalVariable>(
11325 Val: M->getOrInsertGlobal(Name: SymbolName, Ty: Type::getInt32Ty(C&: M->getContext())));
11326 SDValue GA = DAG.getTargetGlobalAddress(GV: RelocSymbol, DL, VT: MVT::i32, offset: 0,
11327 TargetFlags: SIInstrInfo::MO_ABS32_LO);
11328 return {DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: GA), 0};
11329 }
11330 case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
11331 case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
11332 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
11333 case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
11334 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
11335 case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
11336 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
11337 case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
11338 if (Op.getOperand(i: 4).getValueType() == MVT::i32)
11339 return SDValue();
11340
11341 SDLoc SL(Op);
11342 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i32);
11343 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11344 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: Op.getOperand(i: 2),
11345 N4: Op.getOperand(i: 3), N5: IndexKeyi32);
11346 }
11347 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
11348 case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
11349 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
11350 case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
11351 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
11352 case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
11353 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
11354 case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
11355 if (Op.getOperand(i: 4).getValueType() == MVT::i64)
11356 return SDValue();
11357
11358 SDLoc SL(Op);
11359 auto IndexKeyi64 =
11360 Op.getOperand(i: 4).getValueType() == MVT::v2i32
11361 ? DAG.getBitcast(VT: MVT::i64, V: Op.getOperand(i: 4))
11362 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 4), DL: SL, VT: MVT::i64);
11363 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11364 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11365 Op.getOperand(i: 3), IndexKeyi64, Op.getOperand(i: 5),
11366 Op.getOperand(i: 6)});
11367 }
11368 case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
11369 case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
11370 case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
11371 case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
11372 case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
11373 case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8: {
11374 EVT IndexKeyTy = IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
11375 ? MVT::i64
11376 : MVT::i32;
11377 if (Op.getOperand(i: 6).getValueType() == IndexKeyTy)
11378 return SDValue();
11379
11380 SDLoc SL(Op);
11381 auto IndexKey =
11382 Op.getOperand(i: 6).getValueType().isVector()
11383 ? DAG.getBitcast(VT: IndexKeyTy, V: Op.getOperand(i: 6))
11384 : DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: IndexKeyTy);
11385 SmallVector<SDValue> Args{
11386 Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11387 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11388 IndexKey, Op.getOperand(i: 7), Op.getOperand(i: 8)};
11389 if (IntrinsicID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8)
11390 Args.push_back(Elt: Op.getOperand(i: 9));
11391 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(), Ops: Args);
11392 }
11393 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
11394 case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
11395 case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
11396 if (Op.getOperand(i: 6).getValueType() == MVT::i32)
11397 return SDValue();
11398
11399 SDLoc SL(Op);
11400 auto IndexKeyi32 = DAG.getAnyExtOrTrunc(Op: Op.getOperand(i: 6), DL: SL, VT: MVT::i32);
11401 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: Op.getValueType(),
11402 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1), Op.getOperand(i: 2),
11403 Op.getOperand(i: 3), Op.getOperand(i: 4), Op.getOperand(i: 5),
11404 IndexKeyi32, Op.getOperand(i: 7)});
11405 }
11406 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
11407 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
11408 unsigned AFmt = (unsigned)Op.getConstantOperandVal(i: 1);
11409 unsigned BFmt = (unsigned)Op.getConstantOperandVal(i: 3);
11410 unsigned AScaleFmt = (unsigned)Op.getConstantOperandVal(i: 8);
11411 unsigned BScaleFmt = (unsigned)Op.getConstantOperandVal(i: 11);
11412 if (!AMDGPU::isValidWMMAScaleFmtCombination(AFmt, AScale: AScaleFmt, BFmt,
11413 BScale: BScaleFmt)) {
11414 DAG.getMachineFunction().getFunction().getContext().emitError(
11415 ErrorStr: "invalid matrix and scale format combination in wmma call");
11416 Op->print(OS&: errs());
11417 errs() << '\n';
11418 }
11419 return SDValue();
11420 }
11421 case Intrinsic::amdgcn_addrspacecast_nonnull:
11422 return lowerADDRSPACECAST(Op, DAG);
11423 case Intrinsic::amdgcn_readlane:
11424 case Intrinsic::amdgcn_readfirstlane:
11425 case Intrinsic::amdgcn_writelane:
11426 case Intrinsic::amdgcn_permlane16:
11427 case Intrinsic::amdgcn_permlanex16:
11428 case Intrinsic::amdgcn_permlane64:
11429 case Intrinsic::amdgcn_set_inactive:
11430 case Intrinsic::amdgcn_set_inactive_chain_arg:
11431 case Intrinsic::amdgcn_mov_dpp8:
11432 case Intrinsic::amdgcn_update_dpp:
11433 case Intrinsic::amdgcn_permlane_bcast:
11434 case Intrinsic::amdgcn_permlane_up:
11435 case Intrinsic::amdgcn_permlane_down:
11436 case Intrinsic::amdgcn_permlane_xor:
11437 return lowerLaneOp(TLI: *this, N: Op.getNode(), DAG);
11438 case Intrinsic::amdgcn_dead: {
11439 SmallVector<SDValue, 8> Poisons;
11440 for (const EVT ValTy : Op.getNode()->values())
11441 Poisons.push_back(Elt: DAG.getPOISON(VT: ValTy));
11442 return DAG.getMergeValues(Ops: Poisons, dl: SDLoc(Op));
11443 }
11444 case Intrinsic::amdgcn_wave_shuffle:
11445 return lowerWaveShuffle(TLI: *this, N: Op.getNode(), DAG);
11446 default:
11447 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
11448 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
11449 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: false);
11450
11451 return Op;
11452 }
11453}
11454
11455// On targets not supporting constant in soffset field, turn zero to
11456// SGPR_NULL to avoid generating an extra s_mov with zero.
11457static SDValue selectSOffset(SDValue SOffset, SelectionDAG &DAG,
11458 const GCNSubtarget *Subtarget) {
11459 if (Subtarget->hasRestrictedSOffset() && isNullConstant(V: SOffset))
11460 return DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32);
11461 return SOffset;
11462}
11463
11464SDValue SITargetLowering::lowerRawBufferAtomicIntrin(SDValue Op,
11465 SelectionDAG &DAG,
11466 unsigned NewOpcode) const {
11467 SDLoc DL(Op);
11468
11469 SDValue VData = Op.getOperand(i: 2);
11470 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11471 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11472 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11473 SDValue Ops[] = {
11474 Op.getOperand(i: 0), // Chain
11475 VData, // vdata
11476 Rsrc, // rsrc
11477 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11478 VOffset, // voffset
11479 SOffset, // soffset
11480 Offset, // offset
11481 Op.getOperand(i: 6), // cachepolicy
11482 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11483 };
11484
11485 auto *M = cast<MemSDNode>(Val&: Op);
11486
11487 EVT MemVT = VData.getValueType();
11488 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11489 MMO: M->getMemOperand());
11490}
11491
11492SDValue
11493SITargetLowering::lowerStructBufferAtomicIntrin(SDValue Op, SelectionDAG &DAG,
11494 unsigned NewOpcode) const {
11495 SDLoc DL(Op);
11496
11497 SDValue VData = Op.getOperand(i: 2);
11498 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
11499 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
11500 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
11501 SDValue Ops[] = {
11502 Op.getOperand(i: 0), // Chain
11503 VData, // vdata
11504 Rsrc, // rsrc
11505 Op.getOperand(i: 4), // vindex
11506 VOffset, // voffset
11507 SOffset, // soffset
11508 Offset, // offset
11509 Op.getOperand(i: 7), // cachepolicy
11510 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11511 };
11512
11513 auto *M = cast<MemSDNode>(Val&: Op);
11514
11515 EVT MemVT = VData.getValueType();
11516 return DAG.getMemIntrinsicNode(Opcode: NewOpcode, dl: DL, VTList: Op->getVTList(), Ops, MemVT,
11517 MMO: M->getMemOperand());
11518}
11519
11520SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
11521 SelectionDAG &DAG) const {
11522 unsigned IntrID = Op.getConstantOperandVal(i: 1);
11523 SDLoc DL(Op);
11524
11525 switch (IntrID) {
11526 case Intrinsic::amdgcn_ds_ordered_add:
11527 case Intrinsic::amdgcn_ds_ordered_swap: {
11528 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11529 SDValue Chain = M->getOperand(Num: 0);
11530 SDValue M0 = M->getOperand(Num: 2);
11531 SDValue Value = M->getOperand(Num: 3);
11532 unsigned IndexOperand = M->getConstantOperandVal(Num: 7);
11533 unsigned WaveRelease = M->getConstantOperandVal(Num: 8);
11534 unsigned WaveDone = M->getConstantOperandVal(Num: 9);
11535
11536 unsigned OrderedCountIndex = IndexOperand & 0x3f;
11537 IndexOperand &= ~0x3f;
11538 unsigned CountDw = 0;
11539
11540 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) {
11541 CountDw = (IndexOperand >> 24) & 0xf;
11542 IndexOperand &= ~(0xf << 24);
11543
11544 if (CountDw < 1 || CountDw > 4) {
11545 const Function &Fn = DAG.getMachineFunction().getFunction();
11546 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11547 Fn, "ds_ordered_count: dword count must be between 1 and 4",
11548 DL.getDebugLoc()));
11549 CountDw = 1;
11550 }
11551 }
11552
11553 if (IndexOperand) {
11554 const Function &Fn = DAG.getMachineFunction().getFunction();
11555 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11556 Fn, "ds_ordered_count: bad index operand", DL.getDebugLoc()));
11557 }
11558
11559 if (WaveDone && !WaveRelease) {
11560 // TODO: Move this to IR verifier
11561 const Function &Fn = DAG.getMachineFunction().getFunction();
11562 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
11563 Fn, "ds_ordered_count: wave_done requires wave_release",
11564 DL.getDebugLoc()));
11565 }
11566
11567 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1;
11568 unsigned ShaderType =
11569 SIInstrInfo::getDSShaderTypeValue(MF: DAG.getMachineFunction());
11570 unsigned Offset0 = OrderedCountIndex << 2;
11571 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (Instruction << 4);
11572
11573 if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
11574 Offset1 |= (CountDw - 1) << 6;
11575
11576 if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX11)
11577 Offset1 |= ShaderType << 2;
11578
11579 unsigned Offset = Offset0 | (Offset1 << 8);
11580
11581 SDValue Ops[] = {
11582 Chain, Value, DAG.getTargetConstant(Val: Offset, DL, VT: MVT::i16),
11583 copyToM0(DAG, Chain, DL, V: M0).getValue(R: 1), // Glue
11584 };
11585 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::DS_ORDERED_COUNT, dl: DL,
11586 VTList: M->getVTList(), Ops, MemVT: M->getMemoryVT(),
11587 MMO: M->getMemOperand());
11588 }
11589 case Intrinsic::amdgcn_raw_buffer_load:
11590 case Intrinsic::amdgcn_raw_ptr_buffer_load:
11591 case Intrinsic::amdgcn_raw_atomic_buffer_load:
11592 case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
11593 case Intrinsic::amdgcn_raw_buffer_load_format:
11594 case Intrinsic::amdgcn_raw_ptr_buffer_load_format: {
11595 const bool IsFormat =
11596 IntrID == Intrinsic::amdgcn_raw_buffer_load_format ||
11597 IntrID == Intrinsic::amdgcn_raw_ptr_buffer_load_format;
11598
11599 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11600 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
11601 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
11602 SDValue Ops[] = {
11603 Op.getOperand(i: 0), // Chain
11604 Rsrc, // rsrc
11605 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11606 VOffset, // voffset
11607 SOffset, // soffset
11608 Offset, // offset
11609 Op.getOperand(i: 5), // cachepolicy, swizzled buffer
11610 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11611 };
11612
11613 auto *M = cast<MemSDNode>(Val&: Op);
11614 return lowerIntrinsicLoad(M, IsFormat, DAG, Ops);
11615 }
11616 case Intrinsic::amdgcn_struct_buffer_load:
11617 case Intrinsic::amdgcn_struct_ptr_buffer_load:
11618 case Intrinsic::amdgcn_struct_buffer_load_format:
11619 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
11620 case Intrinsic::amdgcn_struct_atomic_buffer_load:
11621 case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
11622 const bool IsFormat =
11623 IntrID == Intrinsic::amdgcn_struct_buffer_load_format ||
11624 IntrID == Intrinsic::amdgcn_struct_ptr_buffer_load_format;
11625
11626 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11627 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11628 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11629 SDValue Ops[] = {
11630 Op.getOperand(i: 0), // Chain
11631 Rsrc, // rsrc
11632 Op.getOperand(i: 3), // vindex
11633 VOffset, // voffset
11634 SOffset, // soffset
11635 Offset, // offset
11636 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
11637 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11638 };
11639
11640 return lowerIntrinsicLoad(M: cast<MemSDNode>(Val&: Op), IsFormat, DAG, Ops);
11641 }
11642 case Intrinsic::amdgcn_raw_tbuffer_load:
11643 case Intrinsic::amdgcn_raw_ptr_tbuffer_load: {
11644 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11645 EVT LoadVT = Op.getValueType();
11646 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11647 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 3), DAG);
11648 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 4), DAG, Subtarget);
11649
11650 SDValue Ops[] = {
11651 Op.getOperand(i: 0), // Chain
11652 Rsrc, // rsrc
11653 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11654 VOffset, // voffset
11655 SOffset, // soffset
11656 Offset, // offset
11657 Op.getOperand(i: 5), // format
11658 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
11659 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11660 };
11661
11662 if (LoadVT.getScalarType() == MVT::f16)
11663 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
11664 Ops);
11665 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
11666 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
11667 DAG);
11668 }
11669 case Intrinsic::amdgcn_struct_tbuffer_load:
11670 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: {
11671 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11672 EVT LoadVT = Op.getValueType();
11673 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
11674 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
11675 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
11676
11677 SDValue Ops[] = {
11678 Op.getOperand(i: 0), // Chain
11679 Rsrc, // rsrc
11680 Op.getOperand(i: 3), // vindex
11681 VOffset, // voffset
11682 SOffset, // soffset
11683 Offset, // offset
11684 Op.getOperand(i: 6), // format
11685 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
11686 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11687 };
11688
11689 if (LoadVT.getScalarType() == MVT::f16)
11690 return adjustLoadValueType(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT_D16, M, DAG,
11691 Ops);
11692 return getMemIntrinsicNode(Opcode: AMDGPUISD::TBUFFER_LOAD_FORMAT, DL,
11693 VTList: Op->getVTList(), Ops, MemVT: LoadVT, MMO: M->getMemOperand(),
11694 DAG);
11695 }
11696 case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
11697 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
11698 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
11699 case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
11700 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
11701 return lowerStructBufferAtomicIntrin(Op, DAG,
11702 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FADD);
11703 case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
11704 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
11705 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
11706 case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
11707 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
11708 return lowerStructBufferAtomicIntrin(Op, DAG,
11709 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMIN);
11710 case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
11711 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
11712 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
11713 case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
11714 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
11715 return lowerStructBufferAtomicIntrin(Op, DAG,
11716 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_FMAX);
11717 case Intrinsic::amdgcn_raw_buffer_atomic_swap:
11718 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
11719 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
11720 case Intrinsic::amdgcn_raw_buffer_atomic_add:
11721 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
11722 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
11723 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
11724 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
11725 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
11726 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
11727 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
11728 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
11729 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
11730 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
11731 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
11732 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
11733 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
11734 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
11735 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
11736 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
11737 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
11738 case Intrinsic::amdgcn_raw_buffer_atomic_and:
11739 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
11740 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
11741 case Intrinsic::amdgcn_raw_buffer_atomic_or:
11742 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
11743 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
11744 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
11745 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
11746 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
11747 case Intrinsic::amdgcn_raw_buffer_atomic_inc:
11748 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
11749 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
11750 case Intrinsic::amdgcn_raw_buffer_atomic_dec:
11751 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
11752 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
11753 case Intrinsic::amdgcn_struct_buffer_atomic_swap:
11754 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
11755 return lowerStructBufferAtomicIntrin(Op, DAG,
11756 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SWAP);
11757 case Intrinsic::amdgcn_struct_buffer_atomic_add:
11758 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
11759 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_ADD);
11760 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
11761 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
11762 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SUB);
11763 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
11764 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
11765 return lowerStructBufferAtomicIntrin(Op, DAG,
11766 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMIN);
11767 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
11768 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
11769 return lowerStructBufferAtomicIntrin(Op, DAG,
11770 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMIN);
11771 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
11772 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
11773 return lowerStructBufferAtomicIntrin(Op, DAG,
11774 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_SMAX);
11775 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
11776 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
11777 return lowerStructBufferAtomicIntrin(Op, DAG,
11778 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_UMAX);
11779 case Intrinsic::amdgcn_struct_buffer_atomic_and:
11780 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
11781 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_AND);
11782 case Intrinsic::amdgcn_struct_buffer_atomic_or:
11783 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
11784 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_OR);
11785 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
11786 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
11787 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_XOR);
11788 case Intrinsic::amdgcn_struct_buffer_atomic_inc:
11789 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
11790 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_INC);
11791 case Intrinsic::amdgcn_struct_buffer_atomic_dec:
11792 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
11793 return lowerStructBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_DEC);
11794 case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
11795 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
11796 return lowerRawBufferAtomicIntrin(Op, DAG, NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
11797 case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
11798 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
11799 return lowerStructBufferAtomicIntrin(Op, DAG,
11800 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_CSUB);
11801 case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
11802 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
11803 return lowerRawBufferAtomicIntrin(Op, DAG,
11804 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
11805 case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
11806 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
11807 return lowerStructBufferAtomicIntrin(Op, DAG,
11808 NewOpcode: AMDGPUISD::BUFFER_ATOMIC_COND_SUB_U32);
11809 case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
11810 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap: {
11811 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 4), DAG);
11812 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
11813 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
11814 SDValue Ops[] = {
11815 Op.getOperand(i: 0), // Chain
11816 Op.getOperand(i: 2), // src
11817 Op.getOperand(i: 3), // cmp
11818 Rsrc, // rsrc
11819 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
11820 VOffset, // voffset
11821 SOffset, // soffset
11822 Offset, // offset
11823 Op.getOperand(i: 7), // cachepolicy
11824 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
11825 };
11826 EVT VT = Op.getValueType();
11827 auto *M = cast<MemSDNode>(Val&: Op);
11828
11829 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
11830 VTList: Op->getVTList(), Ops, MemVT: VT,
11831 MMO: M->getMemOperand());
11832 }
11833 case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
11834 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap: {
11835 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op->getOperand(Num: 4), DAG);
11836 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 6), DAG);
11837 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 7), DAG, Subtarget);
11838 SDValue Ops[] = {
11839 Op.getOperand(i: 0), // Chain
11840 Op.getOperand(i: 2), // src
11841 Op.getOperand(i: 3), // cmp
11842 Rsrc, // rsrc
11843 Op.getOperand(i: 5), // vindex
11844 VOffset, // voffset
11845 SOffset, // soffset
11846 Offset, // offset
11847 Op.getOperand(i: 8), // cachepolicy
11848 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
11849 };
11850 EVT VT = Op.getValueType();
11851 auto *M = cast<MemSDNode>(Val&: Op);
11852
11853 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::BUFFER_ATOMIC_CMPSWAP, dl: DL,
11854 VTList: Op->getVTList(), Ops, MemVT: VT,
11855 MMO: M->getMemOperand());
11856 }
11857 case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
11858 case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
11859 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11860 SDValue NodePtr = M->getOperand(Num: 2);
11861 SDValue RayExtent = M->getOperand(Num: 3);
11862 SDValue InstanceMask = M->getOperand(Num: 4);
11863 SDValue RayOrigin = M->getOperand(Num: 5);
11864 SDValue RayDir = M->getOperand(Num: 6);
11865 SDValue Offsets = M->getOperand(Num: 7);
11866 SDValue TDescr = M->getOperand(Num: 8);
11867
11868 assert(NodePtr.getValueType() == MVT::i64);
11869 assert(RayDir.getValueType() == MVT::v3f32);
11870
11871 bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray;
11872 const unsigned NumVDataDwords = 10;
11873 const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
11874 int Opcode = AMDGPU::getMIMGOpcode(
11875 BaseOpcode: IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
11876 : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
11877 MIMGEncoding: AMDGPU::MIMGEncGfx12, VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
11878 assert(Opcode != -1);
11879
11880 SmallVector<SDValue, 7> Ops;
11881 Ops.push_back(Elt: NodePtr);
11882 Ops.push_back(Elt: DAG.getBuildVector(
11883 VT: MVT::v2i32, DL,
11884 Ops: {DAG.getBitcast(VT: MVT::i32, V: RayExtent),
11885 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: InstanceMask)}));
11886 Ops.push_back(Elt: RayOrigin);
11887 Ops.push_back(Elt: RayDir);
11888 Ops.push_back(Elt: Offsets);
11889 Ops.push_back(Elt: TDescr);
11890 Ops.push_back(Elt: M->getChain());
11891
11892 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
11893 MachineMemOperand *MemRef = M->getMemOperand();
11894 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
11895 return SDValue(NewNode, 0);
11896 }
11897 case Intrinsic::amdgcn_image_bvh_intersect_ray: {
11898 MemSDNode *M = cast<MemSDNode>(Val&: Op);
11899 SDValue NodePtr = M->getOperand(Num: 2);
11900 SDValue RayExtent = M->getOperand(Num: 3);
11901 SDValue RayOrigin = M->getOperand(Num: 4);
11902 SDValue RayDir = M->getOperand(Num: 5);
11903 SDValue RayInvDir = M->getOperand(Num: 6);
11904 SDValue TDescr = M->getOperand(Num: 7);
11905
11906 assert(NodePtr.getValueType() == MVT::i32 ||
11907 NodePtr.getValueType() == MVT::i64);
11908 assert(RayDir.getValueType() == MVT::v3f16 ||
11909 RayDir.getValueType() == MVT::v3f32);
11910
11911 const bool IsGFX11 = AMDGPU::isGFX11(STI: *Subtarget);
11912 const bool IsGFX11Plus = AMDGPU::isGFX11Plus(STI: *Subtarget);
11913 const bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
11914 const bool IsA16 = RayDir.getValueType().getVectorElementType() == MVT::f16;
11915 const bool Is64 = NodePtr.getValueType() == MVT::i64;
11916 const unsigned NumVDataDwords = 4;
11917 const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
11918 const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
11919 const bool UseNSA = (Subtarget->hasNSAEncoding() &&
11920 NumVAddrs <= Subtarget->getNSAMaxSize()) ||
11921 IsGFX12Plus;
11922 const unsigned BaseOpcodes[2][2] = {
11923 {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
11924 {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
11925 AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
11926 int Opcode;
11927 if (UseNSA) {
11928 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
11929 MIMGEncoding: IsGFX12Plus ? AMDGPU::MIMGEncGfx12
11930 : IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
11931 : AMDGPU::MIMGEncGfx10NSA,
11932 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
11933 } else {
11934 assert(!IsGFX12Plus);
11935 Opcode = AMDGPU::getMIMGOpcode(BaseOpcode: BaseOpcodes[Is64][IsA16],
11936 MIMGEncoding: IsGFX11 ? AMDGPU::MIMGEncGfx11Default
11937 : AMDGPU::MIMGEncGfx10Default,
11938 VDataDwords: NumVDataDwords, VAddrDwords: NumVAddrDwords);
11939 }
11940 assert(Opcode != -1);
11941
11942 SmallVector<SDValue, 16> Ops;
11943
11944 auto packLanes = [&DAG, &Ops, &DL](SDValue Op, bool IsAligned) {
11945 SmallVector<SDValue, 3> Lanes;
11946 DAG.ExtractVectorElements(Op, Args&: Lanes, Start: 0, Count: 3);
11947 if (Lanes[0].getValueSizeInBits() == 32) {
11948 for (unsigned I = 0; I < 3; ++I)
11949 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: Lanes[I]));
11950 } else {
11951 if (IsAligned) {
11952 Ops.push_back(Elt: DAG.getBitcast(
11953 VT: MVT::i32,
11954 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[0], Lanes[1]})));
11955 Ops.push_back(Elt: Lanes[2]);
11956 } else {
11957 SDValue Elt0 = Ops.pop_back_val();
11958 Ops.push_back(Elt: DAG.getBitcast(
11959 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Elt0, Lanes[0]})));
11960 Ops.push_back(Elt: DAG.getBitcast(
11961 VT: MVT::i32,
11962 V: DAG.getBuildVector(VT: MVT::v2f16, DL, Ops: {Lanes[1], Lanes[2]})));
11963 }
11964 }
11965 };
11966
11967 if (UseNSA && IsGFX11Plus) {
11968 Ops.push_back(Elt: NodePtr);
11969 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
11970 Ops.push_back(Elt: RayOrigin);
11971 if (IsA16) {
11972 SmallVector<SDValue, 3> DirLanes, InvDirLanes, MergedLanes;
11973 DAG.ExtractVectorElements(Op: RayDir, Args&: DirLanes, Start: 0, Count: 3);
11974 DAG.ExtractVectorElements(Op: RayInvDir, Args&: InvDirLanes, Start: 0, Count: 3);
11975 for (unsigned I = 0; I < 3; ++I) {
11976 MergedLanes.push_back(Elt: DAG.getBitcast(
11977 VT: MVT::i32, V: DAG.getBuildVector(VT: MVT::v2f16, DL,
11978 Ops: {DirLanes[I], InvDirLanes[I]})));
11979 }
11980 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v3i32, DL, Ops: MergedLanes));
11981 } else {
11982 Ops.push_back(Elt: RayDir);
11983 Ops.push_back(Elt: RayInvDir);
11984 }
11985 } else {
11986 if (Is64)
11987 DAG.ExtractVectorElements(Op: DAG.getBitcast(VT: MVT::v2i32, V: NodePtr), Args&: Ops, Start: 0,
11988 Count: 2);
11989 else
11990 Ops.push_back(Elt: NodePtr);
11991
11992 Ops.push_back(Elt: DAG.getBitcast(VT: MVT::i32, V: RayExtent));
11993 packLanes(RayOrigin, true);
11994 packLanes(RayDir, true);
11995 packLanes(RayInvDir, false);
11996 }
11997
11998 if (!UseNSA) {
11999 // Build a single vector containing all the operands so far prepared.
12000 if (NumVAddrDwords > 12) {
12001 SDValue Undef = DAG.getPOISON(VT: MVT::i32);
12002 Ops.append(NumInputs: 16 - Ops.size(), Elt: Undef);
12003 }
12004 assert(Ops.size() >= 8 && Ops.size() <= 12);
12005 SDValue MergedOps =
12006 DAG.getBuildVector(VT: MVT::getVectorVT(VT: MVT::i32, NumElements: Ops.size()), DL, Ops);
12007 Ops.clear();
12008 Ops.push_back(Elt: MergedOps);
12009 }
12010
12011 Ops.push_back(Elt: TDescr);
12012 Ops.push_back(Elt: DAG.getTargetConstant(Val: IsA16, DL, VT: MVT::i1));
12013 Ops.push_back(Elt: M->getChain());
12014
12015 auto *NewNode = DAG.getMachineNode(Opcode, dl: DL, VTs: M->getVTList(), Ops);
12016 MachineMemOperand *MemRef = M->getMemOperand();
12017 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: {MemRef});
12018 return SDValue(NewNode, 0);
12019 }
12020 case Intrinsic::amdgcn_global_atomic_fmin_num:
12021 case Intrinsic::amdgcn_global_atomic_fmax_num:
12022 case Intrinsic::amdgcn_flat_atomic_fmin_num:
12023 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12024 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12025 SDValue Ops[] = {
12026 M->getOperand(Num: 0), // Chain
12027 M->getOperand(Num: 2), // Ptr
12028 M->getOperand(Num: 3) // Value
12029 };
12030 unsigned Opcode = 0;
12031 switch (IntrID) {
12032 case Intrinsic::amdgcn_global_atomic_fmin_num:
12033 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
12034 Opcode = ISD::ATOMIC_LOAD_FMIN;
12035 break;
12036 }
12037 case Intrinsic::amdgcn_global_atomic_fmax_num:
12038 case Intrinsic::amdgcn_flat_atomic_fmax_num: {
12039 Opcode = ISD::ATOMIC_LOAD_FMAX;
12040 break;
12041 }
12042 default:
12043 llvm_unreachable("unhandled atomic opcode");
12044 }
12045 return DAG.getAtomic(Opcode, dl: SDLoc(Op), MemVT: M->getMemoryVT(), VTList: M->getVTList(),
12046 Ops, MMO: M->getMemOperand());
12047 }
12048 case Intrinsic::amdgcn_s_alloc_vgpr: {
12049 SDValue NumVGPRs = Op.getOperand(i: 2);
12050 if (!NumVGPRs->isDivergent())
12051 return Op;
12052
12053 SDValue ReadFirstLaneID =
12054 DAG.getTargetConstant(Val: Intrinsic::amdgcn_readfirstlane, DL, VT: MVT::i32);
12055 NumVGPRs = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
12056 N1: ReadFirstLaneID, N2: NumVGPRs);
12057
12058 return DAG.getNode(Opcode: ISD::INTRINSIC_W_CHAIN, DL, VTList: Op->getVTList(),
12059 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: NumVGPRs);
12060 }
12061 case Intrinsic::amdgcn_s_get_barrier_state:
12062 case Intrinsic::amdgcn_s_get_named_barrier_state: {
12063 SDValue Chain = Op->getOperand(Num: 0);
12064 SmallVector<SDValue, 2> Ops;
12065 unsigned Opc;
12066
12067 if (isa<ConstantSDNode>(Val: Op->getOperand(Num: 2))) {
12068 uint64_t BarID = cast<ConstantSDNode>(Val: Op->getOperand(Num: 2))->getZExtValue();
12069 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state)
12070 BarID = (BarID >> 4) & 0x3F;
12071 Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
12072 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
12073 Ops.push_back(Elt: K);
12074 Ops.push_back(Elt: Chain);
12075 } else {
12076 Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
12077 if (IntrID == Intrinsic::amdgcn_s_get_named_barrier_state) {
12078 SDValue M0Val;
12079 M0Val = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: Op->getOperand(Num: 2),
12080 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
12081 M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M0Val,
12082 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12083 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12084 } else
12085 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: Op->getOperand(Num: 2)).getValue(R: 0));
12086 }
12087
12088 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12089 return SDValue(NewMI, 0);
12090 }
12091 case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
12092 case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
12093 case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
12094 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12095 SDValue Chain = Op->getOperand(Num: 0);
12096 SDValue Ptr = Op->getOperand(Num: 2);
12097 EVT VT = Op->getValueType(ResNo: 0);
12098 return DAG.getAtomicLoad(ExtType: ISD::NON_EXTLOAD, dl: DL, MemVT: MII->getMemoryVT(), VT,
12099 Chain, Ptr, MMO: MII->getMemOperand());
12100 }
12101 case Intrinsic::amdgcn_av_load_b128: {
12102 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12103 SDValue Chain = Op->getOperand(Num: 0);
12104 SDValue Ptr = Op->getOperand(Num: 2);
12105 EVT VT = Op->getValueType(ResNo: 0);
12106 // Lower to a regular ISD::LOAD. The MachineMemOperand carries Monotonic
12107 // ordering and syncscope so that SIMemoryLegalizer sets cache policy bits.
12108 // Address space filtering in the load_global/load_flat PatFrags selects
12109 // the correct GLOBAL vs FLAT instruction.
12110 return DAG.getLoad(VT, dl: DL, Chain, Ptr, MMO: MII->getMemOperand());
12111 }
12112 case Intrinsic::amdgcn_flat_load_monitor_b32:
12113 case Intrinsic::amdgcn_flat_load_monitor_b64:
12114 case Intrinsic::amdgcn_flat_load_monitor_b128: {
12115 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12116 SDValue Chain = Op->getOperand(Num: 0);
12117 SDValue Ptr = Op->getOperand(Num: 2);
12118 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::FLAT_LOAD_MONITOR, dl: DL,
12119 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12120 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12121 }
12122 case Intrinsic::amdgcn_global_load_monitor_b32:
12123 case Intrinsic::amdgcn_global_load_monitor_b64:
12124 case Intrinsic::amdgcn_global_load_monitor_b128: {
12125 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12126 SDValue Chain = Op->getOperand(Num: 0);
12127 SDValue Ptr = Op->getOperand(Num: 2);
12128 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::GLOBAL_LOAD_MONITOR, dl: DL,
12129 VTList: Op->getVTList(), Ops: {Chain, Ptr},
12130 MemVT: MII->getMemoryVT(), MMO: MII->getMemOperand());
12131 }
12132 default:
12133
12134 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12135 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrID))
12136 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
12137
12138 return SDValue();
12139 }
12140}
12141
12142// Call DAG.getMemIntrinsicNode for a load, but first widen a dwordx3 type to
12143// dwordx4 if on SI and handle TFE loads.
12144SDValue SITargetLowering::getMemIntrinsicNode(unsigned Opcode, const SDLoc &DL,
12145 SDVTList VTList,
12146 ArrayRef<SDValue> Ops, EVT MemVT,
12147 MachineMemOperand *MMO,
12148 SelectionDAG &DAG) const {
12149 LLVMContext &C = *DAG.getContext();
12150 MachineFunction &MF = DAG.getMachineFunction();
12151 EVT VT = VTList.VTs[0];
12152
12153 assert(VTList.NumVTs == 2 || VTList.NumVTs == 3);
12154 bool IsTFE = VTList.NumVTs == 3;
12155 if (IsTFE) {
12156 unsigned NumValueDWords = divideCeil(Numerator: VT.getSizeInBits(), Denominator: 32);
12157 unsigned NumOpDWords = NumValueDWords + 1;
12158 EVT OpDWordsVT = EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumOpDWords);
12159 SDVTList OpDWordsVTList = DAG.getVTList(VT1: OpDWordsVT, VT2: VTList.VTs[2]);
12160 MachineMemOperand *OpDWordsMMO =
12161 MF.getMachineMemOperand(MMO, Offset: 0, Size: NumOpDWords * 4);
12162 SDValue Op = getMemIntrinsicNode(Opcode, DL, VTList: OpDWordsVTList, Ops,
12163 MemVT: OpDWordsVT, MMO: OpDWordsMMO, DAG);
12164 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
12165 N2: DAG.getVectorIdxConstant(Val: NumValueDWords, DL));
12166 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL);
12167 SDValue ValueDWords =
12168 NumValueDWords == 1
12169 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op, N2: ZeroIdx)
12170 : DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL,
12171 VT: EVT::getVectorVT(Context&: C, VT: MVT::i32, NumElements: NumValueDWords), N1: Op,
12172 N2: ZeroIdx);
12173 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: ValueDWords);
12174 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
12175 }
12176
12177 if (!Subtarget->hasDwordx3LoadStores() &&
12178 (VT == MVT::v3i32 || VT == MVT::v3f32)) {
12179 EVT WidenedVT = EVT::getVectorVT(Context&: C, VT: VT.getVectorElementType(), NumElements: 4);
12180 EVT WidenedMemVT = EVT::getVectorVT(Context&: C, VT: MemVT.getVectorElementType(), NumElements: 4);
12181 MachineMemOperand *WidenedMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 16);
12182 SDVTList WidenedVTList = DAG.getVTList(VT1: WidenedVT, VT2: VTList.VTs[1]);
12183 SDValue Op = DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList: WidenedVTList, Ops,
12184 MemVT: WidenedMemVT, MMO: WidenedMMO);
12185 SDValue Value = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Op,
12186 N2: DAG.getVectorIdxConstant(Val: 0, DL));
12187 return DAG.getMergeValues(Ops: {Value, SDValue(Op.getNode(), 1)}, dl: DL);
12188 }
12189
12190 return DAG.getMemIntrinsicNode(Opcode, dl: DL, VTList, Ops, MemVT, MMO);
12191}
12192
12193SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG,
12194 bool ImageStore) const {
12195 EVT StoreVT = VData.getValueType();
12196
12197 // No change for f16 and legal vector D16 types.
12198 if (!StoreVT.isVector())
12199 return VData;
12200
12201 SDLoc DL(VData);
12202 unsigned NumElements = StoreVT.getVectorNumElements();
12203
12204 if (Subtarget->hasUnpackedD16VMem()) {
12205 // We need to unpack the packed data to store.
12206 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12207 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12208
12209 EVT EquivStoreVT =
12210 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements);
12211 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: EquivStoreVT, Operand: IntVData);
12212 return DAG.UnrollVectorOp(N: ZExt.getNode());
12213 }
12214
12215 // The sq block of gfx8.1 does not estimate register use correctly for d16
12216 // image store instructions. The data operand is computed as if it were not a
12217 // d16 image instruction.
12218 if (ImageStore && Subtarget->hasImageStoreD16Bug()) {
12219 // Bitcast to i16
12220 EVT IntStoreVT = StoreVT.changeTypeToInteger();
12221 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12222
12223 // Decompose into scalars
12224 SmallVector<SDValue, 4> Elts;
12225 DAG.ExtractVectorElements(Op: IntVData, Args&: Elts);
12226
12227 // Group pairs of i16 into v2i16 and bitcast to i32
12228 SmallVector<SDValue, 4> PackedElts;
12229 for (unsigned I = 0; I < Elts.size() / 2; I += 1) {
12230 SDValue Pair =
12231 DAG.getBuildVector(VT: MVT::v2i16, DL, Ops: {Elts[I * 2], Elts[I * 2 + 1]});
12232 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12233 PackedElts.push_back(Elt: IntPair);
12234 }
12235 if ((NumElements % 2) == 1) {
12236 // Handle v3i16
12237 unsigned I = Elts.size() / 2;
12238 SDValue Pair = DAG.getBuildVector(VT: MVT::v2i16, DL,
12239 Ops: {Elts[I * 2], DAG.getPOISON(VT: MVT::i16)});
12240 SDValue IntPair = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Pair);
12241 PackedElts.push_back(Elt: IntPair);
12242 }
12243
12244 // Pad using UNDEF
12245 PackedElts.resize(N: Elts.size(), NV: DAG.getPOISON(VT: MVT::i32));
12246
12247 // Build final vector
12248 EVT VecVT =
12249 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: PackedElts.size());
12250 return DAG.getBuildVector(VT: VecVT, DL, Ops: PackedElts);
12251 }
12252
12253 if (NumElements == 3) {
12254 EVT IntStoreVT =
12255 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreVT.getStoreSizeInBits());
12256 SDValue IntVData = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntStoreVT, Operand: VData);
12257
12258 EVT WidenedStoreVT = EVT::getVectorVT(
12259 Context&: *DAG.getContext(), VT: StoreVT.getVectorElementType(), NumElements: NumElements + 1);
12260 EVT WidenedIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12261 BitWidth: WidenedStoreVT.getStoreSizeInBits());
12262 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenedIntVT, Operand: IntVData);
12263 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WidenedStoreVT, Operand: ZExt);
12264 }
12265
12266 assert(isTypeLegal(StoreVT));
12267 return VData;
12268}
12269
12270static bool isAsyncLDSDMA(Intrinsic::ID Intr) {
12271 switch (Intr) {
12272 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
12273 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
12274 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
12275 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds:
12276 case Intrinsic::amdgcn_load_async_to_lds:
12277 case Intrinsic::amdgcn_global_load_async_lds:
12278 return true;
12279 }
12280 return false;
12281}
12282
12283SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12284 SelectionDAG &DAG) const {
12285 SDLoc DL(Op);
12286 SDValue Chain = Op.getOperand(i: 0);
12287 unsigned IntrinsicID = Op.getConstantOperandVal(i: 1);
12288
12289 switch (IntrinsicID) {
12290 case Intrinsic::amdgcn_exp_compr: {
12291 SDValue Src0 = Op.getOperand(i: 4);
12292 SDValue Src1 = Op.getOperand(i: 5);
12293 // Hack around illegal type on SI by directly selecting it.
12294 if (isTypeLegal(VT: Src0.getValueType()))
12295 return SDValue();
12296
12297 const ConstantSDNode *Done = cast<ConstantSDNode>(Val: Op.getOperand(i: 6));
12298 SDValue Undef = DAG.getPOISON(VT: MVT::f32);
12299 const SDValue Ops[] = {
12300 Op.getOperand(i: 2), // tgt
12301 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src0), // src0
12302 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: Src1), // src1
12303 Undef, // src2
12304 Undef, // src3
12305 Op.getOperand(i: 7), // vm
12306 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // compr
12307 Op.getOperand(i: 3), // en
12308 Op.getOperand(i: 0) // Chain
12309 };
12310
12311 unsigned Opc = Done->isZero() ? AMDGPU::EXP : AMDGPU::EXP_DONE;
12312 return SDValue(DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops), 0);
12313 }
12314
12315 case Intrinsic::amdgcn_struct_tbuffer_store:
12316 case Intrinsic::amdgcn_struct_ptr_tbuffer_store: {
12317 SDValue VData = Op.getOperand(i: 2);
12318 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
12319 if (IsD16)
12320 VData = handleD16VData(VData, DAG);
12321 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12322 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12323 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12324 SDValue Ops[] = {
12325 Chain,
12326 VData, // vdata
12327 Rsrc, // rsrc
12328 Op.getOperand(i: 4), // vindex
12329 VOffset, // voffset
12330 SOffset, // soffset
12331 Offset, // offset
12332 Op.getOperand(i: 7), // format
12333 Op.getOperand(i: 8), // cachepolicy, swizzled buffer
12334 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12335 };
12336 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12337 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12338 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12339 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12340 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12341 }
12342
12343 case Intrinsic::amdgcn_raw_tbuffer_store:
12344 case Intrinsic::amdgcn_raw_ptr_tbuffer_store: {
12345 SDValue VData = Op.getOperand(i: 2);
12346 bool IsD16 = (VData.getValueType().getScalarType() == MVT::f16);
12347 if (IsD16)
12348 VData = handleD16VData(VData, DAG);
12349 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12350 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12351 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12352 SDValue Ops[] = {
12353 Chain,
12354 VData, // vdata
12355 Rsrc, // rsrc
12356 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12357 VOffset, // voffset
12358 SOffset, // soffset
12359 Offset, // offset
12360 Op.getOperand(i: 6), // format
12361 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12362 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12363 };
12364 unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16
12365 : AMDGPUISD::TBUFFER_STORE_FORMAT;
12366 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12367 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12368 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12369 }
12370
12371 case Intrinsic::amdgcn_raw_buffer_store:
12372 case Intrinsic::amdgcn_raw_ptr_buffer_store:
12373 case Intrinsic::amdgcn_raw_buffer_store_format:
12374 case Intrinsic::amdgcn_raw_ptr_buffer_store_format: {
12375 const bool IsFormat =
12376 IntrinsicID == Intrinsic::amdgcn_raw_buffer_store_format ||
12377 IntrinsicID == Intrinsic::amdgcn_raw_ptr_buffer_store_format;
12378
12379 SDValue VData = Op.getOperand(i: 2);
12380 EVT VDataVT = VData.getValueType();
12381 EVT EltType = VDataVT.getScalarType();
12382 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12383
12384 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12385 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12386 DAG.getMachineFunction().getFunction(),
12387 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12388 return Chain;
12389 }
12390
12391 if (IsD16) {
12392 VData = handleD16VData(VData, DAG);
12393 VDataVT = VData.getValueType();
12394 }
12395
12396 if (!isTypeLegal(VT: VDataVT)) {
12397 VData =
12398 DAG.getNode(Opcode: ISD::BITCAST, DL,
12399 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12400 }
12401
12402 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12403 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 4), DAG);
12404 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 5), DAG, Subtarget);
12405 SDValue Ops[] = {
12406 Chain,
12407 VData,
12408 Rsrc,
12409 DAG.getConstant(Val: 0, DL, VT: MVT::i32), // vindex
12410 VOffset, // voffset
12411 SOffset, // soffset
12412 Offset, // offset
12413 Op.getOperand(i: 6), // cachepolicy, swizzled buffer
12414 DAG.getTargetConstant(Val: 0, DL, VT: MVT::i1), // idxen
12415 };
12416 unsigned Opc =
12417 IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
12418 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12419 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12420
12421 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12422 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12423 return handleByteShortBufferStores(DAG, VDataType: VDataVT, DL, Ops, M);
12424
12425 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12426 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12427 }
12428
12429 case Intrinsic::amdgcn_struct_buffer_store:
12430 case Intrinsic::amdgcn_struct_ptr_buffer_store:
12431 case Intrinsic::amdgcn_struct_buffer_store_format:
12432 case Intrinsic::amdgcn_struct_ptr_buffer_store_format: {
12433 const bool IsFormat =
12434 IntrinsicID == Intrinsic::amdgcn_struct_buffer_store_format ||
12435 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_store_format;
12436
12437 SDValue VData = Op.getOperand(i: 2);
12438 EVT VDataVT = VData.getValueType();
12439 EVT EltType = VDataVT.getScalarType();
12440 bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
12441
12442 if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
12443 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
12444 DAG.getMachineFunction().getFunction(),
12445 "unsupported sub-dword format buffer store", DL.getDebugLoc()));
12446 return Chain;
12447 }
12448
12449 if (IsD16) {
12450 VData = handleD16VData(VData, DAG);
12451 VDataVT = VData.getValueType();
12452 }
12453
12454 if (!isTypeLegal(VT: VDataVT)) {
12455 VData =
12456 DAG.getNode(Opcode: ISD::BITCAST, DL,
12457 VT: getEquivalentMemType(Context&: *DAG.getContext(), VT: VDataVT), Operand: VData);
12458 }
12459
12460 auto Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 3), DAG);
12461 auto [VOffset, Offset] = splitBufferOffsets(Offset: Op.getOperand(i: 5), DAG);
12462 auto SOffset = selectSOffset(SOffset: Op.getOperand(i: 6), DAG, Subtarget);
12463 SDValue Ops[] = {
12464 Chain,
12465 VData,
12466 Rsrc,
12467 Op.getOperand(i: 4), // vindex
12468 VOffset, // voffset
12469 SOffset, // soffset
12470 Offset, // offset
12471 Op.getOperand(i: 7), // cachepolicy, swizzled buffer
12472 DAG.getTargetConstant(Val: 1, DL, VT: MVT::i1), // idxen
12473 };
12474 unsigned Opc =
12475 !IsFormat ? AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT;
12476 Opc = IsD16 ? AMDGPUISD::BUFFER_STORE_FORMAT_D16 : Opc;
12477 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12478
12479 // Handle BUFFER_STORE_BYTE/SHORT overloaded intrinsics
12480 EVT VDataType = VData.getValueType().getScalarType();
12481 if (!IsD16 && !VDataVT.isVector() && EltType.getSizeInBits() < 32)
12482 return handleByteShortBufferStores(DAG, VDataType, DL, Ops, M);
12483
12484 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: Op->getVTList(), Ops,
12485 MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
12486 }
12487 case Intrinsic::amdgcn_raw_buffer_load_lds:
12488 case Intrinsic::amdgcn_raw_buffer_load_async_lds:
12489 case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
12490 case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
12491 case Intrinsic::amdgcn_struct_buffer_load_lds:
12492 case Intrinsic::amdgcn_struct_buffer_load_async_lds:
12493 case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
12494 case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
12495 if (!Subtarget->hasVMemToLDSLoad())
12496 return SDValue();
12497 unsigned Opc;
12498 bool HasVIndex =
12499 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds ||
12500 IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_async_lds ||
12501 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_lds ||
12502 IntrinsicID == Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds;
12503 unsigned OpOffset = HasVIndex ? 1 : 0;
12504 SDValue VOffset = Op.getOperand(i: 5 + OpOffset);
12505 bool HasVOffset = !isNullConstant(V: VOffset);
12506 unsigned Size = Op->getConstantOperandVal(Num: 4);
12507
12508 switch (Size) {
12509 default:
12510 return SDValue();
12511 case 1:
12512 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_BOTHEN
12513 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_IDXEN
12514 : HasVOffset ? AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFEN
12515 : AMDGPU::BUFFER_LOAD_UBYTE_LDS_OFFSET;
12516 break;
12517 case 2:
12518 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_BOTHEN
12519 : AMDGPU::BUFFER_LOAD_USHORT_LDS_IDXEN
12520 : HasVOffset ? AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFEN
12521 : AMDGPU::BUFFER_LOAD_USHORT_LDS_OFFSET;
12522 break;
12523 case 4:
12524 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_BOTHEN
12525 : AMDGPU::BUFFER_LOAD_DWORD_LDS_IDXEN
12526 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFEN
12527 : AMDGPU::BUFFER_LOAD_DWORD_LDS_OFFSET;
12528 break;
12529 case 12:
12530 if (!Subtarget->hasLDSLoadB96_B128())
12531 return SDValue();
12532 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_BOTHEN
12533 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_IDXEN
12534 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFEN
12535 : AMDGPU::BUFFER_LOAD_DWORDX3_LDS_OFFSET;
12536 break;
12537 case 16:
12538 if (!Subtarget->hasLDSLoadB96_B128())
12539 return SDValue();
12540 Opc = HasVIndex ? HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_BOTHEN
12541 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_IDXEN
12542 : HasVOffset ? AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFEN
12543 : AMDGPU::BUFFER_LOAD_DWORDX4_LDS_OFFSET;
12544 break;
12545 }
12546
12547 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12548
12549 SmallVector<SDValue, 8> Ops;
12550
12551 if (HasVIndex && HasVOffset)
12552 Ops.push_back(Elt: DAG.getBuildVector(VT: MVT::v2i32, DL,
12553 Ops: {Op.getOperand(i: 5), // VIndex
12554 VOffset}));
12555 else if (HasVIndex)
12556 Ops.push_back(Elt: Op.getOperand(i: 5));
12557 else if (HasVOffset)
12558 Ops.push_back(Elt: VOffset);
12559
12560 SDValue Rsrc = bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG);
12561 Ops.push_back(Elt: Rsrc);
12562 Ops.push_back(Elt: Op.getOperand(i: 6 + OpOffset)); // soffset
12563 Ops.push_back(Elt: Op.getOperand(i: 7 + OpOffset)); // imm offset
12564 bool IsGFX12Plus = AMDGPU::isGFX12Plus(STI: *Subtarget);
12565 unsigned Aux = Op.getConstantOperandVal(i: 8 + OpOffset);
12566 Ops.push_back(Elt: DAG.getTargetConstant(
12567 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::ALL : AMDGPU::CPol::ALL_pregfx12),
12568 DL, VT: MVT::i8)); // cpol
12569 Ops.push_back(Elt: DAG.getTargetConstant(
12570 Val: Aux & (IsGFX12Plus ? AMDGPU::CPol::SWZ : AMDGPU::CPol::SWZ_pregfx12)
12571 ? 1
12572 : 0,
12573 DL, VT: MVT::i8)); // swz
12574 Ops.push_back(
12575 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
12576 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12577 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12578
12579 auto *M = cast<MemSDNode>(Val&: Op);
12580 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: M->getVTList(), Ops);
12581 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12582
12583 return SDValue(Load, 0);
12584 }
12585 // Buffers are handled by LowerBufferFatPointers, and we're going to go
12586 // for "trust me" that the remaining cases are global pointers until
12587 // such time as we can put two mem operands on an intrinsic.
12588 case Intrinsic::amdgcn_load_to_lds:
12589 case Intrinsic::amdgcn_load_async_to_lds:
12590 case Intrinsic::amdgcn_global_load_lds:
12591 case Intrinsic::amdgcn_global_load_async_lds: {
12592 if (!Subtarget->hasVMemToLDSLoad())
12593 return SDValue();
12594
12595 unsigned Opc;
12596 unsigned Size = Op->getConstantOperandVal(Num: 4);
12597 switch (Size) {
12598 default:
12599 return SDValue();
12600 case 1:
12601 Opc = AMDGPU::GLOBAL_LOAD_LDS_UBYTE;
12602 break;
12603 case 2:
12604 Opc = AMDGPU::GLOBAL_LOAD_LDS_USHORT;
12605 break;
12606 case 4:
12607 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORD;
12608 break;
12609 case 12:
12610 if (!Subtarget->hasLDSLoadB96_B128())
12611 return SDValue();
12612 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX3;
12613 break;
12614 case 16:
12615 if (!Subtarget->hasLDSLoadB96_B128())
12616 return SDValue();
12617 Opc = AMDGPU::GLOBAL_LOAD_LDS_DWORDX4;
12618 break;
12619 }
12620
12621 SDValue M0Val = copyToM0(DAG, Chain, DL, V: Op.getOperand(i: 3));
12622
12623 SmallVector<SDValue, 6> Ops;
12624
12625 SDValue Addr = Op.getOperand(i: 2); // Global ptr
12626 SDValue VOffset;
12627 // Try to split SAddr and VOffset. Global and LDS pointers share the same
12628 // immediate offset, so we cannot use a regular SelectGlobalSAddr().
12629 if (Addr->isDivergent() && Addr->isAnyAdd()) {
12630 SDValue LHS = Addr.getOperand(i: 0);
12631 SDValue RHS = Addr.getOperand(i: 1);
12632
12633 if (LHS->isDivergent())
12634 std::swap(a&: LHS, b&: RHS);
12635
12636 if (!LHS->isDivergent() && RHS.getOpcode() == ISD::ZERO_EXTEND &&
12637 RHS.getOperand(i: 0).getValueType() == MVT::i32) {
12638 // add (i64 sgpr), (zero_extend (i32 vgpr))
12639 Addr = LHS;
12640 VOffset = RHS.getOperand(i: 0);
12641 }
12642 }
12643
12644 Ops.push_back(Elt: Addr);
12645 if (!Addr->isDivergent()) {
12646 Opc = AMDGPU::getGlobalSaddrOp(Opcode: Opc);
12647 if (!VOffset)
12648 VOffset =
12649 SDValue(DAG.getMachineNode(Opcode: AMDGPU::V_MOV_B32_e32, dl: DL, VT: MVT::i32,
12650 Op1: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32)),
12651 0);
12652 Ops.push_back(Elt: VOffset);
12653 }
12654
12655 Ops.push_back(Elt: Op.getOperand(i: 5)); // Offset
12656
12657 unsigned Aux = Op.getConstantOperandVal(i: 6);
12658 Ops.push_back(Elt: DAG.getTargetConstant(Val: Aux & ~AMDGPU::CPol::VIRTUAL_BITS, DL,
12659 VT: MVT::i32)); // CPol
12660 Ops.push_back(
12661 Elt: DAG.getTargetConstant(Val: isAsyncLDSDMA(Intr: IntrinsicID), DL, VT: MVT::i8));
12662
12663 Ops.push_back(Elt: M0Val.getValue(R: 0)); // Chain
12664 Ops.push_back(Elt: M0Val.getValue(R: 1)); // Glue
12665
12666 auto *M = cast<MemSDNode>(Val&: Op);
12667 auto *Load = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12668 DAG.setNodeMemRefs(N: Load, NewMemRefs: M->memoperands());
12669
12670 return SDValue(Load, 0);
12671 }
12672 case Intrinsic::amdgcn_end_cf:
12673 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::SI_END_CF, dl: DL, VT: MVT::Other,
12674 Op1: Op->getOperand(Num: 2), Op2: Chain),
12675 0);
12676 case Intrinsic::amdgcn_s_barrier_signal_var: {
12677 // Member count of 0 means to re-use a previous member count,
12678 // which, if the named barrier is statically chosen, means we can use
12679 // the immarg form. Otherwisee, fall through to constructiong M0 as for
12680 // s_barrier_init.
12681 SDValue CntOp = Op->getOperand(Num: 3);
12682 auto *CntC = dyn_cast<ConstantSDNode>(Val&: CntOp);
12683 if (CntC && CntC->isZero()) {
12684 SDValue Chain = Op->getOperand(Num: 0);
12685 SDValue BarOp = Op->getOperand(Num: 2);
12686 SmallVector<SDValue, 2> Ops;
12687
12688 std::optional<uint64_t> BarVal;
12689 if (auto *C = dyn_cast<ConstantSDNode>(Val&: BarOp))
12690 BarVal = C->getZExtValue();
12691 else if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: BarOp))
12692 if (auto Addr = AMDGPUMachineFunctionInfo::getLDSAbsoluteAddress(
12693 GV: *GA->getGlobal()))
12694 BarVal = *Addr + GA->getOffset();
12695
12696 if (BarVal) {
12697 unsigned BarID = (*BarVal >> 4) & 0x3F;
12698 Ops.push_back(Elt: DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32));
12699 Ops.push_back(Elt: Chain);
12700 auto *NewMI = DAG.getMachineNode(Opcode: AMDGPU::S_BARRIER_SIGNAL_IMM, dl: DL,
12701 VTs: Op->getVTList(), Ops);
12702 return SDValue(NewMI, 0);
12703 }
12704 }
12705 [[fallthrough]];
12706 }
12707 case Intrinsic::amdgcn_s_barrier_init: {
12708 // these two intrinsics have two operands: barrier pointer and member count
12709 SDValue Chain = Op->getOperand(Num: 0);
12710 SmallVector<SDValue, 2> Ops;
12711 SDValue BarOp = Op->getOperand(Num: 2);
12712 SDValue CntOp = Op->getOperand(Num: 3);
12713 SDValue M0Val;
12714 unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
12715 ? AMDGPU::S_BARRIER_INIT_M0
12716 : AMDGPU::S_BARRIER_SIGNAL_M0;
12717 // extract the BarrierID from bits 4-9 of BarOp
12718 SDValue BarID;
12719 BarID = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: BarOp,
12720 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
12721 BarID = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: BarID,
12722 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12723 // Member count should be put into M0[ShAmt:+6]
12724 // Barrier ID should be put into M0[5:0]
12725 SDValue MemberCnt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: CntOp,
12726 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12727 constexpr unsigned ShAmt = 16;
12728 M0Val = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: MemberCnt,
12729 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
12730
12731 M0Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M0Val, N2: BarID);
12732
12733 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12734
12735 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12736 return SDValue(NewMI, 0);
12737 }
12738 case Intrinsic::amdgcn_s_wakeup_barrier: {
12739 if (!Subtarget->hasSWakeupBarrier())
12740 return SDValue();
12741 [[fallthrough]];
12742 }
12743 case Intrinsic::amdgcn_s_barrier_join: {
12744 // these three intrinsics have one operand: barrier pointer
12745 SDValue Chain = Op->getOperand(Num: 0);
12746 SmallVector<SDValue, 2> Ops;
12747 SDValue BarOp = Op->getOperand(Num: 2);
12748 unsigned Opc;
12749
12750 if (isa<ConstantSDNode>(Val: BarOp)) {
12751 uint64_t BarVal = cast<ConstantSDNode>(Val&: BarOp)->getZExtValue();
12752 switch (IntrinsicID) {
12753 default:
12754 return SDValue();
12755 case Intrinsic::amdgcn_s_barrier_join:
12756 Opc = AMDGPU::S_BARRIER_JOIN_IMM;
12757 break;
12758 case Intrinsic::amdgcn_s_wakeup_barrier:
12759 Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
12760 break;
12761 }
12762 // extract the BarrierID from bits 4-9 of the immediate
12763 unsigned BarID = (BarVal >> 4) & 0x3F;
12764 SDValue K = DAG.getTargetConstant(Val: BarID, DL, VT: MVT::i32);
12765 Ops.push_back(Elt: K);
12766 Ops.push_back(Elt: Chain);
12767 } else {
12768 switch (IntrinsicID) {
12769 default:
12770 return SDValue();
12771 case Intrinsic::amdgcn_s_barrier_join:
12772 Opc = AMDGPU::S_BARRIER_JOIN_M0;
12773 break;
12774 case Intrinsic::amdgcn_s_wakeup_barrier:
12775 Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
12776 break;
12777 }
12778 // extract the BarrierID from bits 4-9 of BarOp, copy to M0[5:0]
12779 SDValue M0Val;
12780 M0Val = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: BarOp,
12781 N2: DAG.getShiftAmountConstant(Val: 4, VT: MVT::i32, DL));
12782 M0Val = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M0Val,
12783 N2: DAG.getConstant(Val: 0x3F, DL, VT: MVT::i32));
12784 Ops.push_back(Elt: copyToM0(DAG, Chain, DL, V: M0Val).getValue(R: 0));
12785 }
12786
12787 auto *NewMI = DAG.getMachineNode(Opcode: Opc, dl: DL, VTs: Op->getVTList(), Ops);
12788 return SDValue(NewMI, 0);
12789 }
12790 case Intrinsic::amdgcn_s_prefetch_data:
12791 case Intrinsic::amdgcn_s_prefetch_inst: {
12792 // For non-global address space preserve the chain and remove the call.
12793 if (!AMDGPU::isFlatGlobalAddrSpace(AS: cast<MemSDNode>(Val&: Op)->getAddressSpace()))
12794 return Op.getOperand(i: 0);
12795 return Op;
12796 }
12797 case Intrinsic::amdgcn_s_buffer_prefetch_data: {
12798 SDValue Ops[] = {
12799 Chain, bufferRsrcPtrToVector(MaybePointer: Op.getOperand(i: 2), DAG),
12800 Op.getOperand(i: 3), // offset
12801 Op.getOperand(i: 4), // length
12802 };
12803
12804 MemSDNode *M = cast<MemSDNode>(Val&: Op);
12805 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::SBUFFER_PREFETCH_DATA, dl: DL,
12806 VTList: Op->getVTList(), Ops, MemVT: M->getMemoryVT(),
12807 MMO: M->getMemOperand());
12808 }
12809 case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
12810 case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
12811 case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
12812 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12813 SDValue Chain = Op->getOperand(Num: 0);
12814 SDValue Ptr = Op->getOperand(Num: 2);
12815 SDValue Val = Op->getOperand(Num: 3);
12816 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: MII->getMemoryVT(), Chain, Ptr: Val,
12817 Val: Ptr, MMO: MII->getMemOperand());
12818 }
12819 case Intrinsic::amdgcn_av_store_b128: {
12820 MemIntrinsicSDNode *MII = cast<MemIntrinsicSDNode>(Val&: Op);
12821 SDValue Chain = Op->getOperand(Num: 0);
12822 SDValue Ptr = Op->getOperand(Num: 2);
12823 SDValue Val = Op->getOperand(Num: 3);
12824 return DAG.getStore(Chain, dl: DL, Val, Ptr, MMO: MII->getMemOperand());
12825 }
12826 default: {
12827 if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
12828 AMDGPU::getImageDimIntrinsicInfo(Intr: IntrinsicID))
12829 return lowerImage(Op, Intr: ImageDimIntr, DAG, WithChain: true);
12830
12831 return Op;
12832 }
12833 }
12834}
12835
12836// Return whether the operation has NoUnsignedWrap property.
12837static bool isNoUnsignedWrap(SDValue Addr) {
12838 return (Addr.getOpcode() == ISD::ADD &&
12839 Addr->getFlags().hasNoUnsignedWrap()) ||
12840 Addr->getOpcode() == ISD::OR;
12841}
12842
12843bool SITargetLowering::shouldPreservePtrArith(const Function &F,
12844 EVT PtrVT) const {
12845 return PtrVT == MVT::i64;
12846}
12847
12848bool SITargetLowering::canTransformPtrArithOutOfBounds(const Function &F,
12849 EVT PtrVT) const {
12850 return true;
12851}
12852
12853// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
12854// offset (the offset that is included in bounds checking and swizzling, to be
12855// split between the instruction's voffset and immoffset fields) and soffset
12856// (the offset that is excluded from bounds checking and swizzling, to go in
12857// the instruction's soffset field). This function takes the first kind of
12858// offset and figures out how to split it between voffset and immoffset.
12859std::pair<SDValue, SDValue>
12860SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
12861 SDLoc DL(Offset);
12862 const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST: *Subtarget);
12863 SDValue N0 = Offset;
12864 ConstantSDNode *C1 = nullptr;
12865
12866 if ((C1 = dyn_cast<ConstantSDNode>(Val&: N0)))
12867 N0 = SDValue();
12868 else if (DAG.isBaseWithConstantOffset(Op: N0)) {
12869 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
12870 // being added, so we can only safely match a 32-bit addition with no
12871 // unsigned overflow.
12872 bool CheckNUW = Subtarget->hasGFX1250Insts();
12873 if (!CheckNUW || isNoUnsignedWrap(Addr: N0)) {
12874 C1 = cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
12875 N0 = N0.getOperand(i: 0);
12876 }
12877 }
12878
12879 if (C1) {
12880 unsigned ImmOffset = C1->getZExtValue();
12881 // If the immediate value is too big for the immoffset field, put only bits
12882 // that would normally fit in the immoffset field. The remaining value that
12883 // is copied/added for the voffset field is a large power of 2, and it
12884 // stands more chance of being CSEd with the copy/add for another similar
12885 // load/store.
12886 // However, do not do that rounding down if that is a negative
12887 // number, as it appears to be illegal to have a negative offset in the
12888 // vgpr, even if adding the immediate offset makes it positive.
12889 unsigned Overflow = ImmOffset & ~MaxImm;
12890 ImmOffset -= Overflow;
12891 if ((int32_t)Overflow < 0) {
12892 Overflow += ImmOffset;
12893 ImmOffset = 0;
12894 }
12895 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32));
12896 if (Overflow) {
12897 auto OverflowVal = DAG.getConstant(Val: Overflow, DL, VT: MVT::i32);
12898 if (!N0)
12899 N0 = OverflowVal;
12900 else {
12901 SDValue Ops[] = {N0, OverflowVal};
12902 N0 = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, Ops);
12903 }
12904 }
12905 }
12906 if (!N0)
12907 N0 = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12908 if (!C1)
12909 C1 = cast<ConstantSDNode>(Val: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
12910 return {N0, SDValue(C1, 0)};
12911}
12912
12913// Analyze a combined offset from an amdgcn_s_buffer_load intrinsic and store
12914// the three offsets (voffset, soffset and instoffset) into the SDValue[3] array
12915// pointed to by Offsets.
12916void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
12917 SelectionDAG &DAG, SDValue *Offsets,
12918 Align Alignment) const {
12919 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
12920 SDLoc DL(CombinedOffset);
12921 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CombinedOffset)) {
12922 uint32_t Imm = C->getZExtValue();
12923 uint32_t SOffset, ImmOffset;
12924 if (TII->splitMUBUFOffset(Imm, SOffset, ImmOffset, Alignment)) {
12925 Offsets[0] = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12926 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
12927 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
12928 return;
12929 }
12930 }
12931 if (DAG.isBaseWithConstantOffset(Op: CombinedOffset)) {
12932 // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
12933 // being added, so we can only safely match a 32-bit addition with no
12934 // unsigned overflow.
12935 bool CheckNUW = Subtarget->hasGFX1250Insts();
12936 SDValue N0 = CombinedOffset.getOperand(i: 0);
12937 SDValue N1 = CombinedOffset.getOperand(i: 1);
12938 uint32_t SOffset, ImmOffset;
12939 int Offset = cast<ConstantSDNode>(Val&: N1)->getSExtValue();
12940 if (Offset >= 0 && (!CheckNUW || isNoUnsignedWrap(Addr: CombinedOffset)) &&
12941 TII->splitMUBUFOffset(Imm: Offset, SOffset, ImmOffset, Alignment)) {
12942 Offsets[0] = N0;
12943 Offsets[1] = DAG.getConstant(Val: SOffset, DL, VT: MVT::i32);
12944 Offsets[2] = DAG.getTargetConstant(Val: ImmOffset, DL, VT: MVT::i32);
12945 return;
12946 }
12947 }
12948
12949 SDValue SOffsetZero = Subtarget->hasRestrictedSOffset()
12950 ? DAG.getRegister(Reg: AMDGPU::SGPR_NULL, VT: MVT::i32)
12951 : DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12952
12953 Offsets[0] = CombinedOffset;
12954 Offsets[1] = SOffsetZero;
12955 Offsets[2] = DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32);
12956}
12957
12958SDValue SITargetLowering::bufferRsrcPtrToVector(SDValue MaybePointer,
12959 SelectionDAG &DAG) const {
12960 if (!MaybePointer.getValueType().isScalarInteger())
12961 return MaybePointer;
12962
12963 SDValue Rsrc = DAG.getBitcast(VT: MVT::v4i32, V: MaybePointer);
12964 return Rsrc;
12965}
12966
12967// Wrap a global or flat pointer into a buffer intrinsic using the flags
12968// specified in the intrinsic.
12969SDValue SITargetLowering::lowerPointerAsRsrcIntrin(SDNode *Op,
12970 SelectionDAG &DAG) const {
12971 SDLoc Loc(Op);
12972
12973 SDValue Pointer = Op->getOperand(Num: 1);
12974 SDValue Stride = Op->getOperand(Num: 2);
12975 SDValue NumRecords = Op->getOperand(Num: 3);
12976 SDValue Flags = Op->getOperand(Num: 4);
12977
12978 SDValue ExtStride = DAG.getAnyExtOrTrunc(Op: Stride, DL: Loc, VT: MVT::i32);
12979 SDValue Rsrc;
12980
12981 if (Subtarget->has45BitNumRecordsBufferResource()) {
12982 SDValue Zero = DAG.getConstant(Val: 0, DL: Loc, VT: MVT::i32);
12983 // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
12984 // num_records.
12985 SDValue ExtPointer = DAG.getAnyExtOrTrunc(Op: Pointer, DL: Loc, VT: MVT::i64);
12986 SDValue NumRecordsLHS =
12987 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i64, N1: NumRecords,
12988 N2: DAG.getShiftAmountConstant(Val: 57, VT: MVT::i32, DL: Loc));
12989 SDValue LowHalf =
12990 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: ExtPointer, N2: NumRecordsLHS);
12991
12992 // Build the higher 64-bit value, which has the higher 38-bit num_records,
12993 // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
12994 SDValue NumRecordsRHS =
12995 DAG.getNode(Opcode: ISD::SRL, DL: Loc, VT: MVT::i64, N1: NumRecords,
12996 N2: DAG.getShiftAmountConstant(Val: 7, VT: MVT::i32, DL: Loc));
12997 SDValue ShiftedStride =
12998 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
12999 N2: DAG.getShiftAmountConstant(Val: 12, VT: MVT::i32, DL: Loc));
13000 SDValue ExtShiftedStrideVec =
13001 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedStride);
13002 SDValue ExtShiftedStride =
13003 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedStrideVec);
13004 SDValue ShiftedFlags =
13005 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: Flags,
13006 N2: DAG.getShiftAmountConstant(Val: 28, VT: MVT::i32, DL: Loc));
13007 SDValue ExtShiftedFlagsVec =
13008 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i32, N1: Zero, N2: ShiftedFlags);
13009 SDValue ExtShiftedFlags =
13010 DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i64, Operand: ExtShiftedFlagsVec);
13011 SDValue CombinedFields =
13012 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: NumRecordsRHS, N2: ExtShiftedStride);
13013 SDValue HighHalf =
13014 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i64, N1: CombinedFields, N2: ExtShiftedFlags);
13015
13016 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v2i64, N1: LowHalf, N2: HighHalf);
13017 } else {
13018 NumRecords = DAG.getAnyExtOrTrunc(Op: NumRecords, DL: Loc, VT: MVT::i32);
13019 auto [LowHalf, HighHalf] =
13020 DAG.SplitScalar(N: Pointer, DL: Loc, LoVT: MVT::i32, HiVT: MVT::i32);
13021 SDValue Mask = DAG.getConstant(Val: 0x0000ffff, DL: Loc, VT: MVT::i32);
13022 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL: Loc, VT: MVT::i32, N1: HighHalf, N2: Mask);
13023 SDValue ShiftedStride =
13024 DAG.getNode(Opcode: ISD::SHL, DL: Loc, VT: MVT::i32, N1: ExtStride,
13025 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: Loc));
13026 SDValue NewHighHalf =
13027 DAG.getNode(Opcode: ISD::OR, DL: Loc, VT: MVT::i32, N1: Masked, N2: ShiftedStride);
13028
13029 Rsrc = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: Loc, VT: MVT::v4i32, N1: LowHalf, N2: NewHighHalf,
13030 N3: NumRecords, N4: Flags);
13031 }
13032
13033 SDValue RsrcPtr = DAG.getNode(Opcode: ISD::BITCAST, DL: Loc, VT: MVT::i128, Operand: Rsrc);
13034 return RsrcPtr;
13035}
13036
13037// Handle 8 bit and 16 bit buffer loads
13038SDValue SITargetLowering::handleByteShortBufferLoads(SelectionDAG &DAG,
13039 EVT LoadVT, SDLoc DL,
13040 ArrayRef<SDValue> Ops,
13041 MachineMemOperand *MMO,
13042 bool IsTFE) const {
13043 EVT IntVT = LoadVT.changeTypeToInteger();
13044
13045 if (IsTFE) {
13046 unsigned Opc = (LoadVT.getScalarType() == MVT::i8)
13047 ? AMDGPUISD::BUFFER_LOAD_UBYTE_TFE
13048 : AMDGPUISD::BUFFER_LOAD_USHORT_TFE;
13049 MachineFunction &MF = DAG.getMachineFunction();
13050 MachineMemOperand *OpMMO = MF.getMachineMemOperand(MMO, Offset: 0, Size: 8);
13051 SDVTList VTs = DAG.getVTList(VT1: MVT::v2i32, VT2: MVT::Other);
13052 SDValue Op = getMemIntrinsicNode(Opcode: Opc, DL, VTList: VTs, Ops, MemVT: MVT::v2i32, MMO: OpMMO, DAG);
13053 SDValue Status = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13054 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
13055 SDValue Data = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Op,
13056 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
13057 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: Data);
13058 SDValue Value = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: Trunc);
13059 return DAG.getMergeValues(Ops: {Value, Status, SDValue(Op.getNode(), 1)}, dl: DL);
13060 }
13061
13062 unsigned Opc = LoadVT.getScalarType() == MVT::i8
13063 ? AMDGPUISD::BUFFER_LOAD_UBYTE
13064 : AMDGPUISD::BUFFER_LOAD_USHORT;
13065
13066 SDVTList ResList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
13067 SDValue BufferLoad =
13068 DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: IntVT, MMO);
13069 SDValue LoadVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IntVT, Operand: BufferLoad);
13070 LoadVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LoadVT, Operand: LoadVal);
13071
13072 return DAG.getMergeValues(Ops: {LoadVal, BufferLoad.getValue(R: 1)}, dl: DL);
13073}
13074
13075// Handle 8 bit and 16 bit buffer stores
13076SDValue SITargetLowering::handleByteShortBufferStores(SelectionDAG &DAG,
13077 EVT VDataType, SDLoc DL,
13078 SDValue Ops[],
13079 MemSDNode *M) const {
13080 if (VDataType == MVT::f16 || VDataType == MVT::bf16)
13081 Ops[1] = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Ops[1]);
13082
13083 SDValue BufferStoreExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Ops[1]);
13084 Ops[1] = BufferStoreExt;
13085 unsigned Opc = (VDataType == MVT::i8) ? AMDGPUISD::BUFFER_STORE_BYTE
13086 : AMDGPUISD::BUFFER_STORE_SHORT;
13087 ArrayRef<SDValue> OpsRef = ArrayRef(&Ops[0], 9);
13088 return DAG.getMemIntrinsicNode(Opcode: Opc, dl: DL, VTList: M->getVTList(), Ops: OpsRef, MemVT: VDataType,
13089 MMO: M->getMemOperand());
13090}
13091
13092static SDValue getLoadExtOrTrunc(SelectionDAG &DAG, ISD::LoadExtType ExtType,
13093 SDValue Op, const SDLoc &SL, EVT VT) {
13094 if (VT.bitsLT(VT: Op.getValueType()))
13095 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
13096
13097 switch (ExtType) {
13098 case ISD::SEXTLOAD:
13099 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SL, VT, Operand: Op);
13100 case ISD::ZEXTLOAD:
13101 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT, Operand: Op);
13102 case ISD::EXTLOAD:
13103 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT, Operand: Op);
13104 case ISD::NON_EXTLOAD:
13105 return Op;
13106 }
13107
13108 llvm_unreachable("invalid ext type");
13109}
13110
13111// Try to turn 8 and 16-bit scalar loads into SMEM eligible 32-bit loads.
13112// TODO: Skip this on GFX12 which does have scalar sub-dword loads.
13113SDValue SITargetLowering::widenLoad(LoadSDNode *Ld,
13114 DAGCombinerInfo &DCI) const {
13115 SelectionDAG &DAG = DCI.DAG;
13116 if (Ld->getAlign() < Align(4) || Ld->isDivergent())
13117 return SDValue();
13118
13119 // FIXME: Constant loads should all be marked invariant.
13120 unsigned AS = Ld->getAddressSpace();
13121 if (AS != AMDGPUAS::CONSTANT_ADDRESS &&
13122 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
13123 (AS != AMDGPUAS::GLOBAL_ADDRESS || !Ld->isInvariant()))
13124 return SDValue();
13125
13126 // Don't do this early, since it may interfere with adjacent load merging for
13127 // illegal types. We can avoid losing alignment information for exotic types
13128 // pre-legalize.
13129 EVT MemVT = Ld->getMemoryVT();
13130 if ((MemVT.isSimple() && !DCI.isAfterLegalizeDAG()) ||
13131 MemVT.getSizeInBits() >= 32)
13132 return SDValue();
13133
13134 SDLoc SL(Ld);
13135
13136 assert((!MemVT.isVector() || Ld->getExtensionType() == ISD::NON_EXTLOAD) &&
13137 "unexpected vector extload");
13138
13139 // TODO: Drop only high part of range.
13140 SDValue Ptr = Ld->getBasePtr();
13141 SDValue NewLoad = DAG.getLoad(
13142 AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT: MVT::i32, dl: SL, Chain: Ld->getChain(), Ptr,
13143 Offset: Ld->getOffset(), PtrInfo: Ld->getPointerInfo(), MemVT: MVT::i32, Alignment: Ld->getAlign(),
13144 MMOFlags: Ld->getMemOperand()->getFlags(), AAInfo: Ld->getAAInfo(),
13145 Ranges: nullptr); // Drop ranges
13146
13147 EVT TruncVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
13148 if (MemVT.isFloatingPoint()) {
13149 assert(Ld->getExtensionType() == ISD::NON_EXTLOAD &&
13150 "unexpected fp extload");
13151 TruncVT = MemVT.changeTypeToInteger();
13152 }
13153
13154 SDValue Cvt = NewLoad;
13155 if (Ld->getExtensionType() == ISD::SEXTLOAD) {
13156 Cvt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: SL, VT: MVT::i32, N1: NewLoad,
13157 N2: DAG.getValueType(TruncVT));
13158 } else if (Ld->getExtensionType() == ISD::ZEXTLOAD ||
13159 Ld->getExtensionType() == ISD::NON_EXTLOAD) {
13160 Cvt = DAG.getZeroExtendInReg(Op: NewLoad, DL: SL, VT: TruncVT);
13161 } else {
13162 assert(Ld->getExtensionType() == ISD::EXTLOAD);
13163 }
13164
13165 EVT VT = Ld->getValueType(ResNo: 0);
13166 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
13167
13168 DCI.AddToWorklist(N: Cvt.getNode());
13169
13170 // We may need to handle exotic cases, such as i16->i64 extloads, so insert
13171 // the appropriate extension from the 32-bit load.
13172 Cvt = getLoadExtOrTrunc(DAG, ExtType: Ld->getExtensionType(), Op: Cvt, SL, VT: IntVT);
13173 DCI.AddToWorklist(N: Cvt.getNode());
13174
13175 // Handle conversion back to floating point if necessary.
13176 Cvt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Cvt);
13177
13178 return DAG.getMergeValues(Ops: {Cvt, NewLoad.getValue(R: 1)}, dl: SL);
13179}
13180
13181static bool addressMayBeAccessedAsPrivate(const MachineMemOperand *MMO,
13182 const SIMachineFunctionInfo &Info) {
13183 // TODO: Should check if the address can definitely not access stack.
13184 if (Info.isEntryFunction())
13185 return Info.getUserSGPRInfo().hasFlatScratchInit();
13186 return true;
13187}
13188
13189SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
13190 SDLoc DL(Op);
13191 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
13192 ISD::LoadExtType ExtType = Load->getExtensionType();
13193 EVT MemVT = Load->getMemoryVT();
13194 MachineMemOperand *MMO = Load->getMemOperand();
13195
13196 if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
13197 if (MemVT == MVT::i16 && isTypeLegal(VT: MVT::i16))
13198 return SDValue();
13199
13200 // FIXME: Copied from PPC
13201 // First, load into 32 bits, then truncate to 1 bit.
13202
13203 SDValue Chain = Load->getChain();
13204 SDValue BasePtr = Load->getBasePtr();
13205
13206 EVT RealMemVT = (MemVT == MVT::i1) ? MVT::i8 : MVT::i16;
13207
13208 SDValue NewLD = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: MVT::i32, Chain, Ptr: BasePtr,
13209 MemVT: RealMemVT, MMO);
13210
13211 if (!MemVT.isVector()) {
13212 SDValue Ops[] = {DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MemVT, Operand: NewLD),
13213 NewLD.getValue(R: 1)};
13214
13215 return DAG.getMergeValues(Ops, dl: DL);
13216 }
13217
13218 SmallVector<SDValue, 3> Elts;
13219 for (unsigned I = 0, N = MemVT.getVectorNumElements(); I != N; ++I) {
13220 SDValue Elt = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: NewLD,
13221 N2: DAG.getConstant(Val: I, DL, VT: MVT::i32));
13222
13223 Elts.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Elt));
13224 }
13225
13226 SDValue Ops[] = {DAG.getBuildVector(VT: MemVT, DL, Ops: Elts), NewLD.getValue(R: 1)};
13227
13228 return DAG.getMergeValues(Ops, dl: DL);
13229 }
13230
13231 if (!MemVT.isVector())
13232 return SDValue();
13233
13234 assert(Op.getValueType().getVectorElementType() == MVT::i32 &&
13235 "Custom lowering for non-i32 vectors hasn't been implemented.");
13236
13237 Align Alignment = Load->getAlign();
13238 unsigned AS = Load->getAddressSpace();
13239 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13240 AS == AMDGPUAS::FLAT_ADDRESS &&
13241 Alignment.value() < MemVT.getStoreSize() && MemVT.getSizeInBits() > 32) {
13242 return SplitVectorLoad(Op, DAG);
13243 }
13244
13245 MachineFunction &MF = DAG.getMachineFunction();
13246 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
13247 // If there is a possibility that flat instruction access scratch memory
13248 // then we need to use the same legalization rules we use for private.
13249 if (AS == AMDGPUAS::FLAT_ADDRESS &&
13250 !Subtarget->hasMultiDwordFlatScratchAddressing())
13251 AS = addressMayBeAccessedAsPrivate(MMO: Load->getMemOperand(), Info: *MFI)
13252 ? AMDGPUAS::PRIVATE_ADDRESS
13253 : AMDGPUAS::GLOBAL_ADDRESS;
13254
13255 unsigned NumElements = MemVT.getVectorNumElements();
13256
13257 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13258 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13259 (AS == AMDGPUAS::GLOBAL_ADDRESS &&
13260 Subtarget->getScalarizeGlobalBehavior() && Load->isSimple() &&
13261 (Load->isInvariant() || isMemOpHasNoClobberedMemOperand(N: Load)))) {
13262 if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) &&
13263 Alignment >= Align(4) && NumElements < 32) {
13264 if (MemVT.isPow2VectorType() ||
13265 (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
13266 return SDValue();
13267 return WidenOrSplitVectorLoad(Op, DAG);
13268 }
13269 // Non-uniform loads will be selected to MUBUF instructions, so they
13270 // have the same legalization requirements as global and private
13271 // loads.
13272 //
13273 }
13274 if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
13275 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
13276 AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
13277 if (NumElements > 4)
13278 return SplitVectorLoad(Op, DAG);
13279 // v3 loads not supported on SI.
13280 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13281 return WidenOrSplitVectorLoad(Op, DAG);
13282
13283 // v3 and v4 loads are supported for private and global memory.
13284 return SDValue();
13285 }
13286 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
13287 // Depending on the setting of the private_element_size field in the
13288 // resource descriptor, we can only make private accesses up to a certain
13289 // size.
13290 switch (Subtarget->getMaxPrivateElementSize()) {
13291 case 4: {
13292 auto [Op0, Op1] = scalarizeVectorLoad(LD: Load, DAG);
13293 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13294 }
13295 case 8:
13296 if (NumElements > 2)
13297 return SplitVectorLoad(Op, DAG);
13298 return SDValue();
13299 case 16:
13300 // Same as global/flat
13301 if (NumElements > 4)
13302 return SplitVectorLoad(Op, DAG);
13303 // v3 loads not supported on SI.
13304 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13305 return WidenOrSplitVectorLoad(Op, DAG);
13306
13307 return SDValue();
13308 default:
13309 llvm_unreachable("unsupported private_element_size");
13310 }
13311 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
13312 unsigned Fast = 0;
13313 auto Flags = Load->getMemOperand()->getFlags();
13314 if (allowsMisalignedMemoryAccessesImpl(Size: MemVT.getSizeInBits(), AddrSpace: AS,
13315 Alignment: Load->getAlign(), Flags, IsFast: &Fast) &&
13316 Fast > 1)
13317 return SDValue();
13318
13319 if (MemVT.isVector())
13320 return SplitVectorLoad(Op, DAG);
13321 }
13322
13323 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
13324 VT: MemVT, MMO: *Load->getMemOperand())) {
13325 auto [Op0, Op1] = expandUnalignedLoad(LD: Load, DAG);
13326 return DAG.getMergeValues(Ops: {Op0, Op1}, dl: DL);
13327 }
13328
13329 return SDValue();
13330}
13331
13332SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
13333 EVT VT = Op.getValueType();
13334 if (VT.getSizeInBits() == 128 || VT.getSizeInBits() == 256 ||
13335 VT.getSizeInBits() == 512)
13336 return splitTernaryVectorOp(Op, DAG);
13337
13338 assert(VT.getSizeInBits() == 64);
13339
13340 SDLoc DL(Op);
13341 SDValue Cond = DAG.getFreeze(V: Op.getOperand(i: 0));
13342
13343 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
13344 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
13345
13346 SDValue LHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 1));
13347 SDValue RHS = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v2i32, Operand: Op.getOperand(i: 2));
13348
13349 SDValue Lo0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: Zero);
13350 SDValue Lo1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: Zero);
13351
13352 SDValue Lo = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Lo0, RHS: Lo1);
13353
13354 SDValue Hi0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: LHS, N2: One);
13355 SDValue Hi1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: RHS, N2: One);
13356
13357 SDValue Hi = DAG.getSelect(DL, VT: MVT::i32, Cond, LHS: Hi0, RHS: Hi1);
13358
13359 SDValue Res = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Lo, Hi});
13360 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Res);
13361}
13362
13363// Catch division cases where we can use shortcuts with rcp and rsq
13364// instructions.
13365SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op,
13366 SelectionDAG &DAG) const {
13367 SDLoc SL(Op);
13368 SDValue LHS = Op.getOperand(i: 0);
13369 SDValue RHS = Op.getOperand(i: 1);
13370 EVT VT = Op.getValueType();
13371 const SDNodeFlags Flags = Op->getFlags();
13372
13373 bool AllowInaccurateRcp = Flags.hasApproximateFuncs();
13374
13375 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
13376 // Without !fpmath accuracy information, we can't do more because we don't
13377 // know exactly whether rcp is accurate enough to meet !fpmath requirement.
13378 // f16 is always accurate enough
13379 if (!AllowInaccurateRcp && VT != MVT::f16 && VT != MVT::bf16)
13380 return SDValue();
13381
13382 if (CLHS->isOne()) {
13383 // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
13384 // the CI documentation has a worst case error of 1 ulp.
13385 // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
13386 // use it as long as we aren't trying to use denormals.
13387 //
13388 // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
13389
13390 // 1.0 / sqrt(x) -> rsq(x)
13391
13392 // XXX - Is afn sufficient to do this for f64? The maximum ULP
13393 // error seems really high at 2^29 ULP.
13394 // 1.0 / x -> rcp(x)
13395 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13396 }
13397
13398 // Same as for 1.0, but expand the sign out of the constant.
13399 if (CLHS->isMinusOne()) {
13400 // -1.0 / x -> rcp (fneg x)
13401 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
13402 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: FNegRHS);
13403 }
13404 }
13405
13406 // For f16 and bf16 require afn or arcp.
13407 // For f32 require afn.
13408 if (!AllowInaccurateRcp &&
13409 ((VT != MVT::f16 && VT != MVT::bf16) || !Flags.hasAllowReciprocal()))
13410 return SDValue();
13411
13412 // Turn into multiply by the reciprocal.
13413 // x / y -> x * (1.0 / y)
13414 SDValue Recip = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: RHS);
13415 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LHS, N2: Recip, Flags);
13416}
13417
13418SDValue SITargetLowering::lowerFastUnsafeFDIV64(SDValue Op,
13419 SelectionDAG &DAG) const {
13420 SDLoc SL(Op);
13421 SDValue X = Op.getOperand(i: 0);
13422 SDValue Y = Op.getOperand(i: 1);
13423 EVT VT = Op.getValueType();
13424 const SDNodeFlags Flags = Op->getFlags();
13425
13426 bool AllowInaccurateDiv = Flags.hasApproximateFuncs();
13427 if (!AllowInaccurateDiv)
13428 return SDValue();
13429
13430 const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: X);
13431 bool IsNegRcp = CLHS && CLHS->isMinusOne();
13432
13433 // Pull out the negation so it folds for free into the source modifiers.
13434 if (IsNegRcp)
13435 X = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13436
13437 SDValue NegY = IsNegRcp ? Y : DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y);
13438 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
13439
13440 SDValue R = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: Y);
13441 if (IsNegRcp)
13442 R = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: R);
13443
13444 SDValue Tmp0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13445
13446 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp0, N2: R, N3: R);
13447 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: R, N3: One);
13448 R = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp1, N2: R, N3: R);
13449
13450 // Skip the last 2 correction terms for reciprocal.
13451 if (IsNegRcp || (CLHS && CLHS->isOne()))
13452 return R;
13453
13454 SDValue Ret = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: R);
13455 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: NegY, N2: Ret, N3: X);
13456 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: Tmp2, N2: R, N3: Ret);
13457}
13458
13459static SDValue getFPBinOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13460 EVT VT, SDValue A, SDValue B, SDValue GlueChain,
13461 SDNodeFlags Flags) {
13462 if (GlueChain->getNumValues() <= 1) {
13463 return DAG.getNode(Opcode, DL: SL, VT, N1: A, N2: B, Flags);
13464 }
13465
13466 assert(GlueChain->getNumValues() == 3);
13467
13468 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13469 switch (Opcode) {
13470 default:
13471 llvm_unreachable("no chain equivalent for opcode");
13472 case ISD::FMUL:
13473 Opcode = AMDGPUISD::FMUL_W_CHAIN;
13474 break;
13475 }
13476
13477 return DAG.getNode(Opcode, DL: SL, VTList,
13478 Ops: {GlueChain.getValue(R: 1), A, B, GlueChain.getValue(R: 2)},
13479 Flags);
13480}
13481
13482static SDValue getFPTernOp(SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL,
13483 EVT VT, SDValue A, SDValue B, SDValue C,
13484 SDValue GlueChain, SDNodeFlags Flags) {
13485 if (GlueChain->getNumValues() <= 1) {
13486 return DAG.getNode(Opcode, DL: SL, VT, Ops: {A, B, C}, Flags);
13487 }
13488
13489 assert(GlueChain->getNumValues() == 3);
13490
13491 SDVTList VTList = DAG.getVTList(VT1: VT, VT2: MVT::Other, VT3: MVT::Glue);
13492 switch (Opcode) {
13493 default:
13494 llvm_unreachable("no chain equivalent for opcode");
13495 case ISD::FMA:
13496 Opcode = AMDGPUISD::FMA_W_CHAIN;
13497 break;
13498 }
13499
13500 return DAG.getNode(Opcode, DL: SL, VTList,
13501 Ops: {GlueChain.getValue(R: 1), A, B, C, GlueChain.getValue(R: 2)},
13502 Flags);
13503}
13504
13505SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
13506 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13507 return FastLowered;
13508
13509 SDLoc SL(Op);
13510 EVT VT = Op.getValueType();
13511 SDValue LHS = Op.getOperand(i: 0);
13512 SDValue RHS = Op.getOperand(i: 1);
13513
13514 SDValue LHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: LHS);
13515 SDValue RHSExt = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: RHS);
13516
13517 if (VT == MVT::bf16) {
13518 SDValue ExtDiv =
13519 DAG.getNode(Opcode: ISD::FDIV, DL: SL, VT: MVT::f32, N1: LHSExt, N2: RHSExt, Flags: Op->getFlags());
13520 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::bf16, N1: ExtDiv,
13521 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13522 }
13523
13524 assert(VT == MVT::f16);
13525
13526 // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
13527 // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
13528 // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
13529 // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
13530 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13531 // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
13532 // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
13533 // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
13534 // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
13535 // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
13536 // q16.u = opx(V_CVT_F16_F32, q32.u);
13537 // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
13538
13539 // We will use ISD::FMA on targets that don't support ISD::FMAD.
13540 unsigned FMADOpCode =
13541 isOperationLegal(Op: ISD::FMAD, VT: MVT::f32) ? ISD::FMAD : ISD::FMA;
13542 SDValue NegRHSExt = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHSExt);
13543 SDValue Rcp =
13544 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: RHSExt, Flags: Op->getFlags());
13545 SDValue Quot =
13546 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHSExt, N2: Rcp, Flags: Op->getFlags());
13547 SDValue Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13548 Flags: Op->getFlags());
13549 Quot = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, N3: Quot, Flags: Op->getFlags());
13550 Err = DAG.getNode(Opcode: FMADOpCode, DL: SL, VT: MVT::f32, N1: NegRHSExt, N2: Quot, N3: LHSExt,
13551 Flags: Op->getFlags());
13552 SDValue Tmp = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: Err, N2: Rcp, Flags: Op->getFlags());
13553 SDValue TmpCast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Tmp);
13554 TmpCast = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: TmpCast,
13555 N2: DAG.getConstant(Val: 0xff800000, DL: SL, VT: MVT::i32));
13556 Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: TmpCast);
13557 Quot = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f32, N1: Tmp, N2: Quot, Flags: Op->getFlags());
13558 SDValue RDst = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Quot,
13559 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32));
13560 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f16, N1: RDst, N2: RHS, N3: LHS,
13561 Flags: Op->getFlags());
13562}
13563
13564// Faster 2.5 ULP division that does not support denormals.
13565SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const {
13566 SDNodeFlags Flags = Op->getFlags();
13567 SDLoc SL(Op);
13568 SDValue LHS = Op.getOperand(i: 1);
13569 SDValue RHS = Op.getOperand(i: 2);
13570
13571 // TODO: The combiner should probably handle elimination of redundant fabs.
13572 SDValue r1 = DAG.SignBitIsZeroFP(Op: RHS)
13573 ? RHS
13574 : DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f32, Operand: RHS, Flags);
13575
13576 const APFloat K0Val(0x1p+96f);
13577 const SDValue K0 = DAG.getConstantFP(Val: K0Val, DL: SL, VT: MVT::f32);
13578
13579 const APFloat K1Val(0x1p-32f);
13580 const SDValue K1 = DAG.getConstantFP(Val: K1Val, DL: SL, VT: MVT::f32);
13581
13582 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
13583
13584 EVT SetCCVT =
13585 getSetCCResultType(DL: DAG.getDataLayout(), Ctx&: *DAG.getContext(), VT: MVT::f32);
13586
13587 SDValue r2 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: r1, RHS: K0, Cond: ISD::SETOGT);
13588
13589 SDValue r3 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: r2, N2: K1, N3: One, Flags);
13590
13591 r1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: RHS, N2: r3, Flags);
13592
13593 // rcp does not support denormals.
13594 SDValue r0 = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: r1, Flags);
13595
13596 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: LHS, N2: r0, Flags);
13597
13598 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f32, N1: r3, N2: Mul, Flags);
13599}
13600
13601// Returns immediate value for setting the F32 denorm mode when using the
13602// S_DENORM_MODE instruction.
13603static SDValue getSPDenormModeValue(uint32_t SPDenormMode, SelectionDAG &DAG,
13604 const SIMachineFunctionInfo *Info,
13605 const GCNSubtarget *ST) {
13606 assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE");
13607 uint32_t DPDenormModeDefault = Info->getMode().fpDenormModeDPValue();
13608 uint32_t Mode = SPDenormMode | (DPDenormModeDefault << 2);
13609 return DAG.getTargetConstant(Val: Mode, DL: SDLoc(), VT: MVT::i32);
13610}
13611
13612SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const {
13613 if (SDValue FastLowered = lowerFastUnsafeFDIV(Op, DAG))
13614 return FastLowered;
13615
13616 // The selection matcher assumes anything with a chain selecting to a
13617 // mayRaiseFPException machine instruction. Since we're introducing a chain
13618 // here, we need to explicitly report nofpexcept for the regular fdiv
13619 // lowering.
13620 SDNodeFlags Flags = Op->getFlags();
13621 Flags.setNoFPExcept(true);
13622
13623 SDLoc SL(Op);
13624 SDValue LHS = Op.getOperand(i: 0);
13625 SDValue RHS = Op.getOperand(i: 1);
13626
13627 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f32);
13628
13629 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f32, VT2: MVT::i1);
13630
13631 SDValue DenominatorScaled =
13632 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {RHS, RHS, LHS}, Flags);
13633 SDValue NumeratorScaled =
13634 DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, Ops: {LHS, RHS, LHS}, Flags);
13635
13636 // Denominator is scaled to not be denormal, so using rcp is ok.
13637 SDValue ApproxRcp =
13638 DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
13639 SDValue NegDivScale0 =
13640 DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: DenominatorScaled, Flags);
13641
13642 using namespace AMDGPU::Hwreg;
13643 const unsigned Denorm32Reg = HwregEncoding::encode(Values: ID_MODE, Values: 4, Values: 2);
13644 const SDValue BitField = DAG.getTargetConstant(Val: Denorm32Reg, DL: SL, VT: MVT::i32);
13645
13646 const MachineFunction &MF = DAG.getMachineFunction();
13647 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
13648 const DenormalMode DenormMode = Info->getMode().FP32Denormals;
13649
13650 const bool PreservesDenormals = DenormMode == DenormalMode::getIEEE();
13651 const bool HasDynamicDenormals =
13652 (DenormMode.Input == DenormalMode::Dynamic) ||
13653 (DenormMode.Output == DenormalMode::Dynamic);
13654
13655 SDValue SavedDenormMode;
13656
13657 if (!PreservesDenormals) {
13658 // Note we can't use the STRICT_FMA/STRICT_FMUL for the non-strict FDIV
13659 // lowering. The chain dependence is insufficient, and we need glue. We do
13660 // not need the glue variants in a strictfp function.
13661
13662 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
13663
13664 SDValue Glue = DAG.getEntryNode();
13665 if (HasDynamicDenormals) {
13666 SDNode *GetReg = DAG.getMachineNode(Opcode: AMDGPU::S_GETREG_B32, dl: SL,
13667 VTs: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Glue),
13668 Ops: {BitField, Glue});
13669 SavedDenormMode = SDValue(GetReg, 0);
13670
13671 Glue = DAG.getMergeValues(
13672 Ops: {DAG.getEntryNode(), SDValue(GetReg, 0), SDValue(GetReg, 1)}, dl: SL);
13673 }
13674
13675 SDNode *EnableDenorm;
13676 if (Subtarget->hasDenormModeInst()) {
13677 const SDValue EnableDenormValue =
13678 getSPDenormModeValue(FP_DENORM_FLUSH_NONE, DAG, Info, ST: Subtarget);
13679
13680 EnableDenorm = DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs, N1: Glue,
13681 N2: EnableDenormValue)
13682 .getNode();
13683 } else {
13684 const SDValue EnableDenormValue =
13685 DAG.getConstant(FP_DENORM_FLUSH_NONE, DL: SL, VT: MVT::i32);
13686 EnableDenorm = DAG.getMachineNode(Opcode: AMDGPU::S_SETREG_B32, dl: SL, VTs: BindParamVTs,
13687 Ops: {EnableDenormValue, BitField, Glue});
13688 }
13689
13690 SDValue Ops[3] = {NegDivScale0, SDValue(EnableDenorm, 0),
13691 SDValue(EnableDenorm, 1)};
13692
13693 NegDivScale0 = DAG.getMergeValues(Ops, dl: SL);
13694 }
13695
13696 SDValue Fma0 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0,
13697 B: ApproxRcp, C: One, GlueChain: NegDivScale0, Flags);
13698
13699 SDValue Fma1 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma0, B: ApproxRcp,
13700 C: ApproxRcp, GlueChain: Fma0, Flags);
13701
13702 SDValue Mul = getFPBinOp(DAG, Opcode: ISD::FMUL, SL, VT: MVT::f32, A: NumeratorScaled, B: Fma1,
13703 GlueChain: Fma1, Flags);
13704
13705 SDValue Fma2 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Mul,
13706 C: NumeratorScaled, GlueChain: Mul, Flags);
13707
13708 SDValue Fma3 =
13709 getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: Fma2, B: Fma1, C: Mul, GlueChain: Fma2, Flags);
13710
13711 SDValue Fma4 = getFPTernOp(DAG, Opcode: ISD::FMA, SL, VT: MVT::f32, A: NegDivScale0, B: Fma3,
13712 C: NumeratorScaled, GlueChain: Fma3, Flags);
13713
13714 if (!PreservesDenormals) {
13715 SDNode *DisableDenorm;
13716 if (!HasDynamicDenormals && Subtarget->hasDenormModeInst()) {
13717 const SDValue DisableDenormValue = getSPDenormModeValue(
13718 FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, Info, ST: Subtarget);
13719
13720 SDVTList BindParamVTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
13721 DisableDenorm =
13722 DAG.getNode(Opcode: AMDGPUISD::DENORM_MODE, DL: SL, VTList: BindParamVTs,
13723 N1: Fma4.getValue(R: 1), N2: DisableDenormValue, N3: Fma4.getValue(R: 2))
13724 .getNode();
13725 } else {
13726 assert(HasDynamicDenormals == (bool)SavedDenormMode);
13727 const SDValue DisableDenormValue =
13728 HasDynamicDenormals
13729 ? SavedDenormMode
13730 : DAG.getConstant(FP_DENORM_FLUSH_IN_FLUSH_OUT, DL: SL, VT: MVT::i32);
13731
13732 DisableDenorm = DAG.getMachineNode(
13733 Opcode: AMDGPU::S_SETREG_B32, dl: SL, VT: MVT::Other,
13734 Ops: {DisableDenormValue, BitField, Fma4.getValue(R: 1), Fma4.getValue(R: 2)});
13735 }
13736
13737 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
13738 N1: SDValue(DisableDenorm, 0), N2: DAG.getRoot());
13739 DAG.setRoot(OutputChain);
13740 }
13741
13742 SDValue Scale = NumeratorScaled.getValue(R: 1);
13743 SDValue Fmas = DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f32,
13744 Ops: {Fma4, Fma1, Fma3, Scale}, Flags);
13745
13746 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f32, N1: Fmas, N2: RHS, N3: LHS, Flags);
13747}
13748
13749SDValue SITargetLowering::LowerFDIV64(SDValue Op, SelectionDAG &DAG) const {
13750 if (SDValue FastLowered = lowerFastUnsafeFDIV64(Op, DAG))
13751 return FastLowered;
13752
13753 SDLoc SL(Op);
13754 SDValue X = Op.getOperand(i: 0);
13755 SDValue Y = Op.getOperand(i: 1);
13756
13757 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
13758
13759 SDVTList ScaleVT = DAG.getVTList(VT1: MVT::f64, VT2: MVT::i1);
13760
13761 SDValue DivScale0 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: Y, N2: Y, N3: X);
13762
13763 SDValue NegDivScale0 = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f64, Operand: DivScale0);
13764
13765 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT: MVT::f64, Operand: DivScale0);
13766
13767 SDValue Fma0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Rcp, N3: One);
13768
13769 SDValue Fma1 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Rcp, N2: Fma0, N3: Rcp);
13770
13771 SDValue Fma2 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Fma1, N3: One);
13772
13773 SDValue DivScale1 = DAG.getNode(Opcode: AMDGPUISD::DIV_SCALE, DL: SL, VTList: ScaleVT, N1: X, N2: Y, N3: X);
13774
13775 SDValue Fma3 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: Fma1, N2: Fma2, N3: Fma1);
13776 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: MVT::f64, N1: DivScale1, N2: Fma3);
13777
13778 SDValue Fma4 =
13779 DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: MVT::f64, N1: NegDivScale0, N2: Mul, N3: DivScale1);
13780
13781 SDValue Scale;
13782
13783 if (!Subtarget->hasUsableDivScaleConditionOutput()) {
13784 // Workaround a hardware bug on SI where the condition output from div_scale
13785 // is not usable.
13786
13787 const SDValue Hi = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
13788
13789 // Figure out if the scale to use for div_fmas.
13790 SDValue NumBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: X);
13791 SDValue DenBC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Y);
13792 SDValue Scale0BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale0);
13793 SDValue Scale1BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: DivScale1);
13794
13795 SDValue NumHi =
13796 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: NumBC, N2: Hi);
13797 SDValue DenHi =
13798 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: DenBC, N2: Hi);
13799
13800 SDValue Scale0Hi =
13801 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale0BC, N2: Hi);
13802 SDValue Scale1Hi =
13803 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Scale1BC, N2: Hi);
13804
13805 SDValue CmpDen = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: DenHi, RHS: Scale0Hi, Cond: ISD::SETEQ);
13806 SDValue CmpNum = DAG.getSetCC(DL: SL, VT: MVT::i1, LHS: NumHi, RHS: Scale1Hi, Cond: ISD::SETEQ);
13807 Scale = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: CmpNum, N2: CmpDen);
13808 } else {
13809 Scale = DivScale1.getValue(R: 1);
13810 }
13811
13812 SDValue Fmas =
13813 DAG.getNode(Opcode: AMDGPUISD::DIV_FMAS, DL: SL, VT: MVT::f64, N1: Fma4, N2: Fma3, N3: Mul, N4: Scale);
13814
13815 return DAG.getNode(Opcode: AMDGPUISD::DIV_FIXUP, DL: SL, VT: MVT::f64, N1: Fmas, N2: Y, N3: X);
13816}
13817
13818SDValue SITargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const {
13819 EVT VT = Op.getValueType();
13820
13821 if (VT == MVT::f32)
13822 return LowerFDIV32(Op, DAG);
13823
13824 if (VT == MVT::f64)
13825 return LowerFDIV64(Op, DAG);
13826
13827 if (VT == MVT::f16 || VT == MVT::bf16)
13828 return LowerFDIV16(Op, DAG);
13829
13830 llvm_unreachable("Unexpected type for fdiv");
13831}
13832
13833SDValue SITargetLowering::LowerFFREXP(SDValue Op, SelectionDAG &DAG) const {
13834 SDLoc dl(Op);
13835 SDValue Val = Op.getOperand(i: 0);
13836 EVT VT = Val.getValueType();
13837 EVT ResultExpVT = Op->getValueType(ResNo: 1);
13838 EVT InstrExpVT = VT == MVT::f16 ? MVT::i16 : MVT::i32;
13839
13840 SDValue Mant = DAG.getNode(
13841 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT,
13842 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_mant, DL: dl, VT: MVT::i32), N2: Val);
13843
13844 SDValue Exp = DAG.getNode(
13845 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: InstrExpVT,
13846 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_frexp_exp, DL: dl, VT: MVT::i32), N2: Val);
13847
13848 if (Subtarget->hasFractBug()) {
13849 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val);
13850 SDValue Inf =
13851 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
13852
13853 SDValue IsFinite = DAG.getSetCC(DL: dl, VT: MVT::i1, LHS: Fabs, RHS: Inf, Cond: ISD::SETOLT);
13854 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: InstrExpVT);
13855 Exp = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: InstrExpVT, N1: IsFinite, N2: Exp, N3: Zero);
13856 Mant = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: IsFinite, N2: Mant, N3: Val);
13857 }
13858
13859 SDValue CastExp = DAG.getSExtOrTrunc(Op: Exp, DL: dl, VT: ResultExpVT);
13860 return DAG.getMergeValues(Ops: {Mant, CastExp}, dl);
13861}
13862
13863SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
13864 SDLoc DL(Op);
13865 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
13866 EVT VT = Store->getMemoryVT();
13867
13868 if (VT == MVT::i1) {
13869 return DAG.getTruncStore(
13870 Chain: Store->getChain(), dl: DL,
13871 Val: DAG.getSExtOrTrunc(Op: Store->getValue(), DL, VT: MVT::i32),
13872 Ptr: Store->getBasePtr(), SVT: MVT::i1, MMO: Store->getMemOperand());
13873 }
13874
13875 assert(VT.isVector() &&
13876 Store->getValue().getValueType().getScalarType() == MVT::i32);
13877
13878 unsigned AS = Store->getAddressSpace();
13879 if (Subtarget->hasLDSMisalignedBugInWGPMode() &&
13880 AS == AMDGPUAS::FLAT_ADDRESS &&
13881 Store->getAlign().value() < VT.getStoreSize() &&
13882 VT.getSizeInBits() > 32) {
13883 return SplitVectorStore(Op, DAG);
13884 }
13885
13886 MachineFunction &MF = DAG.getMachineFunction();
13887 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
13888 // If there is a possibility that flat instruction access scratch memory
13889 // then we need to use the same legalization rules we use for private.
13890 if (AS == AMDGPUAS::FLAT_ADDRESS &&
13891 !Subtarget->hasMultiDwordFlatScratchAddressing())
13892 AS = addressMayBeAccessedAsPrivate(MMO: Store->getMemOperand(), Info: *MFI)
13893 ? AMDGPUAS::PRIVATE_ADDRESS
13894 : AMDGPUAS::GLOBAL_ADDRESS;
13895
13896 unsigned NumElements = VT.getVectorNumElements();
13897 if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS) {
13898 if (NumElements > 4)
13899 return SplitVectorStore(Op, DAG);
13900 // v3 stores not supported on SI.
13901 if (NumElements == 3 && !Subtarget->hasDwordx3LoadStores())
13902 return SplitVectorStore(Op, DAG);
13903
13904 if (!allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
13905 VT, MMO: *Store->getMemOperand()))
13906 return expandUnalignedStore(ST: Store, DAG);
13907
13908 return SDValue();
13909 }
13910 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
13911 switch (Subtarget->getMaxPrivateElementSize()) {
13912 case 4:
13913 return scalarizeVectorStore(ST: Store, DAG);
13914 case 8:
13915 if (NumElements > 2)
13916 return SplitVectorStore(Op, DAG);
13917 return SDValue();
13918 case 16:
13919 if (NumElements > 4 ||
13920 (NumElements == 3 && !Subtarget->hasFlatScratchEnabled()))
13921 return SplitVectorStore(Op, DAG);
13922 return SDValue();
13923 default:
13924 llvm_unreachable("unsupported private_element_size");
13925 }
13926 } else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
13927 unsigned Fast = 0;
13928 auto Flags = Store->getMemOperand()->getFlags();
13929 if (allowsMisalignedMemoryAccessesImpl(Size: VT.getSizeInBits(), AddrSpace: AS,
13930 Alignment: Store->getAlign(), Flags, IsFast: &Fast) &&
13931 Fast > 1)
13932 return SDValue();
13933
13934 if (VT.isVector())
13935 return SplitVectorStore(Op, DAG);
13936
13937 return expandUnalignedStore(ST: Store, DAG);
13938 }
13939
13940 // Probably an invalid store. If so we'll end up emitting a selection error.
13941 return SDValue();
13942}
13943
13944// Avoid the full correct expansion for f32 sqrt when promoting from f16.
13945SDValue SITargetLowering::lowerFSQRTF16(SDValue Op, SelectionDAG &DAG) const {
13946 SDLoc SL(Op);
13947 assert(!Subtarget->has16BitInsts());
13948 SDNodeFlags Flags = Op->getFlags();
13949 SDValue Ext =
13950 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Op.getOperand(i: 0), Flags);
13951
13952 SDValue SqrtID = DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL: SL, VT: MVT::i32);
13953 SDValue Sqrt =
13954 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::f32, N1: SqrtID, N2: Ext, Flags);
13955
13956 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: MVT::f16, N1: Sqrt,
13957 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
13958}
13959
13960SDValue SITargetLowering::lowerFSQRTF32(SDValue Op, SelectionDAG &DAG) const {
13961 SDLoc DL(Op);
13962 SDNodeFlags Flags = Op->getFlags();
13963 MVT VT = Op.getValueType().getSimpleVT();
13964 const SDValue X = Op.getOperand(i: 0);
13965
13966 if (allowApproxFunc(DAG, Flags)) {
13967 // Instruction is 1ulp but ignores denormals.
13968 return DAG.getNode(
13969 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT,
13970 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32), N2: X, Flags);
13971 }
13972
13973 SDValue ScaleThreshold = DAG.getConstantFP(Val: 0x1.0p-96f, DL, VT);
13974 SDValue NeedScale = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleThreshold, Cond: ISD::SETOLT);
13975
13976 SDValue ScaleUpFactor = DAG.getConstantFP(Val: 0x1.0p+32f, DL, VT);
13977
13978 SDValue ScaledX = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: ScaleUpFactor, Flags);
13979
13980 SDValue SqrtX =
13981 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledX, N3: X, Flags);
13982
13983 SDValue SqrtS;
13984 if (needsDenormHandlingF32(DAG, Src: X, Flags)) {
13985 SDValue SqrtID =
13986 DAG.getTargetConstant(Val: Intrinsic::amdgcn_sqrt, DL, VT: MVT::i32);
13987 SqrtS = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: SqrtID, N2: SqrtX, Flags);
13988
13989 SDValue SqrtSAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: SqrtS);
13990 SDValue SqrtSNextDownInt =
13991 DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
13992 N2: DAG.getAllOnesConstant(DL, VT: MVT::i32));
13993 SDValue SqrtSNextDown = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextDownInt);
13994
13995 SDValue NegSqrtSNextDown =
13996 DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextDown, Flags);
13997
13998 SDValue SqrtVP =
13999 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextDown, N2: SqrtS, N3: SqrtX, Flags);
14000
14001 SDValue SqrtSNextUpInt = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: SqrtSAsInt,
14002 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
14003 SDValue SqrtSNextUp = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SqrtSNextUpInt);
14004
14005 SDValue NegSqrtSNextUp = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtSNextUp, Flags);
14006 SDValue SqrtVS =
14007 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtSNextUp, N2: SqrtS, N3: SqrtX, Flags);
14008
14009 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
14010 SDValue SqrtVPLE0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVP, RHS: Zero, Cond: ISD::SETOLE);
14011
14012 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPLE0, N2: SqrtSNextDown, N3: SqrtS,
14013 Flags);
14014
14015 SDValue SqrtVPVSGT0 = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtVS, RHS: Zero, Cond: ISD::SETOGT);
14016 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: SqrtVPVSGT0, N2: SqrtSNextUp, N3: SqrtS,
14017 Flags);
14018 } else {
14019 SDValue SqrtR = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT, Operand: SqrtX, Flags);
14020
14021 SqrtS = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtX, N2: SqrtR, Flags);
14022
14023 SDValue Half = DAG.getConstantFP(Val: 0.5f, DL, VT);
14024 SDValue SqrtH = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtR, N2: Half, Flags);
14025 SDValue NegSqrtH = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtH, Flags);
14026
14027 SDValue SqrtE = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtH, N2: SqrtS, N3: Half, Flags);
14028 SqrtH = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtH, N2: SqrtE, N3: SqrtH, Flags);
14029 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtS, N2: SqrtE, N3: SqrtS, Flags);
14030
14031 SDValue NegSqrtS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: SqrtS, Flags);
14032 SDValue SqrtD =
14033 DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegSqrtS, N2: SqrtS, N3: SqrtX, Flags);
14034 SqrtS = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: SqrtD, N2: SqrtH, N3: SqrtS, Flags);
14035 }
14036
14037 SDValue ScaleDownFactor = DAG.getConstantFP(Val: 0x1.0p-16f, DL, VT);
14038
14039 SDValue ScaledDown =
14040 DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: SqrtS, N2: ScaleDownFactor, Flags);
14041
14042 SqrtS = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: NeedScale, N2: ScaledDown, N3: SqrtS, Flags);
14043 SDValue IsZeroOrInf =
14044 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14045 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14046
14047 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtS, Flags);
14048}
14049
14050SDValue SITargetLowering::lowerFSQRTF64(SDValue Op, SelectionDAG &DAG) const {
14051 // For double type, the SQRT and RSQ instructions don't have required
14052 // precision, we apply Goldschmidt's algorithm to improve the result:
14053 //
14054 // y0 = rsq(x)
14055 // g0 = x * y0
14056 // h0 = 0.5 * y0
14057 //
14058 // r0 = 0.5 - h0 * g0
14059 // g1 = g0 * r0 + g0
14060 // h1 = h0 * r0 + h0
14061 //
14062 // r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
14063 // g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
14064 // h2 = h1 * r1 + h1
14065 //
14066 // r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
14067 // g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
14068 //
14069 // sqrt(x) = g3
14070
14071 SDNodeFlags Flags = Op->getFlags();
14072
14073 SDLoc DL(Op);
14074
14075 SDValue X = Op.getOperand(i: 0);
14076 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
14077
14078 SDValue SqrtX = X;
14079 SDValue Scaling;
14080 if (!Flags.hasApproximateFuncs()) {
14081 SDValue ScaleConstant = DAG.getConstantFP(Val: 0x1.0p-767, DL, VT: MVT::f64);
14082 Scaling = DAG.getSetCC(DL, VT: MVT::i1, LHS: X, RHS: ScaleConstant, Cond: ISD::SETOLT);
14083
14084 // Scale up input if it is too small.
14085 SDValue ScaleUpFactor = DAG.getConstant(Val: 256, DL, VT: MVT::i32);
14086 SDValue ScaleUp =
14087 DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling, N2: ScaleUpFactor, N3: ZeroInt);
14088 SqrtX = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: X, N2: ScaleUp, Flags);
14089 }
14090
14091 SDValue SqrtY = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL, VT: MVT::f64, Operand: SqrtX);
14092
14093 SDValue SqrtS0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtX, N2: SqrtY);
14094
14095 SDValue Half = DAG.getConstantFP(Val: 0.5, DL, VT: MVT::f64);
14096 SDValue SqrtH0 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: SqrtY, N2: Half);
14097
14098 SDValue NegSqrtH0 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtH0);
14099 SDValue SqrtR0 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtH0, N2: SqrtS0, N3: Half);
14100
14101 SDValue SqrtH1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtH0, N2: SqrtR0, N3: SqrtH0);
14102
14103 SDValue SqrtS1 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtS0, N2: SqrtR0, N3: SqrtS0);
14104
14105 SDValue NegSqrtS1 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS1);
14106 SDValue SqrtD0 =
14107 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS1, N2: SqrtS1, N3: SqrtX);
14108
14109 SDValue SqrtS2 = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD0, N2: SqrtH1, N3: SqrtS1);
14110
14111 SDValue SqrtRet = SqrtS2;
14112 if (!Flags.hasApproximateFuncs()) {
14113 SDValue NegSqrtS2 = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: SqrtS2);
14114 SDValue SqrtD1 =
14115 DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegSqrtS2, N2: SqrtS2, N3: SqrtX);
14116
14117 SqrtRet = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: SqrtD1, N2: SqrtH1, N3: SqrtS2);
14118
14119 SDValue ScaleDownFactor = DAG.getSignedConstant(Val: -128, DL, VT: MVT::i32);
14120 SDValue ScaleDown = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::i32, N1: Scaling,
14121 N2: ScaleDownFactor, N3: ZeroInt);
14122 SqrtRet = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: SqrtRet, N2: ScaleDown, Flags);
14123 }
14124
14125 // TODO: Check for DAZ and expand to subnormals
14126
14127 SDValue IsZeroOrInf;
14128 if (Flags.hasNoInfs()) {
14129 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
14130 IsZeroOrInf = DAG.getSetCC(DL, VT: MVT::i1, LHS: SqrtX, RHS: Zero, Cond: ISD::SETOEQ);
14131 } else {
14132 IsZeroOrInf =
14133 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: MVT::i1, N1: SqrtX,
14134 N2: DAG.getTargetConstant(Val: fcZero | fcPosInf, DL, VT: MVT::i32));
14135 }
14136
14137 // If x is +INF, +0, or -0, use its original value
14138 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f64, N1: IsZeroOrInf, N2: SqrtX, N3: SqrtRet,
14139 Flags);
14140}
14141
14142SDValue SITargetLowering::LowerTrig(SDValue Op, SelectionDAG &DAG) const {
14143 SDLoc DL(Op);
14144 EVT VT = Op.getValueType();
14145 SDValue Arg = Op.getOperand(i: 0);
14146 SDValue TrigVal;
14147
14148 // Propagate fast-math flags so that the multiply we introduce can be folded
14149 // if Arg is already the result of a multiply by constant.
14150 auto Flags = Op->getFlags();
14151
14152 // AMDGPUISD nodes of vector type must be unrolled here since
14153 // they will not be expanded elsewhere.
14154 auto UnrollIfVec = [&DAG](SDValue V) -> SDValue {
14155 if (!V.getValueType().isVector())
14156 return V;
14157
14158 return DAG.UnrollVectorOp(N: cast<SDNode>(Val&: V));
14159 };
14160
14161 SDValue OneOver2Pi = DAG.getConstantFP(Val: 0.5 * numbers::inv_pi, DL, VT);
14162
14163 if (Subtarget->hasTrigReducedRange()) {
14164 SDValue MulVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14165 TrigVal = UnrollIfVec(DAG.getNode(Opcode: AMDGPUISD::FRACT, DL, VT, Operand: MulVal, Flags));
14166 } else {
14167 TrigVal = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: OneOver2Pi, Flags);
14168 }
14169
14170 switch (Op.getOpcode()) {
14171 case ISD::FCOS:
14172 TrigVal = DAG.getNode(Opcode: AMDGPUISD::COS_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14173 break;
14174 case ISD::FSIN:
14175 TrigVal = DAG.getNode(Opcode: AMDGPUISD::SIN_HW, DL: SDLoc(Op), VT, Operand: TrigVal, Flags);
14176 break;
14177 default:
14178 llvm_unreachable("Wrong trig opcode");
14179 }
14180
14181 return UnrollIfVec(TrigVal);
14182}
14183
14184SDValue SITargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
14185 SelectionDAG &DAG) const {
14186 AtomicSDNode *AtomicNode = cast<AtomicSDNode>(Val&: Op);
14187 assert(AtomicNode->isCompareAndSwap());
14188 unsigned AS = AtomicNode->getAddressSpace();
14189
14190 // No custom lowering required for local address space
14191 if (!AMDGPU::isFlatGlobalAddrSpace(AS))
14192 return Op;
14193
14194 // Non-local address space requires custom lowering for atomic compare
14195 // and swap; cmp and swap should be in a v2i32 or v2i64 in case of _X2
14196 SDLoc DL(Op);
14197 SDValue ChainIn = Op.getOperand(i: 0);
14198 SDValue Addr = Op.getOperand(i: 1);
14199 SDValue Old = Op.getOperand(i: 2);
14200 SDValue New = Op.getOperand(i: 3);
14201 EVT VT = Op.getValueType();
14202 MVT SimpleVT = VT.getSimpleVT();
14203 MVT VecType = MVT::getVectorVT(VT: SimpleVT, NumElements: 2);
14204
14205 SDValue NewOld = DAG.getBuildVector(VT: VecType, DL, Ops: {New, Old});
14206 SDValue Ops[] = {ChainIn, Addr, NewOld};
14207
14208 return DAG.getMemIntrinsicNode(Opcode: AMDGPUISD::ATOMIC_CMP_SWAP, dl: DL,
14209 VTList: Op->getVTList(), Ops, MemVT: VT,
14210 MMO: AtomicNode->getMemOperand());
14211}
14212
14213//===----------------------------------------------------------------------===//
14214// Custom DAG optimizations
14215//===----------------------------------------------------------------------===//
14216
14217SDValue
14218SITargetLowering::performUCharToFloatCombine(SDNode *N,
14219 DAGCombinerInfo &DCI) const {
14220 EVT VT = N->getValueType(ResNo: 0);
14221 EVT ScalarVT = VT.getScalarType();
14222 if (ScalarVT != MVT::f32 && ScalarVT != MVT::f16)
14223 return SDValue();
14224
14225 SelectionDAG &DAG = DCI.DAG;
14226 SDLoc DL(N);
14227
14228 SDValue Src = N->getOperand(Num: 0);
14229 EVT SrcVT = Src.getValueType();
14230
14231 // TODO: We could try to match extracting the higher bytes, which would be
14232 // easier if i8 vectors weren't promoted to i32 vectors, particularly after
14233 // types are legalized. v4i8 -> v4f32 is probably the only case to worry
14234 // about in practice.
14235 if (DCI.isAfterLegalizeDAG() && SrcVT == MVT::i32) {
14236 if (DAG.MaskedValueIsZero(Op: Src, Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 24))) {
14237 SDValue Cvt = DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0, DL, VT: MVT::f32, Operand: Src);
14238 DCI.AddToWorklist(N: Cvt.getNode());
14239
14240 // For the f16 case, fold to a cast to f32 and then cast back to f16.
14241 if (ScalarVT != MVT::f32) {
14242 Cvt = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Cvt,
14243 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
14244 }
14245 return Cvt;
14246 }
14247 }
14248
14249 return SDValue();
14250}
14251
14252SDValue SITargetLowering::performFCopySignCombine(SDNode *N,
14253 DAGCombinerInfo &DCI) const {
14254 SDValue MagnitudeOp = N->getOperand(Num: 0);
14255 SDValue SignOp = N->getOperand(Num: 1);
14256
14257 // The generic combine for fcopysign + fp cast is too conservative with
14258 // vectors, and also gets confused by the splitting we will perform here, so
14259 // peek through FP casts.
14260 if (SignOp.getOpcode() == ISD::FP_EXTEND ||
14261 SignOp.getOpcode() == ISD::FP_ROUND)
14262 SignOp = SignOp.getOperand(i: 0);
14263
14264 SelectionDAG &DAG = DCI.DAG;
14265 SDLoc DL(N);
14266 EVT SignVT = SignOp.getValueType();
14267
14268 // f64 fcopysign is really an f32 copysign on the high bits, so replace the
14269 // lower half with a copy.
14270 // fcopysign f64:x, _:y -> x.lo32, (fcopysign (f32 x.hi32), _:y)
14271 EVT MagVT = MagnitudeOp.getValueType();
14272
14273 unsigned NumElts = MagVT.isVector() ? MagVT.getVectorNumElements() : 1;
14274
14275 if (MagVT.getScalarType() == MVT::f64) {
14276 EVT F32VT = MagVT.isVector()
14277 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14278 : MVT::v2f32;
14279
14280 SDValue MagAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: MagnitudeOp);
14281
14282 SmallVector<SDValue, 8> NewElts;
14283 for (unsigned I = 0; I != NumElts; ++I) {
14284 SDValue MagLo =
14285 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14286 N2: DAG.getConstant(Val: 2 * I, DL, VT: MVT::i32));
14287 SDValue MagHi =
14288 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: MagAsVector,
14289 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14290
14291 SDValue SignOpElt =
14292 MagVT.isVector()
14293 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SignVT.getScalarType(),
14294 N1: SignOp, N2: DAG.getConstant(Val: I, DL, VT: MVT::i32))
14295 : SignOp;
14296
14297 SDValue HiOp =
14298 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: MVT::f32, N1: MagHi, N2: SignOpElt);
14299
14300 SDValue Vector =
14301 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2f32, N1: MagLo, N2: HiOp);
14302
14303 SDValue NewElt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Vector);
14304 NewElts.push_back(Elt: NewElt);
14305 }
14306
14307 if (NewElts.size() == 1)
14308 return NewElts[0];
14309
14310 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MagVT, Ops: NewElts);
14311 }
14312
14313 if (SignVT.getScalarType() != MVT::f64)
14314 return SDValue();
14315
14316 // Reduce width of sign operand, we only need the highest bit.
14317 //
14318 // fcopysign f64:x, f64:y ->
14319 // fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1)
14320 // TODO: In some cases it might make sense to go all the way to f16.
14321
14322 EVT F32VT = MagVT.isVector()
14323 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: 2 * NumElts)
14324 : MVT::v2f32;
14325
14326 SDValue SignAsVector = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: F32VT, Operand: SignOp);
14327
14328 SmallVector<SDValue, 8> F32Signs;
14329 for (unsigned I = 0; I != NumElts; ++I) {
14330 // Take sign from odd elements of cast vector
14331 SDValue SignAsF32 =
14332 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::f32, N1: SignAsVector,
14333 N2: DAG.getConstant(Val: 2 * I + 1, DL, VT: MVT::i32));
14334 F32Signs.push_back(Elt: SignAsF32);
14335 }
14336
14337 SDValue NewSign =
14338 NumElts == 1
14339 ? F32Signs.back()
14340 : DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL,
14341 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::f32, NumElements: NumElts),
14342 Ops: F32Signs);
14343
14344 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
14345 N2: NewSign);
14346}
14347
14348// (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2)
14349// (shl (or x, c1), c2) -> add (shl x, c2), (shl c1, c2) iff x and c1 share no
14350// bits
14351
14352// This is a variant of
14353// (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2),
14354//
14355// The normal DAG combiner will do this, but only if the add has one use since
14356// that would increase the number of instructions.
14357//
14358// This prevents us from seeing a constant offset that can be folded into a
14359// memory instruction's addressing mode. If we know the resulting add offset of
14360// a pointer can be folded into an addressing offset, we can replace the pointer
14361// operand with the add of new constant offset. This eliminates one of the uses,
14362// and may allow the remaining use to also be simplified.
14363//
14364SDValue SITargetLowering::performSHLPtrCombine(SDNode *N, unsigned AddrSpace,
14365 EVT MemVT,
14366 DAGCombinerInfo &DCI) const {
14367 SDValue N0 = N->getOperand(Num: 0);
14368 SDValue N1 = N->getOperand(Num: 1);
14369
14370 // We only do this to handle cases where it's profitable when there are
14371 // multiple uses of the add, so defer to the standard combine.
14372 if ((!N0->isAnyAdd() && N0.getOpcode() != ISD::OR) || N0->hasOneUse())
14373 return SDValue();
14374
14375 const ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val&: N1);
14376 if (!CN1)
14377 return SDValue();
14378
14379 const ConstantSDNode *CAdd = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
14380 if (!CAdd)
14381 return SDValue();
14382
14383 SelectionDAG &DAG = DCI.DAG;
14384
14385 if (N0->getOpcode() == ISD::OR &&
14386 !DAG.haveNoCommonBitsSet(A: N0.getOperand(i: 0), B: N0.getOperand(i: 1)))
14387 return SDValue();
14388
14389 // If the resulting offset is too large, we can't fold it into the
14390 // addressing mode offset.
14391 APInt Offset = CAdd->getAPIntValue() << CN1->getAPIntValue();
14392 Type *Ty = MemVT.getTypeForEVT(Context&: *DCI.DAG.getContext());
14393
14394 AddrMode AM;
14395 AM.HasBaseReg = true;
14396 AM.BaseOffs = Offset.getSExtValue();
14397 if (!isLegalAddressingMode(DL: DCI.DAG.getDataLayout(), AM, Ty, AS: AddrSpace))
14398 return SDValue();
14399
14400 SDLoc SL(N);
14401 EVT VT = N->getValueType(ResNo: 0);
14402
14403 SDValue ShlX = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT, N1: N0.getOperand(i: 0), N2: N1);
14404 SDValue COffset = DAG.getConstant(Val: Offset, DL: SL, VT);
14405
14406 SDNodeFlags Flags;
14407 Flags.setNoUnsignedWrap(
14408 N->getFlags().hasNoUnsignedWrap() &&
14409 (N0.getOpcode() == ISD::OR || N0->getFlags().hasNoUnsignedWrap()));
14410
14411 // Use ISD::ADD even if the original operation was ISD::PTRADD, since we can't
14412 // be sure that the new left operand is a proper base pointer.
14413 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT, N1: ShlX, N2: COffset, Flags);
14414}
14415
14416/// MemSDNode::getBasePtr() does not work for intrinsics, which needs to offset
14417/// by the chain and intrinsic ID. Theoretically we would also need to check the
14418/// specific intrinsic, but they all place the pointer operand first.
14419static unsigned getBasePtrIndex(const MemSDNode *N) {
14420 switch (N->getOpcode()) {
14421 case ISD::STORE:
14422 case ISD::INTRINSIC_W_CHAIN:
14423 case ISD::INTRINSIC_VOID:
14424 return 2;
14425 default:
14426 return 1;
14427 }
14428}
14429
14430SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N,
14431 DAGCombinerInfo &DCI) const {
14432 SelectionDAG &DAG = DCI.DAG;
14433
14434 unsigned PtrIdx = getBasePtrIndex(N);
14435 SDValue Ptr = N->getOperand(Num: PtrIdx);
14436
14437 // TODO: We could also do this for multiplies.
14438 if (Ptr.getOpcode() == ISD::SHL) {
14439 SDValue NewPtr = performSHLPtrCombine(N: Ptr.getNode(), AddrSpace: N->getAddressSpace(),
14440 MemVT: N->getMemoryVT(), DCI);
14441 if (NewPtr) {
14442 SmallVector<SDValue, 8> NewOps(N->ops());
14443
14444 NewOps[PtrIdx] = NewPtr;
14445 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
14446 }
14447 }
14448
14449 return SDValue();
14450}
14451
14452static bool bitOpWithConstantIsReducible(unsigned Opc, uint32_t Val) {
14453 return (Opc == ISD::AND && (Val == 0 || Val == 0xffffffff)) ||
14454 (Opc == ISD::OR && (Val == 0xffffffff || Val == 0)) ||
14455 (Opc == ISD::XOR && Val == 0);
14456}
14457
14458// Break up 64-bit bit operation of a constant into two 32-bit and/or/xor. This
14459// will typically happen anyway for a VALU 64-bit and. This exposes other 32-bit
14460// integer combine opportunities since most 64-bit operations are decomposed
14461// this way. TODO: We won't want this for SALU especially if it is an inline
14462// immediate.
14463SDValue SITargetLowering::splitBinaryBitConstantOp(
14464 DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS,
14465 const ConstantSDNode *CRHS) const {
14466 uint64_t Val = CRHS->getZExtValue();
14467 uint32_t ValLo = Lo_32(Value: Val);
14468 uint32_t ValHi = Hi_32(Value: Val);
14469 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14470
14471 if ((bitOpWithConstantIsReducible(Opc, Val: ValLo) ||
14472 bitOpWithConstantIsReducible(Opc, Val: ValHi)) ||
14473 (CRHS->hasOneUse() && !TII->isInlineConstant(Imm: CRHS->getAPIntValue()))) {
14474 // We have 64-bit scalar and/or/xor, but do not have vector forms.
14475 if (Subtarget->has64BitLiterals() && CRHS->hasOneUse() &&
14476 !CRHS->user_begin()->isDivergent())
14477 return SDValue();
14478
14479 // If we need to materialize a 64-bit immediate, it will be split up later
14480 // anyway. Avoid creating the harder to understand 64-bit immediate
14481 // materialization.
14482 return splitBinaryBitConstantOpImpl(DCI, SL, Opc, LHS, ValLo, ValHi);
14483 }
14484
14485 return SDValue();
14486}
14487
14488bool llvm::isBoolSGPR(SDValue V) {
14489 if (V.getValueType() != MVT::i1)
14490 return false;
14491 switch (V.getOpcode()) {
14492 default:
14493 break;
14494 case ISD::SETCC:
14495 case ISD::IS_FPCLASS:
14496 case AMDGPUISD::FP_CLASS:
14497 return true;
14498 case ISD::AND:
14499 case ISD::OR:
14500 case ISD::XOR:
14501 return isBoolSGPR(V: V.getOperand(i: 0)) && isBoolSGPR(V: V.getOperand(i: 1));
14502 case ISD::SADDO:
14503 case ISD::UADDO:
14504 case ISD::SSUBO:
14505 case ISD::USUBO:
14506 case ISD::SMULO:
14507 case ISD::UMULO:
14508 return V.getResNo() == 1;
14509 case ISD::INTRINSIC_WO_CHAIN: {
14510 unsigned IntrinsicID = V.getConstantOperandVal(i: 0);
14511 switch (IntrinsicID) {
14512 case Intrinsic::amdgcn_is_shared:
14513 case Intrinsic::amdgcn_is_private:
14514 return true;
14515 default:
14516 return false;
14517 }
14518
14519 return false;
14520 }
14521 }
14522 return false;
14523}
14524
14525// If a constant has all zeroes or all ones within each byte return it.
14526// Otherwise return 0.
14527static uint32_t getConstantPermuteMask(uint32_t C) {
14528 // 0xff for any zero byte in the mask
14529 uint32_t ZeroByteMask = 0;
14530 if (!(C & 0x000000ff))
14531 ZeroByteMask |= 0x000000ff;
14532 if (!(C & 0x0000ff00))
14533 ZeroByteMask |= 0x0000ff00;
14534 if (!(C & 0x00ff0000))
14535 ZeroByteMask |= 0x00ff0000;
14536 if (!(C & 0xff000000))
14537 ZeroByteMask |= 0xff000000;
14538 uint32_t NonZeroByteMask = ~ZeroByteMask; // 0xff for any non-zero byte
14539 if ((NonZeroByteMask & C) != NonZeroByteMask)
14540 return 0; // Partial bytes selected.
14541 return C;
14542}
14543
14544// Check if a node selects whole bytes from its operand 0 starting at a byte
14545// boundary while masking the rest. Returns select mask as in the v_perm_b32
14546// or -1 if not succeeded.
14547// Note byte select encoding:
14548// value 0-3 selects corresponding source byte;
14549// value 0xc selects zero;
14550// value 0xff selects 0xff.
14551static uint32_t getPermuteMask(SDValue V) {
14552 assert(V.getValueSizeInBits() == 32);
14553
14554 if (V.getNumOperands() != 2)
14555 return ~0;
14556
14557 ConstantSDNode *N1 = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
14558 if (!N1)
14559 return ~0;
14560
14561 uint32_t C = N1->getZExtValue();
14562
14563 switch (V.getOpcode()) {
14564 default:
14565 break;
14566 case ISD::AND:
14567 if (uint32_t ConstMask = getConstantPermuteMask(C))
14568 return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
14569 break;
14570
14571 case ISD::OR:
14572 if (uint32_t ConstMask = getConstantPermuteMask(C))
14573 return (0x03020100 & ~ConstMask) | ConstMask;
14574 break;
14575
14576 case ISD::SHL:
14577 if (C % 8)
14578 return ~0;
14579
14580 return uint32_t((0x030201000c0c0c0cull << C) >> 32);
14581
14582 case ISD::SRL:
14583 if (C % 8)
14584 return ~0;
14585
14586 return uint32_t(0x0c0c0c0c03020100ull >> C);
14587 }
14588
14589 return ~0;
14590}
14591
14592SDValue SITargetLowering::performAndCombine(SDNode *N,
14593 DAGCombinerInfo &DCI) const {
14594 if (DCI.isBeforeLegalize())
14595 return SDValue();
14596
14597 SelectionDAG &DAG = DCI.DAG;
14598 EVT VT = N->getValueType(ResNo: 0);
14599 SDValue LHS = N->getOperand(Num: 0);
14600 SDValue RHS = N->getOperand(Num: 1);
14601
14602 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
14603 if (VT == MVT::i64 && CRHS) {
14604 if (SDValue Split =
14605 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::AND, LHS, CRHS))
14606 return Split;
14607 }
14608
14609 if (CRHS && VT == MVT::i32) {
14610 // and (srl x, c), mask => shl (bfe x, nb + c, mask >> nb), nb
14611 // nb = number of trailing zeroes in mask
14612 // It can be optimized out using SDWA for GFX8+ in the SDWA peephole pass,
14613 // given that we are selecting 8 or 16 bit fields starting at byte boundary.
14614 uint64_t Mask = CRHS->getZExtValue();
14615 unsigned Bits = llvm::popcount(Value: Mask);
14616 if (getSubtarget()->hasSDWA() && LHS->getOpcode() == ISD::SRL &&
14617 (Bits == 8 || Bits == 16) && isShiftedMask_64(Value: Mask) && !(Mask & 1)) {
14618 if (auto *CShift = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1))) {
14619 unsigned Shift = CShift->getZExtValue();
14620 unsigned NB = CRHS->getAPIntValue().countr_zero();
14621 unsigned Offset = NB + Shift;
14622 if ((Offset & (Bits - 1)) == 0) { // Starts at a byte or word boundary.
14623 SDLoc SL(N);
14624 SDValue BFE =
14625 DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32, N1: LHS->getOperand(Num: 0),
14626 N2: DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32),
14627 N3: DAG.getConstant(Val: Bits, DL: SL, VT: MVT::i32));
14628 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: Bits);
14629 SDValue Ext = DAG.getNode(Opcode: ISD::AssertZext, DL: SL, VT, N1: BFE,
14630 N2: DAG.getValueType(NarrowVT));
14631 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(LHS), VT, N1: Ext,
14632 N2: DAG.getConstant(Val: NB, DL: SDLoc(CRHS), VT: MVT::i32));
14633 return Shl;
14634 }
14635 }
14636 }
14637
14638 // and (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
14639 if (LHS.hasOneUse() && LHS.getOpcode() == AMDGPUISD::PERM &&
14640 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
14641 uint32_t Sel = getConstantPermuteMask(C: Mask);
14642 if (!Sel)
14643 return SDValue();
14644
14645 // Select 0xc for all zero bytes
14646 Sel = (LHS.getConstantOperandVal(i: 2) & Sel) | (~Sel & 0x0c0c0c0c);
14647 SDLoc DL(N);
14648 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
14649 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
14650 }
14651 }
14652
14653 // (and (fcmp ord x, x), (fcmp une (fabs x), inf)) ->
14654 // fp_class x, ~(s_nan | q_nan | n_infinity | p_infinity)
14655 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == ISD::SETCC) {
14656 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
14657 ISD::CondCode RCC = cast<CondCodeSDNode>(Val: RHS.getOperand(i: 2))->get();
14658
14659 SDValue X = LHS.getOperand(i: 0);
14660 SDValue Y = RHS.getOperand(i: 0);
14661 if (Y.getOpcode() != ISD::FABS || Y.getOperand(i: 0) != X ||
14662 !isTypeLegal(VT: X.getValueType()))
14663 return SDValue();
14664
14665 if (LCC == ISD::SETO) {
14666 if (X != LHS.getOperand(i: 1))
14667 return SDValue();
14668
14669 if (RCC == ISD::SETUNE) {
14670 const ConstantFPSDNode *C1 =
14671 dyn_cast<ConstantFPSDNode>(Val: RHS.getOperand(i: 1));
14672 if (!C1 || !C1->isInfinity() || C1->isNegative())
14673 return SDValue();
14674
14675 const uint32_t Mask = SIInstrFlags::N_NORMAL |
14676 SIInstrFlags::N_SUBNORMAL | SIInstrFlags::N_ZERO |
14677 SIInstrFlags::P_ZERO | SIInstrFlags::P_SUBNORMAL |
14678 SIInstrFlags::P_NORMAL;
14679
14680 static_assert(
14681 ((~(SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN |
14682 SIInstrFlags::N_INFINITY | SIInstrFlags::P_INFINITY)) &
14683 0x3ff) == Mask,
14684 "mask not equal");
14685
14686 SDLoc DL(N);
14687 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: X,
14688 N2: DAG.getConstant(Val: Mask, DL, VT: MVT::i32));
14689 }
14690 }
14691 }
14692
14693 if (RHS.getOpcode() == ISD::SETCC && LHS.getOpcode() == AMDGPUISD::FP_CLASS)
14694 std::swap(a&: LHS, b&: RHS);
14695
14696 if (LHS.getOpcode() == ISD::SETCC && RHS.getOpcode() == AMDGPUISD::FP_CLASS &&
14697 RHS.hasOneUse()) {
14698 ISD::CondCode LCC = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
14699 // and (fcmp seto), (fp_class x, mask) -> fp_class x, mask & ~(p_nan |
14700 // n_nan) and (fcmp setuo), (fp_class x, mask) -> fp_class x, mask & (p_nan
14701 // | n_nan)
14702 const ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
14703 if ((LCC == ISD::SETO || LCC == ISD::SETUO) && Mask &&
14704 (RHS.getOperand(i: 0) == LHS.getOperand(i: 0) &&
14705 LHS.getOperand(i: 0) == LHS.getOperand(i: 1))) {
14706 const unsigned OrdMask = SIInstrFlags::S_NAN | SIInstrFlags::Q_NAN;
14707 unsigned NewMask = LCC == ISD::SETO ? Mask->getZExtValue() & ~OrdMask
14708 : Mask->getZExtValue() & OrdMask;
14709
14710 SDLoc DL(N);
14711 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: RHS.getOperand(i: 0),
14712 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
14713 }
14714 }
14715
14716 if (VT == MVT::i32 && (RHS.getOpcode() == ISD::SIGN_EXTEND ||
14717 LHS.getOpcode() == ISD::SIGN_EXTEND)) {
14718 // and x, (sext cc from i1) => select cc, x, 0
14719 if (RHS.getOpcode() != ISD::SIGN_EXTEND)
14720 std::swap(a&: LHS, b&: RHS);
14721 if (isBoolSGPR(V: RHS.getOperand(i: 0)))
14722 return DAG.getSelect(DL: SDLoc(N), VT: MVT::i32, Cond: RHS.getOperand(i: 0), LHS,
14723 RHS: DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32));
14724 }
14725
14726 // and (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
14727 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
14728 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
14729 N->isDivergent() && TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
14730 uint32_t LHSMask = getPermuteMask(V: LHS);
14731 uint32_t RHSMask = getPermuteMask(V: RHS);
14732 if (LHSMask != ~0u && RHSMask != ~0u) {
14733 // Canonicalize the expression in an attempt to have fewer unique masks
14734 // and therefore fewer registers used to hold the masks.
14735 if (LHSMask > RHSMask) {
14736 std::swap(a&: LHSMask, b&: RHSMask);
14737 std::swap(a&: LHS, b&: RHS);
14738 }
14739
14740 // Select 0xc for each lane used from source operand. Zero has 0xc mask
14741 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
14742 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
14743 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
14744
14745 // Check of we need to combine values from two sources within a byte.
14746 if (!(LHSUsedLanes & RHSUsedLanes) &&
14747 // If we select high and lower word keep it for SDWA.
14748 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
14749 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
14750 // Each byte in each mask is either selector mask 0-3, or has higher
14751 // bits set in either of masks, which can be 0xff for 0xff or 0x0c for
14752 // zero. If 0x0c is in either mask it shall always be 0x0c. Otherwise
14753 // mask which is not 0xff wins. By anding both masks we have a correct
14754 // result except that 0x0c shall be corrected to give 0x0c only.
14755 uint32_t Mask = LHSMask & RHSMask;
14756 for (unsigned I = 0; I < 32; I += 8) {
14757 uint32_t ByteSel = 0xff << I;
14758 if ((LHSMask & ByteSel) == 0x0c || (RHSMask & ByteSel) == 0x0c)
14759 Mask &= (0x0c << I) & 0xffffffff;
14760 }
14761
14762 // Add 4 to each active LHS lane. It will not affect any existing 0xff
14763 // or 0x0c.
14764 uint32_t Sel = Mask | (LHSUsedLanes & 0x04040404);
14765 SDLoc DL(N);
14766
14767 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
14768 N2: RHS.getOperand(i: 0),
14769 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
14770 }
14771 }
14772 }
14773
14774 return SDValue();
14775}
14776
14777// A key component of v_perm is a mapping between byte position of the src
14778// operands, and the byte position of the dest. To provide such, we need: 1. the
14779// node that provides x byte of the dest of the OR, and 2. the byte of the node
14780// used to provide that x byte. calculateByteProvider finds which node provides
14781// a certain byte of the dest of the OR, and calculateSrcByte takes that node,
14782// and finds an ultimate src and byte position For example: The supported
14783// LoadCombine pattern for vector loads is as follows
14784// t1
14785// or
14786// / \
14787// t2 t3
14788// zext shl
14789// | | \
14790// t4 t5 16
14791// or anyext
14792// / \ |
14793// t6 t7 t8
14794// srl shl or
14795// / | / \ / \
14796// t9 t10 t11 t12 t13 t14
14797// trunc* 8 trunc* 8 and and
14798// | | / | | \
14799// t15 t16 t17 t18 t19 t20
14800// trunc* 255 srl -256
14801// | / \
14802// t15 t15 16
14803//
14804// *In this example, the truncs are from i32->i16
14805//
14806// calculateByteProvider would find t6, t7, t13, and t14 for bytes 0-3
14807// respectively. calculateSrcByte would find (given node) -> ultimate src &
14808// byteposition: t6 -> t15 & 1, t7 -> t16 & 0, t13 -> t15 & 0, t14 -> t15 & 3.
14809// After finding the mapping, we can combine the tree into vperm t15, t16,
14810// 0x05000407
14811
14812// Find the source and byte position from a node.
14813// \p DestByte is the byte position of the dest of the or that the src
14814// ultimately provides. \p SrcIndex is the byte of the src that maps to this
14815// dest of the or byte. \p Depth tracks how many recursive iterations we have
14816// performed.
14817static const std::optional<ByteProvider<SDValue>>
14818calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
14819 unsigned Depth = 0) {
14820 // We may need to recursively traverse a series of SRLs
14821 if (Depth >= 6)
14822 return std::nullopt;
14823
14824 if (Op.getValueSizeInBits() < 8)
14825 return std::nullopt;
14826
14827 if (Op.getValueType().isVector())
14828 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
14829
14830 switch (Op->getOpcode()) {
14831 case ISD::TRUNCATE: {
14832 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
14833 }
14834
14835 case ISD::ANY_EXTEND:
14836 case ISD::SIGN_EXTEND:
14837 case ISD::ZERO_EXTEND:
14838 case ISD::SIGN_EXTEND_INREG: {
14839 SDValue NarrowOp = Op->getOperand(Num: 0);
14840 auto NarrowVT = NarrowOp.getValueType();
14841 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG) {
14842 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
14843 NarrowVT = VTSign->getVT();
14844 }
14845 if (!NarrowVT.isByteSized())
14846 return std::nullopt;
14847 uint64_t NarrowByteWidth = NarrowVT.getStoreSize();
14848
14849 if (SrcIndex >= NarrowByteWidth)
14850 return std::nullopt;
14851 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex, Depth: Depth + 1);
14852 }
14853
14854 case ISD::SRA:
14855 case ISD::SRL: {
14856 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
14857 if (!ShiftOp)
14858 return std::nullopt;
14859
14860 uint64_t BitShift = ShiftOp->getZExtValue();
14861
14862 if (BitShift % 8 != 0)
14863 return std::nullopt;
14864
14865 uint64_t NewSrcIndex = SrcIndex + BitShift / 8;
14866 if (NewSrcIndex >= Op.getScalarValueSizeInBits() / 8)
14867 return std::nullopt;
14868
14869 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte, SrcIndex: NewSrcIndex,
14870 Depth: Depth + 1);
14871 }
14872
14873 default: {
14874 return ByteProvider<SDValue>::getSrc(Val: Op, ByteOffset: DestByte, VectorOffset: SrcIndex);
14875 }
14876 }
14877 llvm_unreachable("fully handled switch");
14878}
14879
14880// For a byte position in the result of an Or, traverse the tree and find the
14881// node (and the byte of the node) which ultimately provides this {Or,
14882// BytePosition}. \p Op is the operand we are currently examining. \p Index is
14883// the byte position of the Op that corresponds with the originally requested
14884// byte of the Or \p Depth tracks how many recursive iterations we have
14885// performed. \p StartingIndex is the originally requested byte of the Or
14886static const std::optional<ByteProvider<SDValue>>
14887calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
14888 unsigned StartingIndex = 0) {
14889 // Finding Src tree of RHS of or typically requires at least 1 additional
14890 // depth
14891 if (Depth > 6)
14892 return std::nullopt;
14893
14894 unsigned BitWidth = Op.getScalarValueSizeInBits();
14895 if (BitWidth % 8 != 0)
14896 return std::nullopt;
14897 if (Index > BitWidth / 8 - 1)
14898 return std::nullopt;
14899
14900 bool IsVec = Op.getValueType().isVector();
14901 switch (Op.getOpcode()) {
14902 case ISD::OR: {
14903 if (IsVec)
14904 return std::nullopt;
14905
14906 auto RHS = calculateByteProvider(Op: Op.getOperand(i: 1), Index, Depth: Depth + 1,
14907 StartingIndex);
14908 if (!RHS)
14909 return std::nullopt;
14910 auto LHS = calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
14911 StartingIndex);
14912 if (!LHS)
14913 return std::nullopt;
14914 // A well formed Or will have two ByteProviders for each byte, one of which
14915 // is constant zero
14916 if (!LHS->isConstantZero() && !RHS->isConstantZero())
14917 return std::nullopt;
14918 if (!LHS || LHS->isConstantZero())
14919 return RHS;
14920 if (!RHS || RHS->isConstantZero())
14921 return LHS;
14922 return std::nullopt;
14923 }
14924
14925 case ISD::AND: {
14926 if (IsVec)
14927 return std::nullopt;
14928
14929 auto *BitMaskOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
14930 if (!BitMaskOp)
14931 return std::nullopt;
14932
14933 uint32_t BitMask = BitMaskOp->getZExtValue();
14934 // Bits we expect for our StartingIndex
14935 uint32_t IndexMask = 0xFF << (Index * 8);
14936
14937 if ((IndexMask & BitMask) != IndexMask) {
14938 // If the result of the and partially provides the byte, then it
14939 // is not well formatted
14940 if (IndexMask & BitMask)
14941 return std::nullopt;
14942 return ByteProvider<SDValue>::getConstantZero();
14943 }
14944
14945 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex, SrcIndex: Index);
14946 }
14947
14948 case ISD::FSHR: {
14949 if (IsVec)
14950 return std::nullopt;
14951
14952 // fshr(X,Y,Z): (X << (BW - (Z % BW))) | (Y >> (Z % BW))
14953 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
14954 if (!ShiftOp || Op.getValueType().isVector())
14955 return std::nullopt;
14956
14957 uint64_t BitsProvided = Op.getValueSizeInBits();
14958 if (BitsProvided % 8 != 0)
14959 return std::nullopt;
14960
14961 uint64_t BitShift = ShiftOp->getAPIntValue().urem(RHS: BitsProvided);
14962 if (BitShift % 8)
14963 return std::nullopt;
14964
14965 uint64_t ConcatSizeInBytes = BitsProvided / 4;
14966 uint64_t ByteShift = BitShift / 8;
14967
14968 uint64_t NewIndex = (Index + ByteShift) % ConcatSizeInBytes;
14969 uint64_t BytesProvided = BitsProvided / 8;
14970 SDValue NextOp = Op.getOperand(i: NewIndex >= BytesProvided ? 0 : 1);
14971 NewIndex %= BytesProvided;
14972 return calculateByteProvider(Op: NextOp, Index: NewIndex, Depth: Depth + 1, StartingIndex);
14973 }
14974
14975 case ISD::SRA:
14976 case ISD::SRL: {
14977 if (IsVec)
14978 return std::nullopt;
14979
14980 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
14981 if (!ShiftOp)
14982 return std::nullopt;
14983
14984 uint64_t BitShift = ShiftOp->getZExtValue();
14985 if (BitShift % 8)
14986 return std::nullopt;
14987
14988 auto BitsProvided = Op.getScalarValueSizeInBits();
14989 if (BitsProvided % 8 != 0)
14990 return std::nullopt;
14991
14992 uint64_t BytesProvided = BitsProvided / 8;
14993 uint64_t ByteShift = BitShift / 8;
14994 if (Index + ByteShift < BytesProvided)
14995 return calculateSrcByte(Op: Op->getOperand(Num: 0), DestByte: StartingIndex,
14996 SrcIndex: Index + ByteShift);
14997 // SRA's out-of-range bytes are sign bits, not constant zero.
14998 if (Op.getOpcode() == ISD::SRA)
14999 return std::nullopt;
15000 return ByteProvider<SDValue>::getConstantZero();
15001 }
15002
15003 case ISD::SHL: {
15004 if (IsVec)
15005 return std::nullopt;
15006
15007 auto *ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15008 if (!ShiftOp)
15009 return std::nullopt;
15010
15011 uint64_t BitShift = ShiftOp->getZExtValue();
15012 if (BitShift % 8 != 0)
15013 return std::nullopt;
15014 uint64_t ByteShift = BitShift / 8;
15015
15016 // If we are shifting by an amount greater than (or equal to)
15017 // the index we are trying to provide, then it provides 0s. If not,
15018 // then this bytes are not definitively 0s, and the corresponding byte
15019 // of interest is Index - ByteShift of the src
15020 return Index < ByteShift
15021 ? ByteProvider<SDValue>::getConstantZero()
15022 : calculateByteProvider(Op: Op.getOperand(i: 0), Index: Index - ByteShift,
15023 Depth: Depth + 1, StartingIndex);
15024 }
15025 case ISD::ANY_EXTEND:
15026 case ISD::SIGN_EXTEND:
15027 case ISD::ZERO_EXTEND:
15028 case ISD::SIGN_EXTEND_INREG:
15029 case ISD::AssertZext:
15030 case ISD::AssertSext: {
15031 if (IsVec)
15032 return std::nullopt;
15033
15034 SDValue NarrowOp = Op->getOperand(Num: 0);
15035 unsigned NarrowBitWidth = NarrowOp.getValueSizeInBits();
15036 if (Op->getOpcode() == ISD::SIGN_EXTEND_INREG ||
15037 Op->getOpcode() == ISD::AssertZext ||
15038 Op->getOpcode() == ISD::AssertSext) {
15039 auto *VTSign = cast<VTSDNode>(Val: Op->getOperand(Num: 1));
15040 NarrowBitWidth = VTSign->getVT().getSizeInBits();
15041 }
15042 if (NarrowBitWidth % 8 != 0)
15043 return std::nullopt;
15044 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15045
15046 if (Index >= NarrowByteWidth)
15047 return Op.getOpcode() == ISD::ZERO_EXTEND
15048 ? std::optional<ByteProvider<SDValue>>(
15049 ByteProvider<SDValue>::getConstantZero())
15050 : std::nullopt;
15051 return calculateByteProvider(Op: NarrowOp, Index, Depth: Depth + 1, StartingIndex);
15052 }
15053
15054 case ISD::TRUNCATE: {
15055 if (IsVec)
15056 return std::nullopt;
15057
15058 uint64_t NarrowByteWidth = BitWidth / 8;
15059
15060 if (NarrowByteWidth >= Index) {
15061 return calculateByteProvider(Op: Op.getOperand(i: 0), Index, Depth: Depth + 1,
15062 StartingIndex);
15063 }
15064
15065 return std::nullopt;
15066 }
15067
15068 case ISD::CopyFromReg: {
15069 if (BitWidth / 8 > Index)
15070 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15071
15072 return std::nullopt;
15073 }
15074
15075 case ISD::LOAD: {
15076 auto *L = cast<LoadSDNode>(Val: Op.getNode());
15077
15078 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
15079 if (NarrowBitWidth % 8 != 0)
15080 return std::nullopt;
15081 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
15082
15083 // If the width of the load does not reach byte we are trying to provide for
15084 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
15085 // question
15086 if (Index >= NarrowByteWidth) {
15087 return L->getExtensionType() == ISD::ZEXTLOAD
15088 ? std::optional<ByteProvider<SDValue>>(
15089 ByteProvider<SDValue>::getConstantZero())
15090 : std::nullopt;
15091 }
15092
15093 if (NarrowByteWidth > Index) {
15094 return calculateSrcByte(Op, DestByte: StartingIndex, SrcIndex: Index);
15095 }
15096
15097 return std::nullopt;
15098 }
15099
15100 case ISD::BSWAP: {
15101 if (IsVec)
15102 return std::nullopt;
15103
15104 return calculateByteProvider(Op: Op->getOperand(Num: 0), Index: BitWidth / 8 - Index - 1,
15105 Depth: Depth + 1, StartingIndex);
15106 }
15107
15108 case ISD::EXTRACT_VECTOR_ELT: {
15109 auto *IdxOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
15110 if (!IdxOp)
15111 return std::nullopt;
15112 auto VecIdx = IdxOp->getZExtValue();
15113 auto ScalarSize = Op.getScalarValueSizeInBits();
15114 if (ScalarSize < 32)
15115 Index = ScalarSize == 8 ? VecIdx : VecIdx * 2 + Index;
15116 return calculateSrcByte(Op: ScalarSize >= 32 ? Op : Op.getOperand(i: 0),
15117 DestByte: StartingIndex, SrcIndex: Index);
15118 }
15119
15120 case AMDGPUISD::PERM: {
15121 if (IsVec)
15122 return std::nullopt;
15123
15124 auto *PermMask = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 2));
15125 if (!PermMask)
15126 return std::nullopt;
15127
15128 auto IdxMask =
15129 (PermMask->getZExtValue() & (0xFF << (Index * 8))) >> (Index * 8);
15130 if (IdxMask > 0x07 && IdxMask != 0x0c)
15131 return std::nullopt;
15132
15133 auto NextOp = Op.getOperand(i: IdxMask > 0x03 ? 0 : 1);
15134 auto NextIndex = IdxMask > 0x03 ? IdxMask % 4 : IdxMask;
15135
15136 return IdxMask != 0x0c ? calculateSrcByte(Op: NextOp, DestByte: StartingIndex, SrcIndex: NextIndex)
15137 : ByteProvider<SDValue>(
15138 ByteProvider<SDValue>::getConstantZero());
15139 }
15140
15141 default: {
15142 return std::nullopt;
15143 }
15144 }
15145
15146 llvm_unreachable("fully handled switch");
15147}
15148
15149// Returns true if the Operand is a scalar and is 16 bits
15150static bool isExtendedFrom16Bits(SDValue &Operand) {
15151
15152 switch (Operand.getOpcode()) {
15153 case ISD::ANY_EXTEND:
15154 case ISD::SIGN_EXTEND:
15155 case ISD::ZERO_EXTEND: {
15156 auto OpVT = Operand.getOperand(i: 0).getValueType();
15157 return !OpVT.isVector() && OpVT.getSizeInBits() == 16;
15158 }
15159 case ISD::LOAD: {
15160 LoadSDNode *L = cast<LoadSDNode>(Val: Operand.getNode());
15161 auto ExtType = cast<LoadSDNode>(Val: L)->getExtensionType();
15162 if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::SEXTLOAD ||
15163 ExtType == ISD::EXTLOAD) {
15164 auto MemVT = L->getMemoryVT();
15165 return !MemVT.isVector() && MemVT.getSizeInBits() == 16;
15166 }
15167 return L->getMemoryVT().getSizeInBits() == 16;
15168 }
15169 default:
15170 return false;
15171 }
15172}
15173
15174// Returns true if the mask matches consecutive bytes, and the first byte
15175// begins at a power of 2 byte offset from 0th byte
15176static bool addresses16Bits(int Mask) {
15177 int Low8 = Mask & 0xff;
15178 int Hi8 = (Mask & 0xff00) >> 8;
15179
15180 assert(Low8 < 8 && Hi8 < 8);
15181 // Are the bytes contiguous in the order of increasing addresses.
15182 bool IsConsecutive = (Hi8 - Low8 == 1);
15183 // Is the first byte at location that is aligned for 16 bit instructions.
15184 // A counter example is taking 2 consecutive bytes starting at the 8th bit.
15185 // In this case, we still need code to extract the 16 bit operand, so it
15186 // is better to use i8 v_perm
15187 bool Is16Aligned = !(Low8 % 2);
15188
15189 return IsConsecutive && Is16Aligned;
15190}
15191
15192// Do not lower into v_perm if the operands are actually 16 bit
15193// and the selected bits (based on PermMask) correspond with two
15194// easily addressable 16 bit operands.
15195static bool hasNon16BitAccesses(uint64_t PermMask, SDValue &Op,
15196 SDValue &OtherOp) {
15197 int Low16 = PermMask & 0xffff;
15198 int Hi16 = (PermMask & 0xffff0000) >> 16;
15199
15200 auto TempOp = peekThroughBitcasts(V: Op);
15201 auto TempOtherOp = peekThroughBitcasts(V: OtherOp);
15202
15203 auto OpIs16Bit =
15204 TempOp.getValueSizeInBits() == 16 || isExtendedFrom16Bits(Operand&: TempOp);
15205 if (!OpIs16Bit)
15206 return true;
15207
15208 auto OtherOpIs16Bit = TempOtherOp.getValueSizeInBits() == 16 ||
15209 isExtendedFrom16Bits(Operand&: TempOtherOp);
15210 if (!OtherOpIs16Bit)
15211 return true;
15212
15213 // Do we cleanly address both
15214 return !addresses16Bits(Mask: Low16) || !addresses16Bits(Mask: Hi16);
15215}
15216
15217static SDValue getDWordFromOffset(SelectionDAG &DAG, SDLoc SL, SDValue Src,
15218 unsigned DWordOffset) {
15219 SDValue Ret;
15220
15221 auto TypeSize = Src.getValueSizeInBits().getFixedValue();
15222 // ByteProvider must be at least 8 bits
15223 assert(Src.getValueSizeInBits().isKnownMultipleOf(8));
15224
15225 if (TypeSize <= 32)
15226 return DAG.getBitcastedAnyExtOrTrunc(Op: Src, DL: SL, VT: MVT::i32);
15227
15228 if (Src.getValueType().isVector()) {
15229 auto ScalarTySize = Src.getScalarValueSizeInBits();
15230 auto ScalarTy = Src.getValueType().getScalarType();
15231 if (ScalarTySize == 32) {
15232 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Src,
15233 N2: DAG.getConstant(Val: DWordOffset, DL: SL, VT: MVT::i32));
15234 }
15235 if (ScalarTySize > 32) {
15236 Ret = DAG.getNode(
15237 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ScalarTy, N1: Src,
15238 N2: DAG.getConstant(Val: DWordOffset / (ScalarTySize / 32), DL: SL, VT: MVT::i32));
15239 auto ShiftVal = 32 * (DWordOffset % (ScalarTySize / 32));
15240 if (ShiftVal)
15241 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Ret.getValueType(), N1: Ret,
15242 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15243 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15244 }
15245
15246 assert(ScalarTySize < 32);
15247 auto NumElements = TypeSize / ScalarTySize;
15248 auto Trunc32Elements = (ScalarTySize * NumElements) / 32;
15249 auto NormalizedTrunc = Trunc32Elements * 32 / ScalarTySize;
15250 auto NumElementsIn32 = 32 / ScalarTySize;
15251 auto NumAvailElements = DWordOffset < Trunc32Elements
15252 ? NumElementsIn32
15253 : NumElements - NormalizedTrunc;
15254
15255 SmallVector<SDValue, 4> VecSrcs;
15256 DAG.ExtractVectorElements(Op: Src, Args&: VecSrcs, Start: DWordOffset * NumElementsIn32,
15257 Count: NumAvailElements);
15258
15259 Ret = DAG.getBuildVector(
15260 VT: MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ScalarTySize), NumElements: NumAvailElements), DL: SL,
15261 Ops: VecSrcs);
15262 return Ret = DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15263 }
15264
15265 /// Scalar Type
15266 auto ShiftVal = 32 * DWordOffset;
15267 Ret = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: Src.getValueType(), N1: Src,
15268 N2: DAG.getConstant(Val: ShiftVal, DL: SL, VT: MVT::i32));
15269 return DAG.getBitcastedAnyExtOrTrunc(Op: Ret, DL: SL, VT: MVT::i32);
15270}
15271
15272static SDValue matchPERM(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
15273 SelectionDAG &DAG = DCI.DAG;
15274 [[maybe_unused]] EVT VT = N->getValueType(ResNo: 0);
15275 SmallVector<ByteProvider<SDValue>, 8> PermNodes;
15276
15277 // VT is known to be MVT::i32, so we need to provide 4 bytes.
15278 assert(VT == MVT::i32);
15279 for (int i = 0; i < 4; i++) {
15280 // Find the ByteProvider that provides the ith byte of the result of OR
15281 std::optional<ByteProvider<SDValue>> P =
15282 calculateByteProvider(Op: SDValue(N, 0), Index: i, Depth: 0, /*StartingIndex = */ i);
15283 // TODO support constantZero
15284 if (!P || P->isConstantZero())
15285 return SDValue();
15286
15287 PermNodes.push_back(Elt: *P);
15288 }
15289 if (PermNodes.size() != 4)
15290 return SDValue();
15291
15292 std::pair<unsigned, unsigned> FirstSrc(0, PermNodes[0].SrcOffset / 4);
15293 std::optional<std::pair<unsigned, unsigned>> SecondSrc;
15294 uint64_t PermMask = 0x00000000;
15295 for (size_t i = 0; i < PermNodes.size(); i++) {
15296 auto PermOp = PermNodes[i];
15297 // Since the mask is applied to Src1:Src2, Src1 bytes must be offset
15298 // by sizeof(Src2) = 4
15299 int SrcByteAdjust = 4;
15300
15301 // If the Src uses a byte from a different DWORD, then it corresponds
15302 // with a difference source
15303 if (!PermOp.hasSameSrc(Other: PermNodes[FirstSrc.first]) ||
15304 ((PermOp.SrcOffset / 4) != FirstSrc.second)) {
15305 if (SecondSrc)
15306 if (!PermOp.hasSameSrc(Other: PermNodes[SecondSrc->first]) ||
15307 ((PermOp.SrcOffset / 4) != SecondSrc->second))
15308 return SDValue();
15309
15310 // Set the index of the second distinct Src node
15311 SecondSrc = {i, PermNodes[i].SrcOffset / 4};
15312 assert(!(PermNodes[SecondSrc->first].Src->getValueSizeInBits() % 8));
15313 SrcByteAdjust = 0;
15314 }
15315 assert((PermOp.SrcOffset % 4) + SrcByteAdjust < 8);
15316 assert(!DAG.getDataLayout().isBigEndian());
15317 PermMask |= ((PermOp.SrcOffset % 4) + SrcByteAdjust) << (i * 8);
15318 }
15319 SDLoc DL(N);
15320 SDValue Op = *PermNodes[FirstSrc.first].Src;
15321 Op = getDWordFromOffset(DAG, SL: DL, Src: Op, DWordOffset: FirstSrc.second);
15322 assert(Op.getValueSizeInBits() == 32);
15323
15324 // Check that we are not just extracting the bytes in order from an op
15325 if (!SecondSrc) {
15326 int Low16 = PermMask & 0xffff;
15327 int Hi16 = (PermMask & 0xffff0000) >> 16;
15328
15329 bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
15330 bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
15331
15332 // The perm op would really just produce Op. So combine into Op
15333 if (WellFormedLow && WellFormedHi)
15334 return DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: 32), V: Op);
15335 }
15336
15337 SDValue OtherOp = SecondSrc ? *PermNodes[SecondSrc->first].Src : Op;
15338
15339 if (SecondSrc) {
15340 OtherOp = getDWordFromOffset(DAG, SL: DL, Src: OtherOp, DWordOffset: SecondSrc->second);
15341 assert(OtherOp.getValueSizeInBits() == 32);
15342 }
15343
15344 // Check that we haven't just recreated the same FSHR node.
15345 if (N->getOpcode() == ISD::FSHR &&
15346 (N->getOperand(Num: 0) == Op || N->getOperand(Num: 0) == OtherOp) &&
15347 (N->getOperand(Num: 1) == Op || N->getOperand(Num: 1) == OtherOp))
15348 return SDValue();
15349
15350 if (hasNon16BitAccesses(PermMask, Op, OtherOp)) {
15351
15352 assert(Op.getValueType().isByteSized() &&
15353 OtherOp.getValueType().isByteSized());
15354
15355 // If the ultimate src is less than 32 bits, then we will only be
15356 // using bytes 0: Op.getValueSizeInBytes() - 1 in the or.
15357 // CalculateByteProvider would not have returned Op as source if we
15358 // used a byte that is outside its ValueType. Thus, we are free to
15359 // ANY_EXTEND as the extended bits are dont-cares.
15360 Op = DAG.getBitcastedAnyExtOrTrunc(Op, DL, VT: MVT::i32);
15361 OtherOp = DAG.getBitcastedAnyExtOrTrunc(Op: OtherOp, DL, VT: MVT::i32);
15362
15363 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: Op, N2: OtherOp,
15364 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
15365 }
15366 return SDValue();
15367}
15368
15369SDValue SITargetLowering::performOrCombine(SDNode *N,
15370 DAGCombinerInfo &DCI) const {
15371 SelectionDAG &DAG = DCI.DAG;
15372 SDValue LHS = N->getOperand(Num: 0);
15373 SDValue RHS = N->getOperand(Num: 1);
15374
15375 EVT VT = N->getValueType(ResNo: 0);
15376 if (VT == MVT::i1) {
15377 // or (fp_class x, c1), (fp_class x, c2) -> fp_class x, (c1 | c2)
15378 if (LHS.getOpcode() == AMDGPUISD::FP_CLASS &&
15379 RHS.getOpcode() == AMDGPUISD::FP_CLASS) {
15380 SDValue Src = LHS.getOperand(i: 0);
15381 if (Src != RHS.getOperand(i: 0))
15382 return SDValue();
15383
15384 const ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
15385 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: 1));
15386 if (!CLHS || !CRHS)
15387 return SDValue();
15388
15389 // Only 10 bits are used.
15390 static const uint32_t MaxMask = 0x3ff;
15391
15392 uint32_t NewMask =
15393 (CLHS->getZExtValue() | CRHS->getZExtValue()) & MaxMask;
15394 SDLoc DL(N);
15395 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL, VT: MVT::i1, N1: Src,
15396 N2: DAG.getConstant(Val: NewMask, DL, VT: MVT::i32));
15397 }
15398
15399 return SDValue();
15400 }
15401
15402 // or (perm x, y, c1), c2 -> perm x, y, permute_mask(c1, c2)
15403 if (isa<ConstantSDNode>(Val: RHS) && LHS.hasOneUse() &&
15404 LHS.getOpcode() == AMDGPUISD::PERM &&
15405 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2))) {
15406 uint32_t Sel = getConstantPermuteMask(C: N->getConstantOperandVal(Num: 1));
15407 if (!Sel)
15408 return SDValue();
15409
15410 Sel |= LHS.getConstantOperandVal(i: 2);
15411 SDLoc DL(N);
15412 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15413 N2: LHS.getOperand(i: 1), N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15414 }
15415
15416 // or (op x, c1), (op y, c2) -> perm x, y, permute_mask(c1, c2)
15417 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
15418 if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
15419 N->isDivergent() && TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
15420
15421 // If all the uses of an or need to extract the individual elements, do not
15422 // attempt to lower into v_perm
15423 auto usesCombinedOperand = [](SDNode *OrUse) {
15424 // If we have any non-vectorized use, then it is a candidate for v_perm
15425 if (OrUse->getOpcode() != ISD::BITCAST ||
15426 !OrUse->getValueType(ResNo: 0).isVector())
15427 return true;
15428
15429 // If we have any non-vectorized use, then it is a candidate for v_perm
15430 for (auto *VUser : OrUse->users()) {
15431 if (!VUser->getValueType(ResNo: 0).isVector())
15432 return true;
15433
15434 // If the use of a vector is a store, then combining via a v_perm
15435 // is beneficial.
15436 // TODO -- whitelist more uses
15437 for (auto VectorwiseOp : {ISD::STORE, ISD::CopyToReg, ISD::CopyFromReg})
15438 if (VUser->getOpcode() == VectorwiseOp)
15439 return true;
15440 }
15441 return false;
15442 };
15443
15444 if (!any_of(Range: N->users(), P: usesCombinedOperand))
15445 return SDValue();
15446
15447 uint32_t LHSMask = getPermuteMask(V: LHS);
15448 uint32_t RHSMask = getPermuteMask(V: RHS);
15449
15450 if (LHSMask != ~0u && RHSMask != ~0u) {
15451 // Canonicalize the expression in an attempt to have fewer unique masks
15452 // and therefore fewer registers used to hold the masks.
15453 if (LHSMask > RHSMask) {
15454 std::swap(a&: LHSMask, b&: RHSMask);
15455 std::swap(a&: LHS, b&: RHS);
15456 }
15457
15458 // Select 0xc for each lane used from source operand. Zero has 0xc mask
15459 // set, 0xff have 0xff in the mask, actual lanes are in the 0-3 range.
15460 uint32_t LHSUsedLanes = ~(LHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15461 uint32_t RHSUsedLanes = ~(RHSMask & 0x0c0c0c0c) & 0x0c0c0c0c;
15462
15463 // Check of we need to combine values from two sources within a byte.
15464 if (!(LHSUsedLanes & RHSUsedLanes) &&
15465 // If we select high and lower word keep it for SDWA.
15466 // TODO: teach SDWA to work with v_perm_b32 and remove the check.
15467 !(LHSUsedLanes == 0x0c0c0000 && RHSUsedLanes == 0x00000c0c)) {
15468 // Kill zero bytes selected by other mask. Zero value is 0xc.
15469 LHSMask &= ~RHSUsedLanes;
15470 RHSMask &= ~LHSUsedLanes;
15471 // Add 4 to each active LHS lane
15472 LHSMask |= LHSUsedLanes & 0x04040404;
15473 // Combine masks
15474 uint32_t Sel = LHSMask | RHSMask;
15475 SDLoc DL(N);
15476
15477 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: LHS.getOperand(i: 0),
15478 N2: RHS.getOperand(i: 0),
15479 N3: DAG.getConstant(Val: Sel, DL, VT: MVT::i32));
15480 }
15481 }
15482 if (LHSMask == ~0u || RHSMask == ~0u) {
15483 if (SDValue Perm = matchPERM(N, DCI))
15484 return Perm;
15485 }
15486 }
15487
15488 // Detect identity v2i32 OR and replace with identity source node.
15489 // Specifically an Or that has operands constructed from the same source node
15490 // via extract_vector_elt and build_vector. I.E.
15491 // v2i32 or(
15492 // v2i32 build_vector(
15493 // i32 extract_elt(%IdentitySrc, 0),
15494 // i32 0
15495 // ),
15496 // v2i32 build_vector(
15497 // i32 0,
15498 // i32 extract_elt(%IdentitySrc, 1)
15499 // ) )
15500 // =>
15501 // v2i32 %IdentitySrc
15502
15503 if (VT == MVT::v2i32 && LHS->getOpcode() == ISD::BUILD_VECTOR &&
15504 RHS->getOpcode() == ISD::BUILD_VECTOR) {
15505
15506 ConstantSDNode *LC = dyn_cast<ConstantSDNode>(Val: LHS->getOperand(Num: 1));
15507 ConstantSDNode *RC = dyn_cast<ConstantSDNode>(Val: RHS->getOperand(Num: 0));
15508
15509 // Test for and normalise build vectors.
15510 if (LC && RC && LC->getZExtValue() == 0 && RC->getZExtValue() == 0) {
15511
15512 // Get the extract_vector_element operands.
15513 SDValue LEVE = LHS->getOperand(Num: 0);
15514 SDValue REVE = RHS->getOperand(Num: 1);
15515
15516 if (LEVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
15517 REVE->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
15518 // Check that different elements from the same vector are
15519 // extracted.
15520 if (LEVE->getOperand(Num: 0) == REVE->getOperand(Num: 0) &&
15521 LEVE->getOperand(Num: 1) != REVE->getOperand(Num: 1)) {
15522 SDValue IdentitySrc = LEVE.getOperand(i: 0);
15523 return IdentitySrc;
15524 }
15525 }
15526 }
15527 }
15528
15529 if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
15530 return SDValue();
15531
15532 // TODO: This could be a generic combine with a predicate for extracting the
15533 // high half of an integer being free.
15534
15535 // (or i64:x, (zero_extend i32:y)) ->
15536 // i64 (bitcast (v2i32 build_vector (or i32:y, lo_32(x)), hi_32(x)))
15537 if (LHS.getOpcode() == ISD::ZERO_EXTEND &&
15538 RHS.getOpcode() != ISD::ZERO_EXTEND)
15539 std::swap(a&: LHS, b&: RHS);
15540
15541 if (RHS.getOpcode() == ISD::ZERO_EXTEND) {
15542 SDValue ExtSrc = RHS.getOperand(i: 0);
15543 EVT SrcVT = ExtSrc.getValueType();
15544 if (SrcVT == MVT::i32) {
15545 SDLoc SL(N);
15546 auto [LowLHS, HiBits] = split64BitValue(Op: LHS, DAG);
15547 SDValue LowOr = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: LowLHS, N2: ExtSrc);
15548
15549 DCI.AddToWorklist(N: LowOr.getNode());
15550 DCI.AddToWorklist(N: HiBits.getNode());
15551
15552 SDValue Vec =
15553 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32, N1: LowOr, N2: HiBits);
15554 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
15555 }
15556 }
15557
15558 const ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
15559 if (CRHS) {
15560 if (SDValue Split = splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::OR,
15561 LHS: N->getOperand(Num: 0), CRHS))
15562 return Split;
15563 }
15564
15565 return SDValue();
15566}
15567
15568SDValue SITargetLowering::performXorCombine(SDNode *N,
15569 DAGCombinerInfo &DCI) const {
15570 if (SDValue RV = reassociateScalarOps(N, DAG&: DCI.DAG))
15571 return RV;
15572
15573 SDValue LHS = N->getOperand(Num: 0);
15574 SDValue RHS = N->getOperand(Num: 1);
15575
15576 const ConstantSDNode *CRHS = isConstOrConstSplat(N: RHS);
15577 SelectionDAG &DAG = DCI.DAG;
15578
15579 EVT VT = N->getValueType(ResNo: 0);
15580 if (CRHS && VT == MVT::i64) {
15581 if (SDValue Split =
15582 splitBinaryBitConstantOp(DCI, SL: SDLoc(N), Opc: ISD::XOR, LHS, CRHS))
15583 return Split;
15584 }
15585
15586 // v2i32 (xor (vselect cc, x, y), K) ->
15587 // (v2i32 svelect cc, (xor x, K), (xor y, K)) This enables the xor to be
15588 // replaced with source modifiers when the select is lowered to CNDMASK.
15589 unsigned Opc = LHS.getOpcode();
15590 if (((Opc == ISD::VSELECT && VT == MVT::v2i32) ||
15591 (Opc == ISD::SELECT && VT == MVT::i64)) &&
15592 CRHS && CRHS->getAPIntValue().isSignMask()) {
15593 SDValue CC = LHS->getOperand(Num: 0);
15594 SDValue TRUE = LHS->getOperand(Num: 1);
15595 SDValue FALSE = LHS->getOperand(Num: 2);
15596 SDValue XTrue = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: TRUE, N2: RHS);
15597 SDValue XFalse = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: FALSE, N2: RHS);
15598 SDValue XSelect =
15599 DAG.getNode(Opcode: ISD::VSELECT, DL: SDLoc(N), VT, N1: CC, N2: XTrue, N3: XFalse);
15600 return XSelect;
15601 }
15602
15603 // Make sure to apply the 64-bit constant splitting fold before trying to fold
15604 // fneg-like xors into 64-bit select.
15605 if (LHS.getOpcode() == ISD::SELECT && VT == MVT::i32) {
15606 // This looks like an fneg, try to fold as a source modifier.
15607 if (CRHS && CRHS->getAPIntValue().isSignMask() &&
15608 shouldFoldFNegIntoSrc(FNeg: N, FNegSrc: LHS)) {
15609 // xor (select c, a, b), 0x80000000 ->
15610 // bitcast (select c, (fneg (bitcast a)), (fneg (bitcast b)))
15611 SDLoc DL(N);
15612 SDValue CastLHS =
15613 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 1));
15614 SDValue CastRHS =
15615 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: LHS->getOperand(Num: 2));
15616 SDValue FNegLHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastLHS);
15617 SDValue FNegRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f32, Operand: CastRHS);
15618 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL, VT: MVT::f32,
15619 N1: LHS->getOperand(Num: 0), N2: FNegLHS, N3: FNegRHS);
15620 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: NewSelect);
15621 }
15622 }
15623
15624 return SDValue();
15625}
15626
15627SDValue
15628SITargetLowering::performZeroOrAnyExtendCombine(SDNode *N,
15629 DAGCombinerInfo &DCI) const {
15630 if (!Subtarget->has16BitInsts() ||
15631 DCI.getDAGCombineLevel() < AfterLegalizeTypes)
15632 return SDValue();
15633
15634 EVT VT = N->getValueType(ResNo: 0);
15635 if (VT != MVT::i32)
15636 return SDValue();
15637
15638 SDValue Src = N->getOperand(Num: 0);
15639 if (Src.getValueType() != MVT::i16)
15640 return SDValue();
15641
15642 if (!Src->hasOneUse())
15643 return SDValue();
15644
15645 // TODO: We bail out below if SrcOffset is not in the first dword (>= 4). It's
15646 // possible we're missing out on some combine opportunities, but we'd need to
15647 // weigh the cost of extracting the byte from the upper dwords.
15648
15649 std::optional<ByteProvider<SDValue>> BP0 =
15650 calculateByteProvider(Op: SDValue(N, 0), Index: 0, Depth: 0, StartingIndex: 0);
15651 if (!BP0 || BP0->SrcOffset >= 4 || !BP0->Src)
15652 return SDValue();
15653 SDValue V0 = *BP0->Src;
15654
15655 std::optional<ByteProvider<SDValue>> BP1 =
15656 calculateByteProvider(Op: SDValue(N, 0), Index: 1, Depth: 0, StartingIndex: 1);
15657 if (!BP1 || BP1->SrcOffset >= 4 || !BP1->Src)
15658 return SDValue();
15659
15660 SDValue V1 = *BP1->Src;
15661
15662 if (V0 == V1)
15663 return SDValue();
15664
15665 SelectionDAG &DAG = DCI.DAG;
15666 SDLoc DL(N);
15667 uint32_t PermMask = 0x0c0c0c0c;
15668 if (V0) {
15669 V0 = DAG.getBitcastedAnyExtOrTrunc(Op: V0, DL, VT: MVT::i32);
15670 PermMask = (PermMask & ~0xFF) | (BP0->SrcOffset + 4);
15671 }
15672
15673 if (V1) {
15674 V1 = DAG.getBitcastedAnyExtOrTrunc(Op: V1, DL, VT: MVT::i32);
15675 PermMask = (PermMask & ~(0xFF << 8)) | (BP1->SrcOffset << 8);
15676 }
15677
15678 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL, VT: MVT::i32, N1: V0, N2: V1,
15679 N3: DAG.getConstant(Val: PermMask, DL, VT: MVT::i32));
15680}
15681
15682SDValue
15683SITargetLowering::performSignExtendInRegCombine(SDNode *N,
15684 DAGCombinerInfo &DCI) const {
15685 SDValue Src = N->getOperand(Num: 0);
15686 auto *VTSign = cast<VTSDNode>(Val: N->getOperand(Num: 1));
15687
15688 // Combine s_buffer_load_u8 or s_buffer_load_u16 with sext and replace them
15689 // with s_buffer_load_i8 and s_buffer_load_i16 respectively.
15690 if (((Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE &&
15691 VTSign->getVT() == MVT::i8) ||
15692 (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_USHORT &&
15693 VTSign->getVT() == MVT::i16))) {
15694 assert(Subtarget->hasScalarSubwordLoads() &&
15695 "s_buffer_load_{u8, i8} are supported "
15696 "in GFX12 (or newer) architectures.");
15697 EVT VT = Src.getValueType();
15698 unsigned Opc = (Src.getOpcode() == AMDGPUISD::SBUFFER_LOAD_UBYTE)
15699 ? AMDGPUISD::SBUFFER_LOAD_BYTE
15700 : AMDGPUISD::SBUFFER_LOAD_SHORT;
15701 SDLoc DL(N);
15702 SDVTList ResList = DCI.DAG.getVTList(VT: MVT::i32);
15703 SDValue Ops[] = {
15704 Src.getOperand(i: 0), // source register
15705 Src.getOperand(i: 1), // offset
15706 Src.getOperand(i: 2) // cachePolicy
15707 };
15708 auto *M = cast<MemSDNode>(Val&: Src);
15709 SDValue BufferLoad = DCI.DAG.getMemIntrinsicNode(
15710 Opcode: Opc, dl: DL, VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
15711 SDValue LoadVal = DCI.DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: BufferLoad);
15712 return LoadVal;
15713 }
15714 if (((Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE &&
15715 VTSign->getVT() == MVT::i8) ||
15716 (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_USHORT &&
15717 VTSign->getVT() == MVT::i16)) &&
15718 Src.hasOneUse()) {
15719 auto *M = cast<MemSDNode>(Val&: Src);
15720 SDValue Ops[] = {Src.getOperand(i: 0), // Chain
15721 Src.getOperand(i: 1), // rsrc
15722 Src.getOperand(i: 2), // vindex
15723 Src.getOperand(i: 3), // voffset
15724 Src.getOperand(i: 4), // soffset
15725 Src.getOperand(i: 5), // offset
15726 Src.getOperand(i: 6), Src.getOperand(i: 7)};
15727 // replace with BUFFER_LOAD_BYTE/SHORT
15728 SDVTList ResList =
15729 DCI.DAG.getVTList(VT1: MVT::i32, VT2: Src.getOperand(i: 0).getValueType());
15730 unsigned Opc = (Src.getOpcode() == AMDGPUISD::BUFFER_LOAD_UBYTE)
15731 ? AMDGPUISD::BUFFER_LOAD_BYTE
15732 : AMDGPUISD::BUFFER_LOAD_SHORT;
15733 SDValue BufferLoadSignExt = DCI.DAG.getMemIntrinsicNode(
15734 Opcode: Opc, dl: SDLoc(N), VTList: ResList, Ops, MemVT: M->getMemoryVT(), MMO: M->getMemOperand());
15735 return DCI.DAG.getMergeValues(
15736 Ops: {BufferLoadSignExt, BufferLoadSignExt.getValue(R: 1)}, dl: SDLoc(N));
15737 }
15738 return SDValue();
15739}
15740
15741SDValue SITargetLowering::performClassCombine(SDNode *N,
15742 DAGCombinerInfo &DCI) const {
15743 SelectionDAG &DAG = DCI.DAG;
15744 SDValue Mask = N->getOperand(Num: 1);
15745
15746 // fp_class x, 0 -> false
15747 if (isNullConstant(V: Mask))
15748 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT: MVT::i1);
15749
15750 if (N->getOperand(Num: 0).isUndef())
15751 return DAG.getUNDEF(VT: MVT::i1);
15752
15753 return SDValue();
15754}
15755
15756SDValue SITargetLowering::performRcpCombine(SDNode *N,
15757 DAGCombinerInfo &DCI) const {
15758 EVT VT = N->getValueType(ResNo: 0);
15759 SDValue N0 = N->getOperand(Num: 0);
15760
15761 if (N0.isUndef()) {
15762 return DCI.DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()),
15763 DL: SDLoc(N), VT);
15764 }
15765
15766 // TODO: Could handle f32 + amdgcn.sqrt but probably never reaches here.
15767 if ((VT == MVT::f16 && N0.getOpcode() == ISD::FSQRT) &&
15768 N->getFlags().hasAllowContract() && N0->getFlags().hasAllowContract()) {
15769 return DCI.DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(N), VT, Operand: N0.getOperand(i: 0),
15770 Flags: N->getFlags());
15771 }
15772
15773 return AMDGPUTargetLowering::performRcpCombine(N, DCI);
15774}
15775
15776bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
15777 SDNodeFlags UserFlags,
15778 unsigned MaxDepth) const {
15779 unsigned Opcode = Op.getOpcode();
15780 if (Opcode == ISD::FCANONICALIZE)
15781 return true;
15782
15783 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
15784 const auto &F = CFP->getValueAPF();
15785 if (F.isNaN() && F.isSignaling())
15786 return false;
15787 if (!F.isDenormal())
15788 return true;
15789
15790 DenormalMode Mode =
15791 DAG.getMachineFunction().getDenormalMode(FPType: F.getSemantics());
15792 return Mode == DenormalMode::getIEEE();
15793 }
15794
15795 // If source is a result of another standard FP operation it is already in
15796 // canonical form.
15797 if (MaxDepth == 0)
15798 return false;
15799
15800 switch (Opcode) {
15801 // These will flush denorms if required.
15802 case ISD::FADD:
15803 case ISD::FSUB:
15804 case ISD::FMUL:
15805 case ISD::FCEIL:
15806 case ISD::FFLOOR:
15807 case ISD::FMA:
15808 case ISD::FMAD:
15809 case ISD::FSQRT:
15810 case ISD::FDIV:
15811 case ISD::FREM:
15812 case ISD::FP_ROUND:
15813 case ISD::FP_EXTEND:
15814 case ISD::FP16_TO_FP:
15815 case ISD::FP_TO_FP16:
15816 case ISD::BF16_TO_FP:
15817 case ISD::FP_TO_BF16:
15818 case ISD::FLDEXP:
15819 case AMDGPUISD::FMUL_LEGACY:
15820 case AMDGPUISD::FMAD_FTZ:
15821 case AMDGPUISD::RCP:
15822 case AMDGPUISD::RSQ:
15823 case AMDGPUISD::RSQ_CLAMP:
15824 case AMDGPUISD::RCP_LEGACY:
15825 case AMDGPUISD::RCP_IFLAG:
15826 case AMDGPUISD::LOG:
15827 case AMDGPUISD::EXP:
15828 case AMDGPUISD::DIV_SCALE:
15829 case AMDGPUISD::DIV_FMAS:
15830 case AMDGPUISD::DIV_FIXUP:
15831 case AMDGPUISD::FRACT:
15832 case AMDGPUISD::CVT_PKRTZ_F16_F32:
15833 case AMDGPUISD::CVT_F32_UBYTE0:
15834 case AMDGPUISD::CVT_F32_UBYTE1:
15835 case AMDGPUISD::CVT_F32_UBYTE2:
15836 case AMDGPUISD::CVT_F32_UBYTE3:
15837 case AMDGPUISD::FP_TO_FP16:
15838 case AMDGPUISD::SIN_HW:
15839 case AMDGPUISD::COS_HW:
15840 return true;
15841
15842 // It can/will be lowered or combined as a bit operation.
15843 // Need to check their input recursively to handle.
15844 case ISD::FNEG:
15845 case ISD::FABS:
15846 case ISD::FCOPYSIGN:
15847 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1);
15848
15849 case ISD::AND:
15850 if (Op.getValueType() == MVT::i32) {
15851 // Be careful as we only know it is a bitcast floating point type. It
15852 // could be f32, v2f16, we have no way of knowing. Luckily the constant
15853 // value that we optimize for, which comes up in fp32 to bf16 conversions,
15854 // is valid to optimize for all types.
15855 if (auto *RHS = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
15856 if (RHS->getZExtValue() == 0xffff0000) {
15857 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1);
15858 }
15859 }
15860 }
15861 break;
15862
15863 case ISD::FSIN:
15864 case ISD::FCOS:
15865 case ISD::FSINCOS:
15866 return Op.getValueType().getScalarType() != MVT::f16;
15867
15868 case ISD::FMINNUM:
15869 case ISD::FMAXNUM:
15870 case ISD::FMINNUM_IEEE:
15871 case ISD::FMAXNUM_IEEE:
15872 case ISD::FMINIMUM:
15873 case ISD::FMAXIMUM:
15874 case ISD::FMINIMUMNUM:
15875 case ISD::FMAXIMUMNUM:
15876 case AMDGPUISD::CLAMP:
15877 case AMDGPUISD::FMED3:
15878 case AMDGPUISD::FMAX3:
15879 case AMDGPUISD::FMIN3:
15880 case AMDGPUISD::FMAXIMUM3:
15881 case AMDGPUISD::FMINIMUM3: {
15882 // FIXME: Shouldn't treat the generic operations different based these.
15883 // However, we aren't really required to flush the result from
15884 // minnum/maxnum..
15885
15886 // snans will be quieted, so we only need to worry about denormals.
15887 if (Subtarget->supportsMinMaxDenormModes() ||
15888 // FIXME: denormalsEnabledForType is broken for dynamic
15889 denormalsEnabledForType(DAG, VT: Op.getValueType()))
15890 return true;
15891
15892 // Flushing may be required.
15893 // In pre-GFX9 targets V_MIN_F32 and others do not flush denorms. For such
15894 // targets need to check their input recursively.
15895
15896 // FIXME: Does this apply with clamp? It's implemented with max.
15897 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
15898 if (!isCanonicalized(DAG, Op: Op.getOperand(i: I), UserFlags: MaxDepth - 1))
15899 return false;
15900 }
15901
15902 return true;
15903 }
15904 case ISD::SELECT: {
15905 return isCanonicalized(DAG, Op: Op.getOperand(i: 1), UserFlags: MaxDepth - 1) &&
15906 isCanonicalized(DAG, Op: Op.getOperand(i: 2), UserFlags: MaxDepth - 1);
15907 }
15908 case ISD::BUILD_VECTOR: {
15909 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
15910 SDValue SrcOp = Op.getOperand(i);
15911 if (!isCanonicalized(DAG, Op: SrcOp, UserFlags: MaxDepth - 1))
15912 return false;
15913 }
15914
15915 return true;
15916 }
15917 case ISD::EXTRACT_VECTOR_ELT:
15918 case ISD::EXTRACT_SUBVECTOR: {
15919 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1);
15920 }
15921 case ISD::INSERT_VECTOR_ELT: {
15922 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1) &&
15923 isCanonicalized(DAG, Op: Op.getOperand(i: 1), UserFlags: MaxDepth - 1);
15924 }
15925 case ISD::UNDEF:
15926 // Could be anything.
15927 return false;
15928
15929 case ISD::BITCAST:
15930 // TODO: This is incorrect as it loses track of the operand's type. We may
15931 // end up effectively bitcasting from f32 to v2f16 or vice versa, and the
15932 // same bits that are canonicalized in one type need not be in the other.
15933 return isCanonicalized(DAG, Op: Op.getOperand(i: 0), UserFlags: MaxDepth - 1);
15934 case ISD::TRUNCATE: {
15935 // Hack round the mess we make when legalizing extract_vector_elt
15936 if (Op.getValueType() == MVT::i16) {
15937 SDValue TruncSrc = Op.getOperand(i: 0);
15938 if (TruncSrc.getValueType() == MVT::i32 &&
15939 TruncSrc.getOpcode() == ISD::BITCAST &&
15940 TruncSrc.getOperand(i: 0).getValueType() == MVT::v2f16) {
15941 return isCanonicalized(DAG, Op: TruncSrc.getOperand(i: 0), UserFlags: MaxDepth - 1);
15942 }
15943 }
15944 return false;
15945 }
15946 case ISD::INTRINSIC_WO_CHAIN: {
15947 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
15948 // TODO: Handle more intrinsics
15949 switch (IntrinsicID) {
15950 case Intrinsic::amdgcn_cvt_pkrtz:
15951 case Intrinsic::amdgcn_cubeid:
15952 case Intrinsic::amdgcn_frexp_mant:
15953 case Intrinsic::amdgcn_fdot2:
15954 case Intrinsic::amdgcn_rcp:
15955 case Intrinsic::amdgcn_rsq:
15956 case Intrinsic::amdgcn_rsq_clamp:
15957 case Intrinsic::amdgcn_rcp_legacy:
15958 case Intrinsic::amdgcn_rsq_legacy:
15959 case Intrinsic::amdgcn_trig_preop:
15960 case Intrinsic::amdgcn_tanh:
15961 case Intrinsic::amdgcn_log:
15962 case Intrinsic::amdgcn_exp2:
15963 case Intrinsic::amdgcn_sqrt:
15964 return true;
15965 default:
15966 break;
15967 }
15968
15969 break;
15970 }
15971 default:
15972 break;
15973 }
15974
15975 // FIXME: denormalsEnabledForType is broken for dynamic
15976 return denormalsEnabledForType(DAG, VT: Op.getValueType()) &&
15977 (UserFlags.hasNoNaNs() || DAG.isKnownNeverSNaN(Op));
15978}
15979
15980bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF,
15981 unsigned MaxDepth) const {
15982 const MachineRegisterInfo &MRI = MF.getRegInfo();
15983 MachineInstr *MI = MRI.getVRegDef(Reg);
15984 unsigned Opcode = MI->getOpcode();
15985
15986 if (Opcode == AMDGPU::G_FCANONICALIZE)
15987 return true;
15988
15989 std::optional<FPValueAndVReg> FCR;
15990 // Constant splat (can be padded with undef) or scalar constant.
15991 if (mi_match(R: Reg, MRI, P: MIPatternMatch::m_GFCstOrSplat(FPValReg&: FCR))) {
15992 if (FCR->Value.isSignaling())
15993 return false;
15994 if (!FCR->Value.isDenormal())
15995 return true;
15996
15997 DenormalMode Mode = MF.getDenormalMode(FPType: FCR->Value.getSemantics());
15998 return Mode == DenormalMode::getIEEE();
15999 }
16000
16001 if (MaxDepth == 0)
16002 return false;
16003
16004 switch (Opcode) {
16005 case AMDGPU::G_FADD:
16006 case AMDGPU::G_FSUB:
16007 case AMDGPU::G_FMUL:
16008 case AMDGPU::G_FCEIL:
16009 case AMDGPU::G_FFLOOR:
16010 case AMDGPU::G_FRINT:
16011 case AMDGPU::G_FNEARBYINT:
16012 case AMDGPU::G_INTRINSIC_FPTRUNC_ROUND:
16013 case AMDGPU::G_INTRINSIC_TRUNC:
16014 case AMDGPU::G_INTRINSIC_ROUNDEVEN:
16015 case AMDGPU::G_FMA:
16016 case AMDGPU::G_FMAD:
16017 case AMDGPU::G_FSQRT:
16018 case AMDGPU::G_FDIV:
16019 case AMDGPU::G_FREM:
16020 case AMDGPU::G_FPOW:
16021 case AMDGPU::G_FPEXT:
16022 case AMDGPU::G_FLOG:
16023 case AMDGPU::G_FLOG2:
16024 case AMDGPU::G_FLOG10:
16025 case AMDGPU::G_FPTRUNC:
16026 case AMDGPU::G_AMDGPU_RCP_IFLAG:
16027 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0:
16028 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1:
16029 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2:
16030 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3:
16031 return true;
16032 case AMDGPU::G_FNEG:
16033 case AMDGPU::G_FABS:
16034 case AMDGPU::G_FCOPYSIGN:
16035 return isCanonicalized(Reg: MI->getOperand(i: 1).getReg(), MF, MaxDepth: MaxDepth - 1);
16036 case AMDGPU::G_FMINNUM:
16037 case AMDGPU::G_FMAXNUM:
16038 case AMDGPU::G_FMINNUM_IEEE:
16039 case AMDGPU::G_FMAXNUM_IEEE:
16040 case AMDGPU::G_FMINIMUM:
16041 case AMDGPU::G_FMAXIMUM:
16042 case AMDGPU::G_FMINIMUMNUM:
16043 case AMDGPU::G_FMAXIMUMNUM: {
16044 if (Subtarget->supportsMinMaxDenormModes() ||
16045 // FIXME: denormalsEnabledForType is broken for dynamic
16046 denormalsEnabledForType(Ty: MRI.getType(Reg), MF))
16047 return true;
16048
16049 [[fallthrough]];
16050 }
16051 case AMDGPU::G_BUILD_VECTOR:
16052 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands()))
16053 if (!isCanonicalized(Reg: MO.getReg(), MF, MaxDepth: MaxDepth - 1))
16054 return false;
16055 return true;
16056 case AMDGPU::G_INTRINSIC:
16057 case AMDGPU::G_INTRINSIC_CONVERGENT:
16058 switch (cast<GIntrinsic>(Val: MI)->getIntrinsicID()) {
16059 case Intrinsic::amdgcn_fmul_legacy:
16060 case Intrinsic::amdgcn_fmad_ftz:
16061 case Intrinsic::amdgcn_sqrt:
16062 case Intrinsic::amdgcn_fmed3:
16063 case Intrinsic::amdgcn_sin:
16064 case Intrinsic::amdgcn_cos:
16065 case Intrinsic::amdgcn_log:
16066 case Intrinsic::amdgcn_exp2:
16067 case Intrinsic::amdgcn_log_clamp:
16068 case Intrinsic::amdgcn_rcp:
16069 case Intrinsic::amdgcn_rcp_legacy:
16070 case Intrinsic::amdgcn_rsq:
16071 case Intrinsic::amdgcn_rsq_clamp:
16072 case Intrinsic::amdgcn_rsq_legacy:
16073 case Intrinsic::amdgcn_div_scale:
16074 case Intrinsic::amdgcn_div_fmas:
16075 case Intrinsic::amdgcn_div_fixup:
16076 case Intrinsic::amdgcn_fract:
16077 case Intrinsic::amdgcn_cvt_pkrtz:
16078 case Intrinsic::amdgcn_cubeid:
16079 case Intrinsic::amdgcn_cubema:
16080 case Intrinsic::amdgcn_cubesc:
16081 case Intrinsic::amdgcn_cubetc:
16082 case Intrinsic::amdgcn_frexp_mant:
16083 case Intrinsic::amdgcn_fdot2:
16084 case Intrinsic::amdgcn_trig_preop:
16085 case Intrinsic::amdgcn_tanh:
16086 return true;
16087 default:
16088 break;
16089 }
16090
16091 [[fallthrough]];
16092 default:
16093 return false;
16094 }
16095
16096 llvm_unreachable("invalid operation");
16097}
16098
16099// Constant fold canonicalize.
16100SDValue SITargetLowering::getCanonicalConstantFP(SelectionDAG &DAG,
16101 const SDLoc &SL, EVT VT,
16102 const APFloat &C) const {
16103 // Flush denormals to 0 if not enabled.
16104 if (C.isDenormal()) {
16105 DenormalMode Mode =
16106 DAG.getMachineFunction().getDenormalMode(FPType: C.getSemantics());
16107 if (Mode == DenormalMode::getPreserveSign()) {
16108 return DAG.getConstantFP(
16109 Val: APFloat::getZero(Sem: C.getSemantics(), Negative: C.isNegative()), DL: SL, VT);
16110 }
16111
16112 if (Mode != DenormalMode::getIEEE())
16113 return SDValue();
16114 }
16115
16116 if (C.isNaN()) {
16117 APFloat CanonicalQNaN = APFloat::getQNaN(Sem: C.getSemantics());
16118 if (C.isSignaling()) {
16119 // Quiet a signaling NaN.
16120 // FIXME: Is this supposed to preserve payload bits?
16121 return DAG.getConstantFP(Val: CanonicalQNaN, DL: SL, VT);
16122 }
16123
16124 // Make sure it is the canonical NaN bitpattern.
16125 //
16126 // TODO: Can we use -1 as the canonical NaN value since it's an inline
16127 // immediate?
16128 if (C.bitcastToAPInt() != CanonicalQNaN.bitcastToAPInt())
16129 return DAG.getConstantFP(Val: CanonicalQNaN, DL: SL, VT);
16130 }
16131
16132 // Already canonical.
16133 return DAG.getConstantFP(Val: C, DL: SL, VT);
16134}
16135
16136static bool vectorEltWillFoldAway(SDValue Op) {
16137 return Op.isUndef() || isa<ConstantFPSDNode>(Val: Op);
16138}
16139
16140SDValue
16141SITargetLowering::performFCanonicalizeCombine(SDNode *N,
16142 DAGCombinerInfo &DCI) const {
16143 SelectionDAG &DAG = DCI.DAG;
16144 SDValue N0 = N->getOperand(Num: 0);
16145 EVT VT = N->getValueType(ResNo: 0);
16146
16147 // fcanonicalize undef -> qnan
16148 if (N0.isUndef()) {
16149 APFloat QNaN = APFloat::getQNaN(Sem: VT.getFltSemantics());
16150 return DAG.getConstantFP(Val: QNaN, DL: SDLoc(N), VT);
16151 }
16152
16153 if (ConstantFPSDNode *CFP = isConstOrConstSplatFP(N: N0)) {
16154 EVT VT = N->getValueType(ResNo: 0);
16155 return getCanonicalConstantFP(DAG, SL: SDLoc(N), VT, C: CFP->getValueAPF());
16156 }
16157
16158 // fcanonicalize (build_vector x, k) -> build_vector (fcanonicalize x),
16159 // (fcanonicalize k)
16160 //
16161 // fcanonicalize (build_vector x, undef) -> build_vector (fcanonicalize x), 0
16162
16163 // TODO: This could be better with wider vectors that will be split to v2f16,
16164 // and to consider uses since there aren't that many packed operations.
16165 if (N0.getOpcode() == ISD::BUILD_VECTOR && VT == MVT::v2f16 &&
16166 isTypeLegal(VT: MVT::v2f16)) {
16167 SDLoc SL(N);
16168 SDValue NewElts[2];
16169 SDValue Lo = N0.getOperand(i: 0);
16170 SDValue Hi = N0.getOperand(i: 1);
16171 EVT EltVT = Lo.getValueType();
16172
16173 if (vectorEltWillFoldAway(Op: Lo) || vectorEltWillFoldAway(Op: Hi)) {
16174 for (unsigned I = 0; I != 2; ++I) {
16175 SDValue Op = N0.getOperand(i: I);
16176 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
16177 NewElts[I] =
16178 getCanonicalConstantFP(DAG, SL, VT: EltVT, C: CFP->getValueAPF());
16179 } else if (Op.isUndef()) {
16180 // Handled below based on what the other operand is.
16181 NewElts[I] = Op;
16182 } else {
16183 NewElts[I] = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: SL, VT: EltVT, Operand: Op);
16184 }
16185 }
16186
16187 // If one half is undef, and one is constant, prefer a splat vector rather
16188 // than the normal qNaN. If it's a register, prefer 0.0 since that's
16189 // cheaper to use and may be free with a packed operation.
16190 if (NewElts[0].isUndef()) {
16191 if (isa<ConstantFPSDNode>(Val: NewElts[1]))
16192 NewElts[0] = isa<ConstantFPSDNode>(Val: NewElts[1])
16193 ? NewElts[1]
16194 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16195 }
16196
16197 if (NewElts[1].isUndef()) {
16198 NewElts[1] = isa<ConstantFPSDNode>(Val: NewElts[0])
16199 ? NewElts[0]
16200 : DAG.getConstantFP(Val: 0.0f, DL: SL, VT: EltVT);
16201 }
16202
16203 return DAG.getBuildVector(VT, DL: SL, Ops: NewElts);
16204 }
16205 }
16206
16207 return SDValue();
16208}
16209
16210static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
16211 switch (Opc) {
16212 case ISD::FMAXNUM:
16213 case ISD::FMAXNUM_IEEE:
16214 case ISD::FMAXIMUMNUM:
16215 return AMDGPUISD::FMAX3;
16216 case ISD::FMAXIMUM:
16217 return AMDGPUISD::FMAXIMUM3;
16218 case ISD::SMAX:
16219 return AMDGPUISD::SMAX3;
16220 case ISD::UMAX:
16221 return AMDGPUISD::UMAX3;
16222 case ISD::FMINNUM:
16223 case ISD::FMINNUM_IEEE:
16224 case ISD::FMINIMUMNUM:
16225 return AMDGPUISD::FMIN3;
16226 case ISD::FMINIMUM:
16227 return AMDGPUISD::FMINIMUM3;
16228 case ISD::SMIN:
16229 return AMDGPUISD::SMIN3;
16230 case ISD::UMIN:
16231 return AMDGPUISD::UMIN3;
16232 default:
16233 llvm_unreachable("Not a min/max opcode");
16234 }
16235}
16236
16237SDValue SITargetLowering::performIntMed3ImmCombine(SelectionDAG &DAG,
16238 const SDLoc &SL, SDValue Src,
16239 SDValue MinVal,
16240 SDValue MaxVal,
16241 bool Signed) const {
16242
16243 // med3 comes from
16244 // min(max(x, K0), K1), K0 < K1
16245 // max(min(x, K0), K1), K1 < K0
16246 //
16247 // "MinVal" and "MaxVal" respectively refer to the rhs of the
16248 // min/max op.
16249 ConstantSDNode *MinK = dyn_cast<ConstantSDNode>(Val&: MinVal);
16250 ConstantSDNode *MaxK = dyn_cast<ConstantSDNode>(Val&: MaxVal);
16251
16252 if (!MinK || !MaxK)
16253 return SDValue();
16254
16255 if (Signed) {
16256 if (MaxK->getAPIntValue().sge(RHS: MinK->getAPIntValue()))
16257 return SDValue();
16258 } else {
16259 if (MaxK->getAPIntValue().uge(RHS: MinK->getAPIntValue()))
16260 return SDValue();
16261 }
16262
16263 EVT VT = MinK->getValueType(ResNo: 0);
16264 unsigned Med3Opc = Signed ? AMDGPUISD::SMED3 : AMDGPUISD::UMED3;
16265 if (VT == MVT::i32 || (VT == MVT::i16 && Subtarget->hasMed3_16()))
16266 return DAG.getNode(Opcode: Med3Opc, DL: SL, VT, N1: Src, N2: MaxVal, N3: MinVal);
16267
16268 // Note: we could also extend to i32 and use i32 med3 if i16 med3 is
16269 // not available, but this is unlikely to be profitable as constants
16270 // will often need to be materialized & extended, especially on
16271 // pre-GFX10 where VOP3 instructions couldn't take literal operands.
16272 return SDValue();
16273}
16274
16275static ConstantFPSDNode *getSplatConstantFP(SDValue Op) {
16276 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op))
16277 return C;
16278
16279 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
16280 if (ConstantFPSDNode *C = BV->getConstantFPSplatNode())
16281 return C;
16282 }
16283
16284 return nullptr;
16285}
16286
16287SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG,
16288 const SDLoc &SL, SDValue Op0,
16289 SDValue Op1,
16290 bool IsKnownNoNaNs) const {
16291 ConstantFPSDNode *K1 = getSplatConstantFP(Op: Op1);
16292 if (!K1)
16293 return SDValue();
16294
16295 ConstantFPSDNode *K0 = getSplatConstantFP(Op: Op0.getOperand(i: 1));
16296 if (!K0)
16297 return SDValue();
16298
16299 // Ordered >= (although NaN inputs should have folded away by now).
16300 if (K0->getValueAPF() > K1->getValueAPF())
16301 return SDValue();
16302
16303 // med3 with a nan input acts like
16304 // v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32)
16305 //
16306 // So the result depends on whether the IEEE mode bit is enabled or not with a
16307 // signaling nan input.
16308 // ieee=1
16309 // s0 snan: yields s2
16310 // s1 snan: yields s2
16311 // s2 snan: qnan
16312
16313 // s0 qnan: min(s1, s2)
16314 // s1 qnan: min(s0, s2)
16315 // s2 qnan: min(s0, s1)
16316
16317 // ieee=0
16318 // s0 snan: min(s1, s2)
16319 // s1 snan: min(s0, s2)
16320 // s2 snan: qnan
16321
16322 // s0 qnan: min(s1, s2)
16323 // s1 qnan: min(s0, s2)
16324 // s2 qnan: min(s0, s1)
16325 const MachineFunction &MF = DAG.getMachineFunction();
16326 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16327
16328 // TODO: Check IEEE bit enabled. We can form fmed3 with IEEE=0 regardless of
16329 // whether the input is a signaling nan if op0 is fmaximum or fmaximumnum. We
16330 // can only form if op0 is fmaxnum_ieee if IEEE=1.
16331 EVT VT = Op0.getValueType();
16332 if (Info->getMode().DX10Clamp) {
16333 // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the
16334 // hardware fmed3 behavior converting to a min.
16335 // FIXME: Should this be allowing -0.0?
16336 if (K1->isOne() && K0->isPosZero())
16337 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Op0.getOperand(i: 0));
16338 }
16339
16340 // med3 for f16 is only available on gfx9+, and not available for v2f16.
16341 if (VT == MVT::f32 || (VT == MVT::f16 && Subtarget->hasMed3_16())) {
16342 // This isn't safe with signaling NaNs because in IEEE mode, min/max on a
16343 // signaling NaN gives a quiet NaN. The quiet NaN input to the min would
16344 // then give the other result, which is different from med3 with a NaN
16345 // input.
16346 SDValue Var = Op0.getOperand(i: 0);
16347 if (!IsKnownNoNaNs && !DAG.isKnownNeverSNaN(Op: Var))
16348 return SDValue();
16349
16350 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
16351
16352 if ((!K0->hasOneUse() || TII->isInlineConstant(Imm: K0->getValueAPF())) &&
16353 (!K1->hasOneUse() || TII->isInlineConstant(Imm: K1->getValueAPF()))) {
16354 return DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT: K0->getValueType(ResNo: 0), N1: Var,
16355 N2: SDValue(K0, 0), N3: SDValue(K1, 0));
16356 }
16357 }
16358
16359 return SDValue();
16360}
16361
16362/// \return true if the subtarget supports minimum3 and maximum3 with the given
16363/// base min/max opcode \p Opc for type \p VT.
16364static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc,
16365 EVT VT) {
16366 switch (Opc) {
16367 case ISD::FMINNUM:
16368 case ISD::FMAXNUM:
16369 case ISD::FMINNUM_IEEE:
16370 case ISD::FMAXNUM_IEEE:
16371 case ISD::FMINIMUMNUM:
16372 case ISD::FMAXIMUMNUM:
16373 case AMDGPUISD::FMIN_LEGACY:
16374 case AMDGPUISD::FMAX_LEGACY:
16375 return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()) ||
16376 (VT == MVT::v2f16 && Subtarget.hasMin3Max3PKF16());
16377 case ISD::FMINIMUM:
16378 case ISD::FMAXIMUM:
16379 return (VT == MVT::f32 && Subtarget.hasMinimum3Maximum3F32()) ||
16380 (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()) ||
16381 (VT == MVT::v2f16 && Subtarget.hasMinimum3Maximum3PKF16());
16382 case ISD::SMAX:
16383 case ISD::SMIN:
16384 case ISD::UMAX:
16385 case ISD::UMIN:
16386 return (VT == MVT::i32) || (VT == MVT::i16 && Subtarget.hasMin3Max3_16());
16387 default:
16388 return false;
16389 }
16390
16391 llvm_unreachable("not a min/max opcode");
16392}
16393
16394SDValue SITargetLowering::performMinMaxCombine(SDNode *N,
16395 DAGCombinerInfo &DCI) const {
16396 SelectionDAG &DAG = DCI.DAG;
16397
16398 EVT VT = N->getValueType(ResNo: 0);
16399 unsigned Opc = N->getOpcode();
16400 SDValue Op0 = N->getOperand(Num: 0);
16401 SDValue Op1 = N->getOperand(Num: 1);
16402
16403 // Only do this if the inner op has one use since this will just increases
16404 // register pressure for no benefit.
16405
16406 if (supportsMin3Max3(Subtarget: *Subtarget, Opc, VT)) {
16407 auto IsTreeWithCombinableChildren = [Opc](SDValue Op) {
16408 return (Op.getOperand(i: 0).getOpcode() == Opc &&
16409 Op.getOperand(i: 0).hasOneUse()) ||
16410 (Op.getOperand(i: 1).getOpcode() == Opc &&
16411 Op.getOperand(i: 1).hasOneUse());
16412 };
16413
16414 bool CanTreeCombineApply = Op0.getOpcode() == Opc && Op0.hasOneUse() &&
16415 Op1.getOpcode() == Opc && Op1.hasOneUse();
16416 bool HasCombinableTreeChild =
16417 CanTreeCombineApply && (IsTreeWithCombinableChildren(Op0) ||
16418 IsTreeWithCombinableChildren(Op1));
16419
16420 // Tree reduction: when both operands are the same min/max op, restructure
16421 // to keep a 2-op node on top so higher tree levels can still combine.
16422 //
16423 // max(max(a, b), max(c, d)) -> max(max3(a, b, c), d)
16424 // min(min(a, b), min(c, d)) -> min(min3(a, b, c), d)
16425 //
16426 // Defer when either inner op is a tree node with combinable children.
16427 if (CanTreeCombineApply && !HasCombinableTreeChild) {
16428 SDLoc DL(N);
16429 SDValue Inner =
16430 DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT, N1: Op0.getOperand(i: 0),
16431 N2: Op0.getOperand(i: 1), N3: Op1.getOperand(i: 0));
16432 return DAG.getNode(Opcode: Opc, DL, VT, N1: Inner, N2: Op1.getOperand(i: 1));
16433 }
16434
16435 // max(max(a, b), c) -> max3(a, b, c)
16436 // min(min(a, b), c) -> min3(a, b, c)
16437 // Deferred when Op0 is a tree node with combinable children.
16438 if (Op0.getOpcode() == Opc && Op0.hasOneUse() && !HasCombinableTreeChild) {
16439 SDLoc DL(N);
16440 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16441 N1: Op0.getOperand(i: 0), N2: Op0.getOperand(i: 1), N3: Op1);
16442 }
16443
16444 // Try commuted.
16445 // max(a, max(b, c)) -> max3(a, b, c)
16446 // min(a, min(b, c)) -> min3(a, b, c)
16447 // Deferred when Op1 is a tree node with combinable children.
16448 if (Op1.getOpcode() == Opc && Op1.hasOneUse() && !HasCombinableTreeChild) {
16449 SDLoc DL(N);
16450 return DAG.getNode(Opcode: minMaxOpcToMin3Max3Opc(Opc), DL, VT: N->getValueType(ResNo: 0),
16451 N1: Op0, N2: Op1.getOperand(i: 0), N3: Op1.getOperand(i: 1));
16452 }
16453 }
16454
16455 // umin(sffbh(x), bitwidth) -> sffbh(x) if x is known to be not 0 or -1.
16456 SDValue FfbhSrc;
16457 uint64_t Clamp = 0;
16458 if (Opc == ISD::UMIN &&
16459 sd_match(N: Op0,
16460 P: m_IntrinsicWOChain<Intrinsic::amdgcn_sffbh>(Opnds: m_Value(N&: FfbhSrc))) &&
16461 sd_match(N: Op1, P: m_ConstInt(V&: Clamp))) {
16462 unsigned BitWidth = FfbhSrc.getValueType().getScalarSizeInBits();
16463 if (Clamp >= BitWidth) {
16464 KnownBits Known = DAG.computeKnownBits(Op: FfbhSrc);
16465 if (Known.isNonZero() && Known.Zero.getBoolValue())
16466 return Op0;
16467 }
16468 }
16469
16470 // min(max(x, K0), K1), K0 < K1 -> med3(x, K0, K1)
16471 // max(min(x, K0), K1), K1 < K0 -> med3(x, K1, K0)
16472 if (Opc == ISD::SMIN && Op0.getOpcode() == ISD::SMAX && Op0.hasOneUse()) {
16473 if (SDValue Med3 = performIntMed3ImmCombine(
16474 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: true))
16475 return Med3;
16476 }
16477 if (Opc == ISD::SMAX && Op0.getOpcode() == ISD::SMIN && Op0.hasOneUse()) {
16478 if (SDValue Med3 = performIntMed3ImmCombine(
16479 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: true))
16480 return Med3;
16481 }
16482
16483 if (Opc == ISD::UMIN && Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
16484 if (SDValue Med3 = performIntMed3ImmCombine(
16485 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op1, MaxVal: Op0->getOperand(Num: 1), Signed: false))
16486 return Med3;
16487 }
16488 if (Opc == ISD::UMAX && Op0.getOpcode() == ISD::UMIN && Op0.hasOneUse()) {
16489 if (SDValue Med3 = performIntMed3ImmCombine(
16490 DAG, SL: SDLoc(N), Src: Op0->getOperand(Num: 0), MinVal: Op0->getOperand(Num: 1), MaxVal: Op1, Signed: false))
16491 return Med3;
16492 }
16493
16494 // if !is_snan(x):
16495 // fminnum(fmaxnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16496 // fminnum_ieee(fmaxnum_ieee(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16497 // fminnumnum(fmaxnumnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16498 // fmin_legacy(fmax_legacy(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1)
16499 if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) ||
16500 (Opc == ISD::FMINNUM_IEEE && Op0.getOpcode() == ISD::FMAXNUM_IEEE) ||
16501 (Opc == ISD::FMINIMUMNUM && Op0.getOpcode() == ISD::FMAXIMUMNUM) ||
16502 (Opc == AMDGPUISD::FMIN_LEGACY &&
16503 Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) &&
16504 (VT == MVT::f32 || VT == MVT::f64 ||
16505 (VT == MVT::f16 && Subtarget->has16BitInsts()) ||
16506 (VT == MVT::bf16 && Subtarget->hasBF16PackedInsts()) ||
16507 (VT == MVT::v2bf16 && Subtarget->hasBF16PackedInsts()) ||
16508 (VT == MVT::v2f16 && Subtarget->hasVOP3PInsts())) &&
16509 Op0.hasOneUse()) {
16510 if (SDValue Res = performFPMed3ImmCombine(DAG, SL: SDLoc(N), Op0, Op1,
16511 IsKnownNoNaNs: N->getFlags().hasNoNaNs()))
16512 return Res;
16513 }
16514
16515 // Prefer fminnum_ieee over fminimum. For gfx950, minimum/maximum are legal
16516 // for some types, but at a higher cost since it's implemented with a 3
16517 // operand form.
16518 const SDNodeFlags Flags = N->getFlags();
16519 if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) && Flags.hasNoNaNs() &&
16520 !Subtarget->hasIEEEMinimumMaximumInsts() &&
16521 isOperationLegal(Op: ISD::FMINNUM_IEEE, VT: VT.getScalarType())) {
16522 unsigned NewOpc =
16523 Opc == ISD::FMINIMUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
16524 return DAG.getNode(Opcode: NewOpc, DL: SDLoc(N), VT, N1: Op0, N2: Op1, Flags);
16525 }
16526
16527 return SDValue();
16528}
16529
16530static bool isClampZeroToOne(SDValue A, SDValue B) {
16531 if (ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A)) {
16532 if (ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B)) {
16533 // FIXME: Should this be allowing -0.0?
16534 return (CA->isPosZero() && CB->isOne()) ||
16535 (CA->isOne() && CB->isPosZero());
16536 }
16537 }
16538
16539 return false;
16540}
16541
16542// FIXME: Should only worry about snans for version with chain.
16543SDValue SITargetLowering::performFMed3Combine(SDNode *N,
16544 DAGCombinerInfo &DCI) const {
16545 EVT VT = N->getValueType(ResNo: 0);
16546 // v_med3_f32 and v_max_f32 behave identically wrt denorms, exceptions and
16547 // NaNs. With a NaN input, the order of the operands may change the result.
16548
16549 SelectionDAG &DAG = DCI.DAG;
16550 SDLoc SL(N);
16551
16552 SDValue Src0 = N->getOperand(Num: 0);
16553 SDValue Src1 = N->getOperand(Num: 1);
16554 SDValue Src2 = N->getOperand(Num: 2);
16555
16556 if (isClampZeroToOne(A: Src0, B: Src1)) {
16557 // const_a, const_b, x -> clamp is safe in all cases including signaling
16558 // nans.
16559 // FIXME: Should this be allowing -0.0?
16560 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src2);
16561 }
16562
16563 const MachineFunction &MF = DAG.getMachineFunction();
16564 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
16565
16566 // FIXME: dx10_clamp behavior assumed in instcombine. Should we really bother
16567 // handling no dx10-clamp?
16568 if (Info->getMode().DX10Clamp) {
16569 // If NaNs is clamped to 0, we are free to reorder the inputs.
16570
16571 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
16572 std::swap(a&: Src0, b&: Src1);
16573
16574 if (isa<ConstantFPSDNode>(Val: Src1) && !isa<ConstantFPSDNode>(Val: Src2))
16575 std::swap(a&: Src1, b&: Src2);
16576
16577 if (isa<ConstantFPSDNode>(Val: Src0) && !isa<ConstantFPSDNode>(Val: Src1))
16578 std::swap(a&: Src0, b&: Src1);
16579
16580 if (isClampZeroToOne(A: Src1, B: Src2))
16581 return DAG.getNode(Opcode: AMDGPUISD::CLAMP, DL: SL, VT, Operand: Src0);
16582 }
16583
16584 return SDValue();
16585}
16586
16587SDValue SITargetLowering::performCvtPkRTZCombine(SDNode *N,
16588 DAGCombinerInfo &DCI) const {
16589 SDValue Src0 = N->getOperand(Num: 0);
16590 SDValue Src1 = N->getOperand(Num: 1);
16591 if (Src0.isUndef() && Src1.isUndef())
16592 return DCI.DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
16593 return SDValue();
16594}
16595
16596// Check if EXTRACT_VECTOR_ELT/INSERT_VECTOR_ELT (<n x e>, var-idx) should be
16597// expanded into a set of cmp/select instructions.
16598bool SITargetLowering::shouldExpandVectorDynExt(unsigned EltSize,
16599 unsigned NumElem,
16600 bool IsDivergentIdx,
16601 const GCNSubtarget *Subtarget) {
16602 if (UseDivergentRegisterIndexing)
16603 return false;
16604
16605 unsigned VecSize = EltSize * NumElem;
16606
16607 // Sub-dword vectors of size 2 dword or less have better implementation.
16608 if (VecSize <= 64 && EltSize < 32)
16609 return false;
16610
16611 // Always expand the rest of sub-dword instructions, otherwise it will be
16612 // lowered via memory.
16613 if (EltSize < 32)
16614 return true;
16615
16616 // Always do this if var-idx is divergent, otherwise it will become a loop.
16617 if (IsDivergentIdx)
16618 return true;
16619
16620 // Large vectors would yield too many compares and v_cndmask_b32 instructions.
16621 unsigned NumInsts = NumElem /* Number of compares */ +
16622 ((EltSize + 31) / 32) * NumElem /* Number of cndmasks */;
16623
16624 // On some architectures (GFX9) movrel is not available and it's better
16625 // to expand.
16626 if (Subtarget->useVGPRIndexMode())
16627 return NumInsts <= 16;
16628
16629 // If movrel is available, use it instead of expanding for vector of 8
16630 // elements.
16631 if (Subtarget->hasMovrel())
16632 return NumInsts <= 15;
16633
16634 return true;
16635}
16636
16637bool SITargetLowering::shouldExpandVectorDynExt(SDNode *N) const {
16638 SDValue Idx = N->getOperand(Num: N->getNumOperands() - 1);
16639 if (isa<ConstantSDNode>(Val: Idx))
16640 return false;
16641
16642 SDValue Vec = N->getOperand(Num: 0);
16643 EVT VecVT = Vec.getValueType();
16644 EVT EltVT = VecVT.getVectorElementType();
16645 unsigned EltSize = EltVT.getSizeInBits();
16646 unsigned NumElem = VecVT.getVectorNumElements();
16647
16648 return SITargetLowering::shouldExpandVectorDynExt(
16649 EltSize, NumElem, IsDivergentIdx: Idx->isDivergent(), Subtarget: getSubtarget());
16650}
16651
16652SDValue
16653SITargetLowering::performExtractVectorEltCombine(SDNode *N,
16654 DAGCombinerInfo &DCI) const {
16655 SDValue Vec = N->getOperand(Num: 0);
16656 SelectionDAG &DAG = DCI.DAG;
16657
16658 EVT VecVT = Vec.getValueType();
16659 EVT VecEltVT = VecVT.getVectorElementType();
16660 EVT ResVT = N->getValueType(ResNo: 0);
16661
16662 unsigned VecSize = VecVT.getSizeInBits();
16663 unsigned VecEltSize = VecEltVT.getSizeInBits();
16664
16665 if ((Vec.getOpcode() == ISD::FNEG || Vec.getOpcode() == ISD::FABS) &&
16666 allUsesHaveSourceMods(N)) {
16667 SDLoc SL(N);
16668 SDValue Idx = N->getOperand(Num: 1);
16669 SDValue Elt =
16670 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec.getOperand(i: 0), N2: Idx);
16671 return DAG.getNode(Opcode: Vec.getOpcode(), DL: SL, VT: ResVT, Operand: Elt);
16672 }
16673
16674 // (extract_vector_element (and {y0, y1}, (build_vector 0x1f, 0x1f)), index)
16675 // -> (and (extract_vector_element {y0, y1}, index), 0x1f)
16676 // There are optimisations to transform 64-bit shifts into 32-bit shifts
16677 // depending on the shift operand. See e.g. performSraCombine().
16678 // This combine ensures that the optimisation is compatible with v2i32
16679 // legalised AND.
16680 if (VecVT == MVT::v2i32 && Vec->getOpcode() == ISD::AND &&
16681 Vec->getOperand(Num: 1)->getOpcode() == ISD::BUILD_VECTOR) {
16682
16683 const ConstantSDNode *C = isConstOrConstSplat(N: Vec.getOperand(i: 1));
16684 if (!C || C->getZExtValue() != 0x1f)
16685 return SDValue();
16686
16687 SDLoc SL(N);
16688 SDValue AndMask = DAG.getConstant(Val: 0x1f, DL: SL, VT: MVT::i32);
16689 SDValue EVE = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32,
16690 N1: Vec->getOperand(Num: 0), N2: N->getOperand(Num: 1));
16691 SDValue A = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: EVE, N2: AndMask);
16692 DAG.ReplaceAllUsesWith(From: N, To: A.getNode());
16693 }
16694
16695 // ScalarRes = EXTRACT_VECTOR_ELT ((vector-BINOP Vec1, Vec2), Idx)
16696 // =>
16697 // Vec1Elt = EXTRACT_VECTOR_ELT(Vec1, Idx)
16698 // Vec2Elt = EXTRACT_VECTOR_ELT(Vec2, Idx)
16699 // ScalarRes = scalar-BINOP Vec1Elt, Vec2Elt
16700 if (Vec.hasOneUse() && DCI.isBeforeLegalize() && VecEltVT == ResVT) {
16701 SDLoc SL(N);
16702 SDValue Idx = N->getOperand(Num: 1);
16703 unsigned Opc = Vec.getOpcode();
16704
16705 switch (Opc) {
16706 default:
16707 break;
16708 // TODO: Support other binary operations.
16709 case ISD::FADD:
16710 case ISD::FSUB:
16711 case ISD::FMUL:
16712 case ISD::ADD:
16713 case ISD::UMIN:
16714 case ISD::UMAX:
16715 case ISD::SMIN:
16716 case ISD::SMAX:
16717 case ISD::FMAXNUM:
16718 case ISD::FMINNUM:
16719 case ISD::FMAXNUM_IEEE:
16720 case ISD::FMINNUM_IEEE:
16721 case ISD::FMAXIMUM:
16722 case ISD::FMINIMUM: {
16723 SDValue Elt0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
16724 N1: Vec.getOperand(i: 0), N2: Idx);
16725 SDValue Elt1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT,
16726 N1: Vec.getOperand(i: 1), N2: Idx);
16727
16728 DCI.AddToWorklist(N: Elt0.getNode());
16729 DCI.AddToWorklist(N: Elt1.getNode());
16730 return DAG.getNode(Opcode: Opc, DL: SL, VT: ResVT, N1: Elt0, N2: Elt1, Flags: Vec->getFlags());
16731 }
16732 }
16733 }
16734
16735 // EXTRACT_VECTOR_ELT (<n x e>, var-idx) => n x select (e, const-idx)
16736 if (shouldExpandVectorDynExt(N)) {
16737 SDLoc SL(N);
16738 SDValue Idx = N->getOperand(Num: 1);
16739 SDValue V;
16740 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
16741 SDValue IC = DAG.getVectorIdxConstant(Val: I, DL: SL);
16742 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: ResVT, N1: Vec, N2: IC);
16743 if (I == 0)
16744 V = Elt;
16745 else
16746 V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Elt, False: V, Cond: ISD::SETEQ);
16747 }
16748 return V;
16749 }
16750
16751 // EXTRACT_VECTOR_ELT (v2i32 bitcast (i64/f64:k), Idx)
16752 // =>
16753 // i32:Lo(k) if Idx == 0, or
16754 // i32:Hi(k) if Idx == 1
16755 auto *Idx = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
16756 if (Vec.getOpcode() == ISD::BITCAST && VecVT == MVT::v2i32 && Idx) {
16757 SDLoc SL(N);
16758 SDValue PeekThrough = Vec.getOperand(i: 0);
16759 auto *KImm = dyn_cast<ConstantSDNode>(Val&: PeekThrough);
16760 if (KImm && KImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
16761 uint64_t KImmValue = KImm->getZExtValue();
16762 return DAG.getConstant(
16763 Val: (KImmValue >> (32 * Idx->getZExtValue())) & 0xffffffff, DL: SL, VT: MVT::i32);
16764 }
16765 auto *KFPImm = dyn_cast<ConstantFPSDNode>(Val&: PeekThrough);
16766 if (KFPImm && KFPImm->getValueType(ResNo: 0).getSizeInBits() == 64) {
16767 uint64_t KFPImmValue =
16768 KFPImm->getValueAPF().bitcastToAPInt().getZExtValue();
16769 return DAG.getConstant(Val: (KFPImmValue >> (32 * Idx->getZExtValue())) &
16770 0xffffffff,
16771 DL: SL, VT: MVT::i32);
16772 }
16773 }
16774
16775 if (!DCI.isBeforeLegalize())
16776 return SDValue();
16777
16778 // Try to turn sub-dword accesses of vectors into accesses of the same 32-bit
16779 // elements. This exposes more load reduction opportunities by replacing
16780 // multiple small extract_vector_elements with a single 32-bit extract.
16781 if (isa<MemSDNode>(Val: Vec) && VecEltSize <= 16 && VecEltVT.isByteSized() &&
16782 VecSize > 32 && VecSize % 32 == 0 && Idx) {
16783 EVT NewVT = getEquivalentMemType(Context&: *DAG.getContext(), VT: VecVT);
16784
16785 unsigned BitIndex = Idx->getZExtValue() * VecEltSize;
16786 unsigned EltIdx = BitIndex / 32;
16787 unsigned LeftoverBitIdx = BitIndex % 32;
16788 SDLoc SL(N);
16789
16790 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewVT, Operand: Vec);
16791 DCI.AddToWorklist(N: Cast.getNode());
16792
16793 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Cast,
16794 N2: DAG.getConstant(Val: EltIdx, DL: SL, VT: MVT::i32));
16795 DCI.AddToWorklist(N: Elt.getNode());
16796 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: MVT::i32, N1: Elt,
16797 N2: DAG.getConstant(Val: LeftoverBitIdx, DL: SL, VT: MVT::i32));
16798 DCI.AddToWorklist(N: Srl.getNode());
16799
16800 EVT VecEltAsIntVT = VecEltVT.changeTypeToInteger();
16801 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: VecEltAsIntVT, Operand: Srl);
16802 DCI.AddToWorklist(N: Trunc.getNode());
16803
16804 if (VecEltVT == ResVT) {
16805 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: VecEltVT, Operand: Trunc);
16806 }
16807
16808 assert(ResVT.isScalarInteger());
16809 return DAG.getAnyExtOrTrunc(Op: Trunc, DL: SL, VT: ResVT);
16810 }
16811
16812 return SDValue();
16813}
16814
16815SDValue
16816SITargetLowering::performInsertVectorEltCombine(SDNode *N,
16817 DAGCombinerInfo &DCI) const {
16818 SDValue Vec = N->getOperand(Num: 0);
16819 SDValue Idx = N->getOperand(Num: 2);
16820 EVT VecVT = Vec.getValueType();
16821 EVT EltVT = VecVT.getVectorElementType();
16822
16823 // INSERT_VECTOR_ELT (<n x e>, var-idx)
16824 // => BUILD_VECTOR n x select (e, const-idx)
16825 if (!shouldExpandVectorDynExt(N))
16826 return SDValue();
16827
16828 SelectionDAG &DAG = DCI.DAG;
16829 SDLoc SL(N);
16830 SDValue Ins = N->getOperand(Num: 1);
16831 EVT IdxVT = Idx.getValueType();
16832
16833 SmallVector<SDValue, 16> Ops;
16834 for (unsigned I = 0, E = VecVT.getVectorNumElements(); I < E; ++I) {
16835 SDValue IC = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
16836 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: EltVT, N1: Vec, N2: IC);
16837 SDValue V = DAG.getSelectCC(DL: SL, LHS: Idx, RHS: IC, True: Ins, False: Elt, Cond: ISD::SETEQ);
16838 Ops.push_back(Elt: V);
16839 }
16840
16841 return DAG.getBuildVector(VT: VecVT, DL: SL, Ops);
16842}
16843
16844/// Return the source of an fp_extend from f16 to f32, or a converted FP
16845/// constant.
16846static SDValue strictFPExtFromF16(SelectionDAG &DAG, SDValue Src) {
16847 if (Src.getOpcode() == ISD::FP_EXTEND &&
16848 Src.getOperand(i: 0).getValueType() == MVT::f16) {
16849 return Src.getOperand(i: 0);
16850 }
16851
16852 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
16853 APFloat Val = CFP->getValueAPF();
16854 bool LosesInfo = true;
16855 Val.convert(ToSemantics: APFloat::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, losesInfo: &LosesInfo);
16856 if (!LosesInfo)
16857 return DAG.getConstantFP(Val, DL: SDLoc(Src), VT: MVT::f16);
16858 }
16859
16860 return SDValue();
16861}
16862
16863SDValue SITargetLowering::performFPRoundCombine(SDNode *N,
16864 DAGCombinerInfo &DCI) const {
16865 assert(Subtarget->has16BitInsts() && !Subtarget->hasMed3_16() &&
16866 "combine only useful on gfx8");
16867
16868 SDValue TruncSrc = N->getOperand(Num: 0);
16869 EVT VT = N->getValueType(ResNo: 0);
16870 if (VT != MVT::f16)
16871 return SDValue();
16872
16873 if (TruncSrc.getOpcode() != AMDGPUISD::FMED3 ||
16874 TruncSrc.getValueType() != MVT::f32 || !TruncSrc.hasOneUse())
16875 return SDValue();
16876
16877 SelectionDAG &DAG = DCI.DAG;
16878 SDLoc SL(N);
16879
16880 // Optimize f16 fmed3 pattern performed on f32. On gfx8 there is no f16 fmed3,
16881 // and expanding it with min/max saves 1 instruction vs. casting to f32 and
16882 // casting back.
16883
16884 // fptrunc (f32 (fmed3 (fpext f16:a, fpext f16:b, fpext f16:c))) =>
16885 // fmin(fmax(a, b), fmax(fmin(a, b), c))
16886 SDValue A = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 0));
16887 if (!A)
16888 return SDValue();
16889
16890 SDValue B = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 1));
16891 if (!B)
16892 return SDValue();
16893
16894 SDValue C = strictFPExtFromF16(DAG, Src: TruncSrc.getOperand(i: 2));
16895 if (!C)
16896 return SDValue();
16897
16898 // This changes signaling nan behavior. If an input is a signaling nan, it
16899 // would have been quieted by the fpext originally. We don't care because
16900 // these are unconstrained ops. If we needed to insert quieting canonicalizes
16901 // we would be worse off than just doing the promotion.
16902 SDValue A1 = DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: A, N2: B);
16903 SDValue B1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A, N2: B);
16904 SDValue C1 = DAG.getNode(Opcode: ISD::FMAXNUM_IEEE, DL: SL, VT, N1: A1, N2: C);
16905 return DAG.getNode(Opcode: ISD::FMINNUM_IEEE, DL: SL, VT, N1: B1, N2: C1);
16906}
16907
16908unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG,
16909 const SDNode *N0,
16910 const SDNode *N1) const {
16911 EVT VT = N0->getValueType(ResNo: 0);
16912
16913 // Only do this if we are not trying to support denormals. v_mad_f32 does not
16914 // support denormals ever.
16915 if (((VT == MVT::f32 &&
16916 denormalModeIsFlushAllF32(MF: DAG.getMachineFunction())) ||
16917 (VT == MVT::f16 && Subtarget->hasMadF16() &&
16918 denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction()))) &&
16919 isOperationLegal(Op: ISD::FMAD, VT))
16920 return ISD::FMAD;
16921
16922 const TargetOptions &Options = DAG.getTarget().Options;
16923 if ((Options.AllowFPOpFusion == FPOpFusion::Fast ||
16924 (N0->getFlags().hasAllowContract() &&
16925 N1->getFlags().hasAllowContract())) &&
16926 isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
16927 return ISD::FMA;
16928 }
16929
16930 return 0;
16931}
16932
16933// For a reassociatable opcode perform:
16934// op x, (op y, z) -> op (op x, z), y, if x and z are uniform
16935SDValue SITargetLowering::reassociateScalarOps(SDNode *N,
16936 SelectionDAG &DAG) const {
16937 EVT VT = N->getValueType(ResNo: 0);
16938 if (VT != MVT::i32 && VT != MVT::i64)
16939 return SDValue();
16940
16941 if (DAG.isBaseWithConstantOffset(Op: SDValue(N, 0)))
16942 return SDValue();
16943
16944 unsigned Opc = N->getOpcode();
16945 SDValue Op0 = N->getOperand(Num: 0);
16946 SDValue Op1 = N->getOperand(Num: 1);
16947
16948 if (!(Op0->isDivergent() ^ Op1->isDivergent()))
16949 return SDValue();
16950
16951 if (Op0->isDivergent())
16952 std::swap(a&: Op0, b&: Op1);
16953
16954 if (Op1.getOpcode() != Opc || !Op1.hasOneUse())
16955 return SDValue();
16956
16957 SDValue Op2 = Op1.getOperand(i: 1);
16958 Op1 = Op1.getOperand(i: 0);
16959 if (!(Op1->isDivergent() ^ Op2->isDivergent()))
16960 return SDValue();
16961
16962 if (Op1->isDivergent())
16963 std::swap(a&: Op1, b&: Op2);
16964
16965 SDLoc SL(N);
16966 SDValue Add1 = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Op0, N2: Op1);
16967 return DAG.getNode(Opcode: Opc, DL: SL, VT, N1: Add1, N2: Op2);
16968}
16969
16970static SDValue getMad64_32(SelectionDAG &DAG, const SDLoc &SL, EVT VT,
16971 SDValue N0, SDValue N1, SDValue N2, bool Signed) {
16972 unsigned MadOpc = Signed ? AMDGPUISD::MAD_I64_I32 : AMDGPUISD::MAD_U64_U32;
16973 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i1);
16974 SDValue Mad = DAG.getNode(Opcode: MadOpc, DL: SL, VTList: VTs, N1: N0, N2: N1, N3: N2);
16975 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Mad);
16976}
16977
16978// Fold
16979// y = lshr i64 x, 32
16980// res = add (mul i64 y, Const), x where "Const" is a 64-bit constant
16981// with Const.hi == -1
16982// To
16983// res = mad_u64_u32 y.lo ,Const.lo, x.lo
16984static SDValue tryFoldMADwithSRL(SelectionDAG &DAG, const SDLoc &SL,
16985 SDValue MulLHS, SDValue MulRHS,
16986 SDValue AddRHS) {
16987 if (MulRHS.getOpcode() == ISD::SRL)
16988 std::swap(a&: MulLHS, b&: MulRHS);
16989
16990 if (MulLHS.getValueType() != MVT::i64 || MulLHS.getOpcode() != ISD::SRL)
16991 return SDValue();
16992
16993 ConstantSDNode *ShiftVal = dyn_cast<ConstantSDNode>(Val: MulLHS.getOperand(i: 1));
16994 if (!ShiftVal || ShiftVal->getAsZExtVal() != 32 ||
16995 MulLHS.getOperand(i: 0) != AddRHS)
16996 return SDValue();
16997
16998 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: MulRHS.getNode());
16999 if (!Const || Hi_32(Value: Const->getZExtValue()) != uint32_t(-1))
17000 return SDValue();
17001
17002 SDValue ConstMul =
17003 DAG.getConstant(Val: Lo_32(Value: Const->getZExtValue()), DL: SL, VT: MVT::i32);
17004 return getMad64_32(DAG, SL, VT: MVT::i64,
17005 N0: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS), N1: ConstMul,
17006 N2: DAG.getZeroExtendInReg(Op: AddRHS, DL: SL, VT: MVT::i32), Signed: false);
17007}
17008
17009// Fold (add (mul x, y), z) --> (mad_[iu]64_[iu]32 x, y, z) plus high
17010// multiplies, if any.
17011//
17012// Full 64-bit multiplies that feed into an addition are lowered here instead
17013// of using the generic expansion. The generic expansion ends up with
17014// a tree of ADD nodes that prevents us from using the "add" part of the
17015// MAD instruction. The expansion produced here results in a chain of ADDs
17016// instead of a tree.
17017SDValue SITargetLowering::tryFoldToMad64_32(SDNode *N,
17018 DAGCombinerInfo &DCI) const {
17019 assert(N->isAnyAdd());
17020
17021 SelectionDAG &DAG = DCI.DAG;
17022 EVT VT = N->getValueType(ResNo: 0);
17023 SDLoc SL(N);
17024 SDValue LHS = N->getOperand(Num: 0);
17025 SDValue RHS = N->getOperand(Num: 1);
17026
17027 if (VT.isVector())
17028 return SDValue();
17029
17030 // S_MUL_HI_[IU]32 was added in gfx9, which allows us to keep the overall
17031 // result in scalar registers for uniform values.
17032 if (!N->isDivergent() && Subtarget->hasSMulHi())
17033 return SDValue();
17034
17035 unsigned NumBits = VT.getScalarSizeInBits();
17036 if (NumBits <= 32 || NumBits > 64)
17037 return SDValue();
17038
17039 if (LHS.getOpcode() != ISD::MUL) {
17040 assert(RHS.getOpcode() == ISD::MUL);
17041 std::swap(a&: LHS, b&: RHS);
17042 }
17043
17044 // Avoid the fold if it would unduly increase the number of multiplies due to
17045 // multiple uses, except on hardware with full-rate multiply-add (which is
17046 // part of full-rate 64-bit ops).
17047 if (!Subtarget->hasFullRate64Ops()) {
17048 unsigned NumUsers = 0;
17049 for (SDNode *User : LHS->users()) {
17050 // There is a use that does not feed into addition, so the multiply can't
17051 // be removed. We prefer MUL + ADD + ADDC over MAD + MUL.
17052 if (!User->isAnyAdd())
17053 return SDValue();
17054
17055 // We prefer 2xMAD over MUL + 2xADD + 2xADDC (code density), and prefer
17056 // MUL + 3xADD + 3xADDC over 3xMAD.
17057 ++NumUsers;
17058 if (NumUsers >= 3)
17059 return SDValue();
17060 }
17061 }
17062
17063 SDValue MulLHS = LHS.getOperand(i: 0);
17064 SDValue MulRHS = LHS.getOperand(i: 1);
17065 SDValue AddRHS = RHS;
17066
17067 if (SDValue FoldedMAD = tryFoldMADwithSRL(DAG, SL, MulLHS, MulRHS, AddRHS))
17068 return FoldedMAD;
17069
17070 // Always check whether operands are small unsigned values, since that
17071 // knowledge is useful in more cases. Check for small signed values only if
17072 // doing so can unlock a shorter code sequence.
17073 bool MulLHSUnsigned32 = numBitsUnsigned(Op: MulLHS, DAG) <= 32;
17074 bool MulRHSUnsigned32 = numBitsUnsigned(Op: MulRHS, DAG) <= 32;
17075
17076 bool MulSignedLo = false;
17077 if (!MulLHSUnsigned32 || !MulRHSUnsigned32) {
17078 MulSignedLo =
17079 numBitsSigned(Op: MulLHS, DAG) <= 32 && numBitsSigned(Op: MulRHS, DAG) <= 32;
17080 }
17081
17082 // The operands and final result all have the same number of bits. If
17083 // operands need to be extended, they can be extended with garbage. The
17084 // resulting garbage in the high bits of the mad_[iu]64_[iu]32 result is
17085 // truncated away in the end.
17086 if (VT != MVT::i64) {
17087 MulLHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulLHS);
17088 MulRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: MulRHS);
17089 AddRHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i64, Operand: AddRHS);
17090 }
17091
17092 // The basic code generated is conceptually straightforward. Pseudo code:
17093 //
17094 // accum = mad_64_32 lhs.lo, rhs.lo, accum
17095 // accum.hi = add (mul lhs.hi, rhs.lo), accum.hi
17096 // accum.hi = add (mul lhs.lo, rhs.hi), accum.hi
17097 //
17098 // The second and third lines are optional, depending on whether the factors
17099 // are {sign,zero}-extended or not.
17100 //
17101 // The actual DAG is noisier than the pseudo code, but only due to
17102 // instructions that disassemble values into low and high parts, and
17103 // assemble the final result.
17104 SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
17105
17106 auto MulLHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulLHS);
17107 auto MulRHSLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: MulRHS);
17108 SDValue Accum =
17109 getMad64_32(DAG, SL, VT: MVT::i64, N0: MulLHSLo, N1: MulRHSLo, N2: AddRHS, Signed: MulSignedLo);
17110
17111 if (!MulSignedLo && (!MulLHSUnsigned32 || !MulRHSUnsigned32)) {
17112 auto [AccumLo, AccumHi] = DAG.SplitScalar(N: Accum, DL: SL, LoVT: MVT::i32, HiVT: MVT::i32);
17113
17114 if (!MulLHSUnsigned32) {
17115 auto MulLHSHi =
17116 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulLHS, N2: One);
17117 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSHi, N2: MulRHSLo);
17118 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17119 }
17120
17121 if (!MulRHSUnsigned32) {
17122 auto MulRHSHi =
17123 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SL, VT: MVT::i32, N1: MulRHS, N2: One);
17124 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: MVT::i32, N1: MulLHSLo, N2: MulRHSHi);
17125 AccumHi = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: MulHi, N2: AccumHi);
17126 }
17127
17128 Accum = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {AccumLo, AccumHi});
17129 Accum = DAG.getBitcast(VT: MVT::i64, V: Accum);
17130 }
17131
17132 if (VT != MVT::i64)
17133 Accum = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Accum);
17134 return Accum;
17135}
17136
17137SDValue
17138SITargetLowering::foldAddSub64WithZeroLowBitsTo32(SDNode *N,
17139 DAGCombinerInfo &DCI) const {
17140 SDValue RHS = N->getOperand(Num: 1);
17141 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
17142 if (!CRHS)
17143 return SDValue();
17144
17145 // TODO: Worth using computeKnownBits? Maybe expensive since it's so
17146 // common.
17147 uint64_t Val = CRHS->getZExtValue();
17148 if (countr_zero(Val) >= 32) {
17149 SelectionDAG &DAG = DCI.DAG;
17150 SDLoc SL(N);
17151 SDValue LHS = N->getOperand(Num: 0);
17152
17153 // Avoid carry machinery if we know the low half of the add does not
17154 // contribute to the final result.
17155 //
17156 // add i64:x, K if computeTrailingZeros(K) >= 32
17157 // => build_pair (add x.hi, K.hi), x.lo
17158
17159 // Breaking the 64-bit add here with this strange constant is unlikely
17160 // to interfere with addressing mode patterns.
17161
17162 SDValue Hi = getHiHalf64(Op: LHS, DAG);
17163 SDValue ConstHi32 = DAG.getConstant(Val: Hi_32(Value: Val), DL: SL, VT: MVT::i32);
17164 unsigned Opcode = N->getOpcode();
17165 if (Opcode == ISD::PTRADD)
17166 Opcode = ISD::ADD;
17167 SDValue AddHi =
17168 DAG.getNode(Opcode, DL: SL, VT: MVT::i32, N1: Hi, N2: ConstHi32, Flags: N->getFlags());
17169
17170 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: LHS);
17171 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: Lo, N2: AddHi);
17172 }
17173
17174 return SDValue();
17175}
17176
17177// Collect the ultimate src of each of the mul node's operands, and confirm
17178// each operand is 8 bytes.
17179static std::optional<ByteProvider<SDValue>>
17180handleMulOperand(const SDValue &MulOperand) {
17181 auto Byte0 = calculateByteProvider(Op: MulOperand, Index: 0, Depth: 0);
17182 if (!Byte0 || Byte0->isConstantZero()) {
17183 return std::nullopt;
17184 }
17185 auto Byte1 = calculateByteProvider(Op: MulOperand, Index: 1, Depth: 0);
17186 if (Byte1 && !Byte1->isConstantZero()) {
17187 return std::nullopt;
17188 }
17189 return Byte0;
17190}
17191
17192static unsigned addPermMasks(unsigned First, unsigned Second) {
17193 unsigned FirstCs = First & 0x0c0c0c0c;
17194 unsigned SecondCs = Second & 0x0c0c0c0c;
17195 unsigned FirstNoCs = First & ~0x0c0c0c0c;
17196 unsigned SecondNoCs = Second & ~0x0c0c0c0c;
17197
17198 assert((FirstCs & 0xFF) | (SecondCs & 0xFF));
17199 assert((FirstCs & 0xFF00) | (SecondCs & 0xFF00));
17200 assert((FirstCs & 0xFF0000) | (SecondCs & 0xFF0000));
17201 assert((FirstCs & 0xFF000000) | (SecondCs & 0xFF000000));
17202
17203 return (FirstNoCs | SecondNoCs) | (FirstCs & SecondCs);
17204}
17205
17206struct DotSrc {
17207 SDValue SrcOp;
17208 int64_t PermMask;
17209 int64_t DWordOffset;
17210};
17211
17212static void placeSources(ByteProvider<SDValue> &Src0,
17213 ByteProvider<SDValue> &Src1,
17214 SmallVectorImpl<DotSrc> &Src0s,
17215 SmallVectorImpl<DotSrc> &Src1s, int Step) {
17216
17217 assert(Src0.Src.has_value() && Src1.Src.has_value());
17218 // Src0s and Src1s are empty, just place arbitrarily.
17219 if (Step == 0) {
17220 Src0s.push_back(Elt: {.SrcOp: *Src0.Src, .PermMask: ((Src0.SrcOffset % 4) << 24) + 0x0c0c0c,
17221 .DWordOffset: Src0.SrcOffset / 4});
17222 Src1s.push_back(Elt: {.SrcOp: *Src1.Src, .PermMask: ((Src1.SrcOffset % 4) << 24) + 0x0c0c0c,
17223 .DWordOffset: Src1.SrcOffset / 4});
17224 return;
17225 }
17226
17227 for (int BPI = 0; BPI < 2; BPI++) {
17228 std::pair<ByteProvider<SDValue>, ByteProvider<SDValue>> BPP = {Src0, Src1};
17229 if (BPI == 1) {
17230 BPP = {Src1, Src0};
17231 }
17232 unsigned ZeroMask = 0x0c0c0c0c;
17233 unsigned FMask = 0xFF << (8 * (3 - Step));
17234
17235 unsigned FirstMask =
17236 (BPP.first.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17237 unsigned SecondMask =
17238 (BPP.second.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask);
17239 // Attempt to find Src vector which contains our SDValue, if so, add our
17240 // perm mask to the existing one. If we are unable to find a match for the
17241 // first SDValue, attempt to find match for the second.
17242 int FirstGroup = -1;
17243 for (int I = 0; I < 2; I++) {
17244 SmallVectorImpl<DotSrc> &Srcs = I == 0 ? Src0s : Src1s;
17245 auto MatchesFirst = [&BPP](DotSrc &IterElt) {
17246 return IterElt.SrcOp == *BPP.first.Src &&
17247 (IterElt.DWordOffset == (BPP.first.SrcOffset / 4));
17248 };
17249
17250 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesFirst);
17251 if (Match != Srcs.end()) {
17252 Match->PermMask = addPermMasks(First: FirstMask, Second: Match->PermMask);
17253 FirstGroup = I;
17254 break;
17255 }
17256 }
17257 if (FirstGroup != -1) {
17258 SmallVectorImpl<DotSrc> &Srcs = FirstGroup == 1 ? Src0s : Src1s;
17259 auto MatchesSecond = [&BPP](DotSrc &IterElt) {
17260 return IterElt.SrcOp == *BPP.second.Src &&
17261 (IterElt.DWordOffset == (BPP.second.SrcOffset / 4));
17262 };
17263 auto *Match = llvm::find_if(Range&: Srcs, P: MatchesSecond);
17264 if (Match != Srcs.end()) {
17265 Match->PermMask = addPermMasks(First: SecondMask, Second: Match->PermMask);
17266 } else
17267 Srcs.push_back(Elt: {.SrcOp: *BPP.second.Src, .PermMask: SecondMask, .DWordOffset: BPP.second.SrcOffset / 4});
17268 return;
17269 }
17270 }
17271
17272 // If we have made it here, then we could not find a match in Src0s or Src1s
17273 // for either Src0 or Src1, so just place them arbitrarily.
17274
17275 unsigned ZeroMask = 0x0c0c0c0c;
17276 unsigned FMask = 0xFF << (8 * (3 - Step));
17277
17278 Src0s.push_back(
17279 Elt: {.SrcOp: *Src0.Src,
17280 .PermMask: ((Src0.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17281 .DWordOffset: Src0.SrcOffset / 4});
17282 Src1s.push_back(
17283 Elt: {.SrcOp: *Src1.Src,
17284 .PermMask: ((Src1.SrcOffset % 4) << (8 * (3 - Step)) | (ZeroMask & ~FMask)),
17285 .DWordOffset: Src1.SrcOffset / 4});
17286}
17287
17288static SDValue resolveSources(SelectionDAG &DAG, SDLoc SL,
17289 SmallVectorImpl<DotSrc> &Srcs, bool IsSigned,
17290 bool IsAny) {
17291
17292 // If we just have one source, just permute it accordingly.
17293 if (Srcs.size() == 1) {
17294 auto *Elt = Srcs.begin();
17295 auto EltOp = getDWordFromOffset(DAG, SL, Src: Elt->SrcOp, DWordOffset: Elt->DWordOffset);
17296
17297 // v_perm will produce the original value
17298 if (Elt->PermMask == 0x3020100)
17299 return EltOp;
17300
17301 return DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17302 N3: DAG.getConstant(Val: Elt->PermMask, DL: SL, VT: MVT::i32));
17303 }
17304
17305 auto *FirstElt = Srcs.begin();
17306 auto *SecondElt = std::next(x: FirstElt);
17307
17308 SmallVector<SDValue, 2> Perms;
17309
17310 // If we have multiple sources in the chain, combine them via perms (using
17311 // calculated perm mask) and Ors.
17312 while (true) {
17313 auto FirstMask = FirstElt->PermMask;
17314 auto SecondMask = SecondElt->PermMask;
17315
17316 unsigned FirstCs = FirstMask & 0x0c0c0c0c;
17317 unsigned FirstPlusFour = FirstMask | 0x04040404;
17318 // 0x0c + 0x04 = 0x10, so anding with 0x0F will produced 0x00 for any
17319 // original 0x0C.
17320 FirstMask = (FirstPlusFour & 0x0F0F0F0F) | FirstCs;
17321
17322 auto PermMask = addPermMasks(First: FirstMask, Second: SecondMask);
17323 auto FirstVal =
17324 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17325 auto SecondVal =
17326 getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp, DWordOffset: SecondElt->DWordOffset);
17327
17328 Perms.push_back(Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: FirstVal,
17329 N2: SecondVal,
17330 N3: DAG.getConstant(Val: PermMask, DL: SL, VT: MVT::i32)));
17331
17332 FirstElt = std::next(x: SecondElt);
17333 if (FirstElt == Srcs.end())
17334 break;
17335
17336 SecondElt = std::next(x: FirstElt);
17337 // If we only have a FirstElt, then just combine that into the cumulative
17338 // source node.
17339 if (SecondElt == Srcs.end()) {
17340 auto EltOp =
17341 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17342
17343 Perms.push_back(
17344 Elt: DAG.getNode(Opcode: AMDGPUISD::PERM, DL: SL, VT: MVT::i32, N1: EltOp, N2: EltOp,
17345 N3: DAG.getConstant(Val: FirstElt->PermMask, DL: SL, VT: MVT::i32)));
17346 break;
17347 }
17348 }
17349
17350 assert(Perms.size() == 1 || Perms.size() == 2);
17351 return Perms.size() == 2
17352 ? DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Perms[0], N2: Perms[1])
17353 : Perms[0];
17354}
17355
17356static void fixMasks(SmallVectorImpl<DotSrc> &Srcs, unsigned ChainLength) {
17357 for (auto &[EntryVal, EntryMask, EntryOffset] : Srcs) {
17358 EntryMask = EntryMask >> ((4 - ChainLength) * 8);
17359 auto ZeroMask = ChainLength == 2 ? 0x0c0c0000 : 0x0c000000;
17360 EntryMask += ZeroMask;
17361 }
17362}
17363
17364static bool isMul(const SDValue Op) {
17365 auto Opcode = Op.getOpcode();
17366
17367 return (Opcode == ISD::MUL || Opcode == AMDGPUISD::MUL_U24 ||
17368 Opcode == AMDGPUISD::MUL_I24);
17369}
17370
17371static std::optional<bool>
17372checkDot4MulSignedness(const SDValue &N, ByteProvider<SDValue> &Src0,
17373 ByteProvider<SDValue> &Src1, const SDValue &S0Op,
17374 const SDValue &S1Op, const SelectionDAG &DAG) {
17375 // If we both ops are i8s (pre legalize-dag), then the signedness semantics
17376 // of the dot4 is irrelevant.
17377 if (S0Op.getValueSizeInBits() == 8 && S1Op.getValueSizeInBits() == 8)
17378 return false;
17379
17380 auto Known0 = DAG.computeKnownBits(Op: S0Op, Depth: 0);
17381 bool S0IsUnsigned = Known0.countMinLeadingZeros() > 0;
17382 bool S0IsSigned = Known0.countMinLeadingOnes() > 0;
17383 auto Known1 = DAG.computeKnownBits(Op: S1Op, Depth: 0);
17384 bool S1IsUnsigned = Known1.countMinLeadingZeros() > 0;
17385 bool S1IsSigned = Known1.countMinLeadingOnes() > 0;
17386
17387 assert(!(S0IsUnsigned && S0IsSigned));
17388 assert(!(S1IsUnsigned && S1IsSigned));
17389
17390 // There are 9 possible permutations of
17391 // {S0IsUnsigned, S0IsSigned, S1IsUnsigned, S1IsSigned}
17392
17393 // In two permutations, the sign bits are known to be the same for both Ops,
17394 // so simply return Signed / Unsigned corresponding to the MSB
17395
17396 if ((S0IsUnsigned && S1IsUnsigned) || (S0IsSigned && S1IsSigned))
17397 return S0IsSigned;
17398
17399 // In another two permutations, the sign bits are known to be opposite. In
17400 // this case return std::nullopt to indicate a bad match.
17401
17402 if ((S0IsUnsigned && S1IsSigned) || (S0IsSigned && S1IsUnsigned))
17403 return std::nullopt;
17404
17405 // In the remaining five permutations, we don't know the value of the sign
17406 // bit for at least one Op. Since we have a valid ByteProvider, we know that
17407 // the upper bits must be extension bits. Thus, the only ways for the sign
17408 // bit to be unknown is if it was sign extended from unknown value, or if it
17409 // was any extended. In either case, it is correct to use the signed
17410 // version of the signedness semantics of dot4
17411
17412 // In two of such permutations, we known the sign bit is set for
17413 // one op, and the other is unknown. It is okay to used signed version of
17414 // dot4.
17415 if ((S0IsSigned && !(S1IsSigned || S1IsUnsigned)) ||
17416 ((S1IsSigned && !(S0IsSigned || S0IsUnsigned))))
17417 return true;
17418
17419 // In one such permutation, we don't know either of the sign bits. It is okay
17420 // to used the signed version of dot4.
17421 if ((!(S1IsSigned || S1IsUnsigned) && !(S0IsSigned || S0IsUnsigned)))
17422 return true;
17423
17424 // In two of such permutations, we known the sign bit is unset for
17425 // one op, and the other is unknown. Return std::nullopt to indicate a
17426 // bad match.
17427 if ((S0IsUnsigned && !(S1IsSigned || S1IsUnsigned)) ||
17428 ((S1IsUnsigned && !(S0IsSigned || S0IsUnsigned))))
17429 return std::nullopt;
17430
17431 llvm_unreachable("Fully covered condition");
17432}
17433
17434SDValue SITargetLowering::performAddCombine(SDNode *N,
17435 DAGCombinerInfo &DCI) const {
17436 SelectionDAG &DAG = DCI.DAG;
17437 EVT VT = N->getValueType(ResNo: 0);
17438 SDLoc SL(N);
17439 SDValue LHS = N->getOperand(Num: 0);
17440 SDValue RHS = N->getOperand(Num: 1);
17441
17442 if (LHS.getOpcode() == ISD::MUL || RHS.getOpcode() == ISD::MUL) {
17443 if (Subtarget->hasMad64_32()) {
17444 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
17445 return Folded;
17446 }
17447 }
17448
17449 if (SDValue V = reassociateScalarOps(N, DAG)) {
17450 return V;
17451 }
17452
17453 if (VT == MVT::i64) {
17454 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17455 return Folded;
17456 }
17457
17458 if ((isMul(Op: LHS) || isMul(Op: RHS)) && Subtarget->hasDot7Insts() &&
17459 (Subtarget->hasDot1Insts() || Subtarget->hasDot8Insts())) {
17460 SDValue TempNode(N, 0);
17461 std::optional<bool> IsSigned;
17462 SmallVector<DotSrc, 4> Src0s;
17463 SmallVector<DotSrc, 4> Src1s;
17464 SmallVector<SDValue, 4> Src2s;
17465
17466 // Match the v_dot4 tree, while collecting src nodes.
17467 int ChainLength = 0;
17468 for (int I = 0; I < 4; I++) {
17469 auto MulIdx = isMul(Op: LHS) ? 0 : isMul(Op: RHS) ? 1 : -1;
17470 if (MulIdx == -1)
17471 break;
17472 auto Src0 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0));
17473 if (!Src0)
17474 break;
17475 auto Src1 = handleMulOperand(MulOperand: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1));
17476 if (!Src1)
17477 break;
17478
17479 auto IterIsSigned = checkDot4MulSignedness(
17480 N: TempNode->getOperand(Num: MulIdx), Src0&: *Src0, Src1&: *Src1,
17481 S0Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 0),
17482 S1Op: TempNode->getOperand(Num: MulIdx)->getOperand(Num: 1), DAG);
17483 if (!IterIsSigned)
17484 break;
17485 if (!IsSigned)
17486 IsSigned = *IterIsSigned;
17487 if (*IterIsSigned != *IsSigned)
17488 break;
17489 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I);
17490 auto AddIdx = 1 - MulIdx;
17491 // Allow the special case where add (add (mul24, 0), mul24) became ->
17492 // add (mul24, mul24).
17493 if (I == 2 && isMul(Op: TempNode->getOperand(Num: AddIdx))) {
17494 Src2s.push_back(Elt: TempNode->getOperand(Num: AddIdx));
17495 auto Src0 =
17496 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0));
17497 if (!Src0)
17498 break;
17499 auto Src1 =
17500 handleMulOperand(MulOperand: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1));
17501 if (!Src1)
17502 break;
17503 auto IterIsSigned = checkDot4MulSignedness(
17504 N: TempNode->getOperand(Num: AddIdx), Src0&: *Src0, Src1&: *Src1,
17505 S0Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 0),
17506 S1Op: TempNode->getOperand(Num: AddIdx)->getOperand(Num: 1), DAG);
17507 if (!IterIsSigned)
17508 break;
17509 assert(IsSigned);
17510 if (*IterIsSigned != *IsSigned)
17511 break;
17512 placeSources(Src0&: *Src0, Src1&: *Src1, Src0s, Src1s, Step: I + 1);
17513 Src2s.push_back(Elt: DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32));
17514 ChainLength = I + 2;
17515 break;
17516 }
17517
17518 TempNode = TempNode->getOperand(Num: AddIdx);
17519 Src2s.push_back(Elt: TempNode);
17520 ChainLength = I + 1;
17521 if (TempNode->getNumOperands() < 2)
17522 break;
17523 LHS = TempNode->getOperand(Num: 0);
17524 RHS = TempNode->getOperand(Num: 1);
17525 }
17526
17527 if (ChainLength < 2)
17528 return SDValue();
17529
17530 // Masks were constructed with assumption that we would find a chain of
17531 // length 4. If not, then we need to 0 out the MSB bits (via perm mask of
17532 // 0x0c) so they do not affect dot calculation.
17533 if (ChainLength < 4) {
17534 fixMasks(Srcs&: Src0s, ChainLength);
17535 fixMasks(Srcs&: Src1s, ChainLength);
17536 }
17537
17538 SDValue Src0, Src1;
17539
17540 // If we are just using a single source for both, and have permuted the
17541 // bytes consistently, we can just use the sources without permuting
17542 // (commutation).
17543 bool UseOriginalSrc = false;
17544 if (ChainLength == 4 && Src0s.size() == 1 && Src1s.size() == 1 &&
17545 Src0s.begin()->PermMask == Src1s.begin()->PermMask &&
17546 Src0s.begin()->SrcOp.getValueSizeInBits() >= 32 &&
17547 Src1s.begin()->SrcOp.getValueSizeInBits() >= 32) {
17548 SmallVector<unsigned, 4> SrcBytes;
17549 auto Src0Mask = Src0s.begin()->PermMask;
17550 SrcBytes.push_back(Elt: Src0Mask & 0xFF000000);
17551 bool UniqueEntries = true;
17552 for (auto I = 1; I < 4; I++) {
17553 auto NextByte = Src0Mask & (0xFF << ((3 - I) * 8));
17554
17555 if (is_contained(Range&: SrcBytes, Element: NextByte)) {
17556 UniqueEntries = false;
17557 break;
17558 }
17559 SrcBytes.push_back(Elt: NextByte);
17560 }
17561
17562 if (UniqueEntries) {
17563 UseOriginalSrc = true;
17564
17565 auto *FirstElt = Src0s.begin();
17566 auto FirstEltOp =
17567 getDWordFromOffset(DAG, SL, Src: FirstElt->SrcOp, DWordOffset: FirstElt->DWordOffset);
17568
17569 auto *SecondElt = Src1s.begin();
17570 auto SecondEltOp = getDWordFromOffset(DAG, SL, Src: SecondElt->SrcOp,
17571 DWordOffset: SecondElt->DWordOffset);
17572
17573 Src0 = DAG.getBitcastedAnyExtOrTrunc(Op: FirstEltOp, DL: SL,
17574 VT: MVT::getIntegerVT(BitWidth: 32));
17575 Src1 = DAG.getBitcastedAnyExtOrTrunc(Op: SecondEltOp, DL: SL,
17576 VT: MVT::getIntegerVT(BitWidth: 32));
17577 }
17578 }
17579
17580 if (!UseOriginalSrc) {
17581 Src0 = resolveSources(DAG, SL, Srcs&: Src0s, IsSigned: false, IsAny: true);
17582 Src1 = resolveSources(DAG, SL, Srcs&: Src1s, IsSigned: false, IsAny: true);
17583 }
17584
17585 assert(IsSigned);
17586 SDValue Src2 =
17587 DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Src2s[ChainLength - 1], DL: SL, VT: MVT::i32);
17588
17589 SDValue IID = DAG.getTargetConstant(Val: *IsSigned ? Intrinsic::amdgcn_sdot4
17590 : Intrinsic::amdgcn_udot4,
17591 DL: SL, VT: MVT::i64);
17592
17593 assert(!VT.isVector());
17594 auto Dot = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32, N1: IID, N2: Src0,
17595 N3: Src1, N4: Src2, N5: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
17596
17597 return DAG.getExtOrTrunc(IsSigned: *IsSigned, Op: Dot, DL: SL, VT);
17598 }
17599
17600 if (VT != MVT::i32 || !DCI.isAfterLegalizeDAG())
17601 return SDValue();
17602
17603 // add x, zext (setcc) => uaddo_carry x, 0, setcc
17604 // add x, sext (setcc) => usubo_carry x, 0, setcc
17605 unsigned Opc = LHS.getOpcode();
17606 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND ||
17607 Opc == ISD::ANY_EXTEND || Opc == ISD::UADDO_CARRY)
17608 std::swap(a&: RHS, b&: LHS);
17609
17610 Opc = RHS.getOpcode();
17611 switch (Opc) {
17612 default:
17613 break;
17614 case ISD::ZERO_EXTEND:
17615 case ISD::SIGN_EXTEND:
17616 case ISD::ANY_EXTEND: {
17617 auto Cond = RHS.getOperand(i: 0);
17618 // If this won't be a real VOPC output, we would still need to insert an
17619 // extra instruction anyway.
17620 if (!isBoolSGPR(V: Cond))
17621 break;
17622 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
17623 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
17624 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::USUBO_CARRY : ISD::UADDO_CARRY;
17625 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
17626 }
17627 case ISD::UADDO_CARRY: {
17628 // add x, (uaddo_carry y, 0, cc) => uaddo_carry x, y, cc
17629 if (!isNullConstant(V: RHS.getOperand(i: 1)))
17630 break;
17631 SDValue Args[] = {LHS, RHS.getOperand(i: 0), RHS.getOperand(i: 2)};
17632 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: RHS->getVTList(), Ops: Args);
17633 }
17634 }
17635 return SDValue();
17636}
17637
17638SDValue SITargetLowering::performPtrAddCombine(SDNode *N,
17639 DAGCombinerInfo &DCI) const {
17640 SelectionDAG &DAG = DCI.DAG;
17641 SDLoc DL(N);
17642 EVT VT = N->getValueType(ResNo: 0);
17643 SDValue N0 = N->getOperand(Num: 0);
17644 SDValue N1 = N->getOperand(Num: 1);
17645
17646 // The following folds transform PTRADDs into regular arithmetic in cases
17647 // where the PTRADD wouldn't be folded as an immediate offset into memory
17648 // instructions anyway. They are target-specific in that other targets might
17649 // prefer to not lose information about the pointer arithmetic.
17650
17651 // Fold (ptradd x, shl(0 - v, k)) -> sub(x, shl(v, k)).
17652 // Adapted from DAGCombiner::visitADDLikeCommutative.
17653 SDValue V, K;
17654 if (sd_match(N: N1, P: m_Shl(L: m_Neg(V: m_Value(N&: V)), R: m_Value(N&: K)))) {
17655 SDNodeFlags ShlFlags = N1->getFlags();
17656 // If the original shl is NUW and NSW, the first k+1 bits of 0-v are all 0,
17657 // so v is either 0 or the first k+1 bits of v are all 1 -> NSW can be
17658 // preserved.
17659 SDNodeFlags NewShlFlags =
17660 ShlFlags.hasNoUnsignedWrap() && ShlFlags.hasNoSignedWrap()
17661 ? SDNodeFlags::NoSignedWrap
17662 : SDNodeFlags();
17663 SDValue Inner = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: V, N2: K, Flags: NewShlFlags);
17664 DCI.AddToWorklist(N: Inner.getNode());
17665 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Inner);
17666 }
17667
17668 // Fold into Mad64 if the right-hand side is a MUL. Analogous to a fold in
17669 // performAddCombine.
17670 if (N1.getOpcode() == ISD::MUL) {
17671 if (Subtarget->hasMad64_32()) {
17672 if (SDValue Folded = tryFoldToMad64_32(N, DCI))
17673 return Folded;
17674 }
17675 }
17676
17677 // If the 32 low bits of the constant are all zero, there is nothing to fold
17678 // into an immediate offset, so it's better to eliminate the unnecessary
17679 // addition for the lower 32 bits than to preserve the PTRADD.
17680 // Analogous to a fold in performAddCombine.
17681 if (VT == MVT::i64) {
17682 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17683 return Folded;
17684 }
17685
17686 if (N1.getOpcode() != ISD::ADD || !N1.hasOneUse())
17687 return SDValue();
17688
17689 SDValue X = N0;
17690 SDValue Y = N1.getOperand(i: 0);
17691 SDValue Z = N1.getOperand(i: 1);
17692 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Y);
17693 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Z);
17694
17695 if (!YIsConstant && !ZIsConstant && !X->isDivergent() &&
17696 Y->isDivergent() != Z->isDivergent()) {
17697 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if x and
17698 // y are uniform and z isn't.
17699 // Reassociate (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if x and
17700 // z are uniform and y isn't.
17701 // The goal is to push uniform operands up in the computation, so that they
17702 // can be handled with scalar operations. We can't use reassociateScalarOps
17703 // for this since it requires two identical commutative operations to
17704 // reassociate.
17705 if (Y->isDivergent())
17706 std::swap(a&: Y, b&: Z);
17707 // If both additions in the original were NUW, reassociation preserves that.
17708 SDNodeFlags ReassocFlags =
17709 (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap;
17710 SDValue UniformInner = DAG.getMemBasePlusOffset(Base: X, Offset: Y, DL, Flags: ReassocFlags);
17711 DCI.AddToWorklist(N: UniformInner.getNode());
17712 return DAG.getMemBasePlusOffset(Base: UniformInner, Offset: Z, DL, Flags: ReassocFlags);
17713 }
17714
17715 return SDValue();
17716}
17717
17718static bool isCtlzOpc(unsigned Opc) {
17719 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
17720}
17721
17722SDValue SITargetLowering::performSubCombine(SDNode *N,
17723 DAGCombinerInfo &DCI) const {
17724 SelectionDAG &DAG = DCI.DAG;
17725 EVT VT = N->getValueType(ResNo: 0);
17726
17727 if (VT == MVT::i64) {
17728 if (SDValue Folded = foldAddSub64WithZeroLowBitsTo32(N, DCI))
17729 return Folded;
17730 }
17731
17732 if (VT != MVT::i32)
17733 return SDValue();
17734
17735 SDLoc SL(N);
17736 SDValue LHS = N->getOperand(Num: 0);
17737 SDValue RHS = N->getOperand(Num: 1);
17738
17739 // sub x, zext (setcc) => usubo_carry x, 0, setcc
17740 // sub x, sext (setcc) => uaddo_carry x, 0, setcc
17741 unsigned Opc = RHS.getOpcode();
17742 switch (Opc) {
17743 default:
17744 break;
17745 case ISD::ZERO_EXTEND:
17746 case ISD::SIGN_EXTEND:
17747 case ISD::ANY_EXTEND: {
17748 auto Cond = RHS.getOperand(i: 0);
17749 // If this won't be a real VOPC output, we would still need to insert an
17750 // extra instruction anyway.
17751 if (!isBoolSGPR(V: Cond))
17752 break;
17753 SDVTList VTList = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1);
17754 SDValue Args[] = {LHS, DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32), Cond};
17755 Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
17756 return DAG.getNode(Opcode: Opc, DL: SL, VTList, Ops: Args);
17757 }
17758 }
17759
17760 if (LHS.getOpcode() == ISD::USUBO_CARRY) {
17761 // sub (usubo_carry x, 0, cc), y => usubo_carry x, y, cc
17762 if (!isNullConstant(V: LHS.getOperand(i: 1)))
17763 return SDValue();
17764 SDValue Args[] = {LHS.getOperand(i: 0), RHS, LHS.getOperand(i: 2)};
17765 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: SDLoc(N), VTList: LHS->getVTList(), Ops: Args);
17766 }
17767
17768 // sub (ctlz (xor x, (sra x, 31))), 1 -> ctls x.
17769 if (isOneConstant(V: RHS) && isCtlzOpc(Opc: LHS.getOpcode())) {
17770 SDValue CtlzSrc = LHS.getOperand(i: 0);
17771 // Check for xor x, (sra x, 31) pattern.
17772 if (CtlzSrc.getOpcode() == ISD::XOR) {
17773 SDValue X = CtlzSrc.getOperand(i: 0);
17774 SDValue SignExt = CtlzSrc.getOperand(i: 1);
17775 // Try both ordering of XOR operands.
17776 if (SignExt.getOpcode() != ISD::SRA)
17777 std::swap(a&: X, b&: SignExt);
17778 if (SignExt.getOpcode() == ISD::SRA && SignExt.getOperand(i: 0) == X) {
17779 ConstantSDNode *ShiftAmt =
17780 dyn_cast<ConstantSDNode>(Val: SignExt.getOperand(i: 1));
17781 unsigned BitWidth = X.getValueType().getScalarSizeInBits();
17782 if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1)
17783 return DAG.getNode(Opcode: ISD::CTLS, DL: SL, VT, Operand: X);
17784 }
17785 }
17786 }
17787
17788 return SDValue();
17789}
17790
17791SDValue SITargetLowering::performFAddCombine(SDNode *N,
17792 DAGCombinerInfo &DCI) const {
17793 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
17794 return SDValue();
17795
17796 SelectionDAG &DAG = DCI.DAG;
17797 EVT VT = N->getValueType(ResNo: 0);
17798
17799 SDLoc SL(N);
17800 SDValue LHS = N->getOperand(Num: 0);
17801 SDValue RHS = N->getOperand(Num: 1);
17802
17803 // These should really be instruction patterns, but writing patterns with
17804 // source modifiers is a pain.
17805
17806 // fadd (fadd (a, a), b) -> mad 2.0, a, b
17807 if (LHS.getOpcode() == ISD::FADD) {
17808 SDValue A = LHS.getOperand(i: 0);
17809 if (A == LHS.getOperand(i: 1)) {
17810 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
17811 if (FusedOp != 0) {
17812 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
17813 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: RHS);
17814 }
17815 }
17816 }
17817
17818 // fadd (b, fadd (a, a)) -> mad 2.0, a, b
17819 if (RHS.getOpcode() == ISD::FADD) {
17820 SDValue A = RHS.getOperand(i: 0);
17821 if (A == RHS.getOperand(i: 1)) {
17822 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
17823 if (FusedOp != 0) {
17824 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
17825 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: LHS);
17826 }
17827 }
17828 }
17829
17830 return SDValue();
17831}
17832
17833SDValue SITargetLowering::performFSubCombine(SDNode *N,
17834 DAGCombinerInfo &DCI) const {
17835 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
17836 return SDValue();
17837
17838 SelectionDAG &DAG = DCI.DAG;
17839 SDLoc SL(N);
17840 EVT VT = N->getValueType(ResNo: 0);
17841 assert(!VT.isVector());
17842
17843 // Try to get the fneg to fold into the source modifier. This undoes generic
17844 // DAG combines and folds them into the mad.
17845 //
17846 // Only do this if we are not trying to support denormals. v_mad_f32 does
17847 // not support denormals ever.
17848 SDValue LHS = N->getOperand(Num: 0);
17849 SDValue RHS = N->getOperand(Num: 1);
17850 if (LHS.getOpcode() == ISD::FADD) {
17851 // (fsub (fadd a, a), c) -> mad 2.0, a, (fneg c)
17852 SDValue A = LHS.getOperand(i: 0);
17853 if (A == LHS.getOperand(i: 1)) {
17854 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: LHS.getNode());
17855 if (FusedOp != 0) {
17856 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL: SL, VT);
17857 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
17858
17859 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: Two, N3: NegRHS);
17860 }
17861 }
17862 }
17863
17864 if (RHS.getOpcode() == ISD::FADD) {
17865 // (fsub c, (fadd a, a)) -> mad -2.0, a, c
17866
17867 SDValue A = RHS.getOperand(i: 0);
17868 if (A == RHS.getOperand(i: 1)) {
17869 unsigned FusedOp = getFusedOpcode(DAG, N0: N, N1: RHS.getNode());
17870 if (FusedOp != 0) {
17871 const SDValue NegTwo = DAG.getConstantFP(Val: -2.0, DL: SL, VT);
17872 return DAG.getNode(Opcode: FusedOp, DL: SL, VT, N1: A, N2: NegTwo, N3: LHS);
17873 }
17874 }
17875 }
17876
17877 return SDValue();
17878}
17879
17880SDValue SITargetLowering::performFDivCombine(SDNode *N,
17881 DAGCombinerInfo &DCI) const {
17882 SelectionDAG &DAG = DCI.DAG;
17883 SDLoc SL(N);
17884 EVT VT = N->getValueType(ResNo: 0);
17885
17886 if (VT != MVT::f16 && VT != MVT::bf16)
17887 return SDValue();
17888
17889 SDValue LHS = N->getOperand(Num: 0);
17890 SDValue RHS = N->getOperand(Num: 1);
17891
17892 SDNodeFlags Flags = N->getFlags();
17893 SDNodeFlags RHSFlags = RHS->getFlags();
17894 if (!Flags.hasAllowContract() || !RHSFlags.hasAllowContract() ||
17895 !RHS->hasOneUse())
17896 return SDValue();
17897
17898 if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(Val&: LHS)) {
17899 bool IsNegative = false;
17900 if (CLHS->isOne() || (IsNegative = CLHS->isMinusOne())) {
17901 // fdiv contract 1.0, (sqrt contract x) -> rsq
17902 // fdiv contract -1.0, (sqrt contract x) -> fneg(rsq)
17903 if (RHS.getOpcode() == ISD::FSQRT) {
17904 // TODO: Or in RHS flags, somehow missing from SDNodeFlags
17905 SDValue SqrtOp = RHS.getOperand(i: 0);
17906 SDValue Rsq;
17907 if (isOperationLegal(Op: ISD::FSQRT, VT)) {
17908 // fsqrt legality correlates to rsq availability of the same type.
17909 Rsq = DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT, Operand: SqrtOp, Flags);
17910 } else if (VT == MVT::f16) {
17911 // Targets without 16-bit instructions (gfx6/gfx7) have no f16 rsq,
17912 // but v_rsq_f32 is more than accurate enough for f16. Unlike bf16,
17913 // every f16 value (including denormals) extends to a normal f32, and
17914 // an f16 rsq result is never denormal, so the f32 reciprocal square
17915 // root needs no denormal handling. Compute it in f32 and round back.
17916 SDValue Ext =
17917 DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: SqrtOp, Flags);
17918 SDValue F32Rsq =
17919 DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
17920 Rsq = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: F32Rsq,
17921 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
17922 } else {
17923 // bf16 shares f32's exponent range, so bf16 denormals would extend to
17924 // f32 denormals that v_rsq_f32 does not handle. Leave it expanded.
17925 return SDValue();
17926 }
17927 return IsNegative ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Rsq, Flags) : Rsq;
17928 }
17929 }
17930 }
17931
17932 return SDValue();
17933}
17934
17935SDValue SITargetLowering::performFMulCombine(SDNode *N,
17936 DAGCombinerInfo &DCI) const {
17937 SelectionDAG &DAG = DCI.DAG;
17938 EVT VT = N->getValueType(ResNo: 0);
17939 EVT ScalarVT = VT.getScalarType();
17940 EVT IntVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
17941
17942 if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() &&
17943 (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
17944 // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32.
17945 return SDValue();
17946 }
17947
17948 SDValue LHS = N->getOperand(Num: 0);
17949 SDValue RHS = N->getOperand(Num: 1);
17950
17951 // It is cheaper to realize i32 inline constants as compared against
17952 // materializing f16 or f64 (or even non-inline f32) values,
17953 // possible via ldexp usage, as shown below :
17954 //
17955 // Given : A = 2^a & B = 2^b ; where a and b are integers.
17956 // fmul x, (select y, A, B) -> ldexp( x, (select i32 y, a, b) )
17957 // fmul x, (select y, -A, -B) -> ldexp( (fneg x), (select i32 y, a, b) )
17958 if ((ScalarVT == MVT::f64 || ScalarVT == MVT::f32 || ScalarVT == MVT::f16) &&
17959 (RHS.hasOneUse() && RHS.getOpcode() == ISD::SELECT)) {
17960 const ConstantFPSDNode *TrueNode = isConstOrConstSplatFP(N: RHS.getOperand(i: 1));
17961 if (!TrueNode)
17962 return SDValue();
17963 const ConstantFPSDNode *FalseNode =
17964 isConstOrConstSplatFP(N: RHS.getOperand(i: 2));
17965 if (!FalseNode)
17966 return SDValue();
17967
17968 if (TrueNode->isNegative() != FalseNode->isNegative())
17969 return SDValue();
17970
17971 // For f32, only non-inline constants should be transformed.
17972 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
17973 if (ScalarVT == MVT::f32 &&
17974 TII->isInlineConstant(Imm: TrueNode->getValueAPF()) &&
17975 TII->isInlineConstant(Imm: FalseNode->getValueAPF()))
17976 return SDValue();
17977
17978 int TrueNodeExpVal = TrueNode->getValueAPF().getExactLog2Abs();
17979 if (TrueNodeExpVal == INT_MIN)
17980 return SDValue();
17981 int FalseNodeExpVal = FalseNode->getValueAPF().getExactLog2Abs();
17982 if (FalseNodeExpVal == INT_MIN)
17983 return SDValue();
17984
17985 SDLoc SL(N);
17986 SDValue SelectNode =
17987 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: IntVT, N1: RHS.getOperand(i: 0),
17988 N2: DAG.getSignedConstant(Val: TrueNodeExpVal, DL: SL, VT: IntVT),
17989 N3: DAG.getSignedConstant(Val: FalseNodeExpVal, DL: SL, VT: IntVT));
17990
17991 LHS = TrueNode->isNegative()
17992 ? DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS, Flags: LHS->getFlags())
17993 : LHS;
17994
17995 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: LHS, N2: SelectNode, Flags: N->getFlags());
17996 }
17997
17998 return SDValue();
17999}
18000
18001SDValue SITargetLowering::performFMACombine(SDNode *N,
18002 DAGCombinerInfo &DCI) const {
18003 SelectionDAG &DAG = DCI.DAG;
18004 EVT VT = N->getValueType(ResNo: 0);
18005 SDLoc SL(N);
18006
18007 if (!Subtarget->hasDot10Insts() || VT != MVT::f32)
18008 return SDValue();
18009
18010 // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
18011 // FDOT2((V2F16)S0, (V2F16)S1, (F32)z))
18012 SDValue Op1 = N->getOperand(Num: 0);
18013 SDValue Op2 = N->getOperand(Num: 1);
18014 SDValue FMA = N->getOperand(Num: 2);
18015
18016 if (FMA.getOpcode() != ISD::FMA || Op1.getOpcode() != ISD::FP_EXTEND ||
18017 Op2.getOpcode() != ISD::FP_EXTEND)
18018 return SDValue();
18019
18020 // fdot2_f32_f16 always flushes fp32 denormal operand and output to zero,
18021 // regardless of the denorm mode setting. Therefore,
18022 // fp-contract is sufficient to allow generating fdot2.
18023 const TargetOptions &Options = DAG.getTarget().Options;
18024 if (Options.AllowFPOpFusion == FPOpFusion::Fast ||
18025 (N->getFlags().hasAllowContract() &&
18026 FMA->getFlags().hasAllowContract())) {
18027 Op1 = Op1.getOperand(i: 0);
18028 Op2 = Op2.getOperand(i: 0);
18029 if (Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18030 Op2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18031 return SDValue();
18032
18033 SDValue Vec1 = Op1.getOperand(i: 0);
18034 SDValue Idx1 = Op1.getOperand(i: 1);
18035 SDValue Vec2 = Op2.getOperand(i: 0);
18036
18037 SDValue FMAOp1 = FMA.getOperand(i: 0);
18038 SDValue FMAOp2 = FMA.getOperand(i: 1);
18039 SDValue FMAAcc = FMA.getOperand(i: 2);
18040
18041 if (FMAOp1.getOpcode() != ISD::FP_EXTEND ||
18042 FMAOp2.getOpcode() != ISD::FP_EXTEND)
18043 return SDValue();
18044
18045 FMAOp1 = FMAOp1.getOperand(i: 0);
18046 FMAOp2 = FMAOp2.getOperand(i: 0);
18047 if (FMAOp1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18048 FMAOp2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18049 return SDValue();
18050
18051 SDValue Vec3 = FMAOp1.getOperand(i: 0);
18052 SDValue Vec4 = FMAOp2.getOperand(i: 0);
18053 SDValue Idx2 = FMAOp1.getOperand(i: 1);
18054
18055 if (Idx1 != Op2.getOperand(i: 1) || Idx2 != FMAOp2.getOperand(i: 1) ||
18056 // Idx1 and Idx2 cannot be the same.
18057 Idx1 == Idx2)
18058 return SDValue();
18059
18060 if (Vec1 == Vec2 || Vec3 == Vec4)
18061 return SDValue();
18062
18063 if (Vec1.getValueType() != MVT::v2f16 || Vec2.getValueType() != MVT::v2f16)
18064 return SDValue();
18065
18066 if ((Vec1 == Vec3 && Vec2 == Vec4) || (Vec1 == Vec4 && Vec2 == Vec3)) {
18067 return DAG.getNode(Opcode: AMDGPUISD::FDOT2, DL: SL, VT: MVT::f32, N1: Vec1, N2: Vec2, N3: FMAAcc,
18068 N4: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i1));
18069 }
18070 }
18071 return SDValue();
18072}
18073
18074// Given a double-precision ordered or unordered comparison, return the
18075// condition code for an equivalent integral comparison of the operands' upper
18076// 32 bits, or `SETCC_INVALID` if not possible.
18077// For simplicity, no simplification occurs if the operands are not both known
18078// to have sign bit zero.
18079//
18080// EQ/NE:
18081// If LHS.lo32 == RHS.lo32:
18082// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18083// If LHS.lo32 != RHS.lo32:
18084// setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18085// The reduction is not possible if operands may be +0 and -0.
18086// For ordered eq / unordered ne, at most one operand may be NaN.
18087// For unordered eq / ordered ne, neither operand can be NaN.
18088//
18089// LT/GE:
18090// If LHS.lo32 >= RHS.lo32 (unsigned):
18091// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18092// If LHS.lo32 < RHS.lo32 (unsigned):
18093// setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18094// The reduction is only supported if both operands are nonnegative.
18095// For ordered lt / unordered ge, the RHS cannot be NaN.
18096// For unordered lt / ordered ge, neither operand can be NaN.
18097//
18098// LE/GT:
18099// If LHS.lo32 > RHS.lo32 (unsigned):
18100// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18101// If LHS.lo32 <= RHS.lo32 (unsigned):
18102// setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18103// The reduction is only supported if both operands are nonnegative.
18104// For unordered le / ordered gt, the LHS cannot be NaN.
18105// For ordered le / unordered gt, neither operand can be NaN.
18106static ISD::CondCode tryReduceF64CompareToHiHalf(const ISD::CondCode CC,
18107 const SDValue LHS,
18108 const SDValue RHS,
18109 const SelectionDAG &DAG) {
18110 EVT VT = LHS.getValueType();
18111 assert(VT == MVT::f64 && "Incorrect operand type!");
18112
18113 const KnownBits RHSBits = DAG.computeKnownBits(Op: RHS);
18114 // Bail if RHS sign bit is not known to be zero.
18115 if (!RHSBits.Zero.isSignBitSet())
18116 return ISD::SETCC_INVALID;
18117
18118 const KnownBits RHSKnownLo32 = RHSBits.trunc(BitWidth: 32);
18119 const KnownFPClass RHSFPClass =
18120 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: RHSBits);
18121 const bool RHSMaybeNaN = !RHSFPClass.isKnownNeverNaN();
18122
18123 const KnownBits LHSBits = DAG.computeKnownBits(Op: LHS);
18124 const KnownBits LHSKnownLo32 = LHSBits.trunc(BitWidth: 32);
18125 const KnownFPClass LHSFPClass =
18126 KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits: LHSBits);
18127 const bool LHSMaybeNaN = !LHSFPClass.isKnownNeverNaN();
18128
18129 // Bail if LHS sign bit is not known to be zero.
18130 if (!LHSBits.Zero.isSignBitSet())
18131 return ISD::SETCC_INVALID;
18132
18133 switch (CC) {
18134 default:
18135 break;
18136 case ISD::SETEQ:
18137 case ISD::SETOEQ:
18138 case ISD::SETUEQ:
18139 case ISD::SETONE:
18140 case ISD::SETUNE: {
18141 // OEQ should be false if either operand is NaN, so it suffices that at
18142 // least one operand is not NaN.
18143 if (CC == ISD::SETOEQ && LHSMaybeNaN && RHSMaybeNaN)
18144 break;
18145 // UEQ should be true if either operand is NaN, but this cannot be checked
18146 // on underlying bits.
18147 if (CC == ISD::SETUEQ && (LHSMaybeNaN || RHSMaybeNaN))
18148 break;
18149 // ONE should be false if either operand is NaN, but this cannot be
18150 // checked on underlying bits.
18151 if (CC == ISD::SETONE && (LHSMaybeNaN || RHSMaybeNaN))
18152 break;
18153 // UNE should be true if either operand is NaN, so it suffices that they
18154 // are not both NaN.
18155 if (CC == ISD::SETUNE && LHSMaybeNaN && RHSMaybeNaN)
18156 break;
18157
18158 const std::optional<bool> KnownEq =
18159 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18160
18161 if (!KnownEq)
18162 break;
18163
18164 if (*KnownEq)
18165 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18166 ? ISD::SETEQ
18167 : ISD::SETNE;
18168
18169 return (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ)
18170 ? ISD::SETFALSE
18171 : ISD::SETTRUE;
18172 }
18173 case ISD::SETLT:
18174 case ISD::SETOLT:
18175 case ISD::SETULT:
18176 case ISD::SETGE:
18177 case ISD::SETOGE:
18178 case ISD::SETUGE: {
18179 // OLT should be false if either operand is NaN.
18180 // Since NaNs have maximum exponent and nonzero mantissa, false positives
18181 // are only possible if the RHS is NaN. (No issue with RHS == +inf since
18182 // the inequality is strict)
18183 if (CC == ISD::SETOLT && RHSMaybeNaN)
18184 break;
18185 // ULT should be true if either operand is NaN, but this cannot be ensured
18186 // with a truncated comparison.
18187 if (CC == ISD::SETULT && (LHSMaybeNaN || RHSMaybeNaN))
18188 break;
18189 // OGE should be false if either operand is NaN, but this cannot be
18190 // ensured with a truncated comparison.
18191 if (CC == ISD::SETOGE && (LHSMaybeNaN || RHSMaybeNaN))
18192 break;
18193 // UGE should be true if either operand is NaN.
18194 // False negatives are only possible if the RHS is NaN.
18195 // (No issue with RHS == +inf since the inequality is inclusive)
18196 if (CC == ISD::SETUGE && RHSMaybeNaN)
18197 break;
18198
18199 const std::optional<bool> KnownUge =
18200 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18201
18202 if (!KnownUge)
18203 break;
18204
18205 if (*KnownUge) {
18206 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18207 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18208 ? ISD::SETLT
18209 : ISD::SETGE;
18210 }
18211 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18212 return (CC == ISD::SETLT || CC == ISD::SETOLT || CC == ISD::SETULT)
18213 ? ISD::SETLE
18214 : ISD::SETGT;
18215 }
18216 case ISD::SETLE:
18217 case ISD::SETOLE:
18218 case ISD::SETULE:
18219 case ISD::SETGT:
18220 case ISD::SETOGT:
18221 case ISD::SETUGT: {
18222 // OLE should be false if either operand is NaN, but this cannot be
18223 // ensured with a truncated comparison.
18224 if (CC == ISD::SETOLE && (LHSMaybeNaN || RHSMaybeNaN))
18225 break;
18226 // ULE should be true if either operand is NaN.
18227 // False negatives are only possible if the LHS is NaN.
18228 // (No issue with LHS == +inf since the inequality is inclusive)
18229 if (CC == ISD::SETULE && LHSMaybeNaN)
18230 break;
18231 // OGT should be false if either operand is NaN.
18232 // False positives are only possible if the LHS is NaN.
18233 // (No issue with LHS == +inf since the inequality is strict)
18234 if (CC == ISD::SETOGT && LHSMaybeNaN)
18235 break;
18236 // UGT should be true if either operand is NaN, but this cannot be ensured
18237 // with a truncated comparison.
18238 if (CC == ISD::SETUGT && (LHSMaybeNaN || RHSMaybeNaN))
18239 break;
18240
18241 const std::optional<bool> KnownUle =
18242 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18243
18244 if (!KnownUle)
18245 break;
18246
18247 if (*KnownUle) {
18248 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18249 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18250 ? ISD::SETLE
18251 : ISD::SETGT;
18252 }
18253 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18254 return (CC == ISD::SETLE || CC == ISD::SETOLE || CC == ISD::SETULE)
18255 ? ISD::SETLT
18256 : ISD::SETGE;
18257 }
18258 }
18259
18260 return ISD::SETCC_INVALID;
18261}
18262
18263SDValue SITargetLowering::performSetCCCombine(SDNode *N,
18264 DAGCombinerInfo &DCI) const {
18265 SelectionDAG &DAG = DCI.DAG;
18266 SDLoc SL(N);
18267
18268 SDValue LHS = N->getOperand(Num: 0);
18269 SDValue RHS = N->getOperand(Num: 1);
18270 EVT VT = LHS.getValueType();
18271 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
18272
18273 auto *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
18274 if (!CRHS) {
18275 CRHS = dyn_cast<ConstantSDNode>(Val&: LHS);
18276 if (CRHS) {
18277 std::swap(a&: LHS, b&: RHS);
18278 CC = getSetCCSwappedOperands(Operation: CC);
18279 }
18280 }
18281
18282 if (CRHS) {
18283 if (VT == MVT::i32 && LHS.getOpcode() == ISD::SIGN_EXTEND &&
18284 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18285 // setcc (sext from i1 cc), -1, ne|sgt|ult) => not cc => xor cc, -1
18286 // setcc (sext from i1 cc), -1, eq|sle|uge) => cc
18287 // setcc (sext from i1 cc), 0, eq|sge|ule) => not cc => xor cc, -1
18288 // setcc (sext from i1 cc), 0, ne|ugt|slt) => cc
18289 if ((CRHS->isAllOnes() &&
18290 (CC == ISD::SETNE || CC == ISD::SETGT || CC == ISD::SETULT)) ||
18291 (CRHS->isZero() &&
18292 (CC == ISD::SETEQ || CC == ISD::SETGE || CC == ISD::SETULE)))
18293 return DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
18294 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i1));
18295 if ((CRHS->isAllOnes() &&
18296 (CC == ISD::SETEQ || CC == ISD::SETLE || CC == ISD::SETUGE)) ||
18297 (CRHS->isZero() &&
18298 (CC == ISD::SETNE || CC == ISD::SETUGT || CC == ISD::SETLT)))
18299 return LHS.getOperand(i: 0);
18300 }
18301
18302 const APInt &CRHSVal = CRHS->getAPIntValue();
18303 if ((CC == ISD::SETEQ || CC == ISD::SETNE) &&
18304 LHS.getOpcode() == ISD::SELECT &&
18305 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
18306 isa<ConstantSDNode>(Val: LHS.getOperand(i: 2)) &&
18307 isBoolSGPR(V: LHS.getOperand(i: 0))) {
18308 // Given CT != FT:
18309 // setcc (select cc, CT, CF), CF, eq => xor cc, -1
18310 // setcc (select cc, CT, CF), CF, ne => cc
18311 // setcc (select cc, CT, CF), CT, ne => xor cc, -1
18312 // setcc (select cc, CT, CF), CT, eq => cc
18313 const APInt &CT = LHS.getConstantOperandAPInt(i: 1);
18314 const APInt &CF = LHS.getConstantOperandAPInt(i: 2);
18315
18316 if (CT != CF) {
18317 if ((CF == CRHSVal && CC == ISD::SETEQ) ||
18318 (CT == CRHSVal && CC == ISD::SETNE))
18319 return DAG.getNOT(DL: SL, Val: LHS.getOperand(i: 0), VT: MVT::i1);
18320 if ((CF == CRHSVal && CC == ISD::SETNE) ||
18321 (CT == CRHSVal && CC == ISD::SETEQ))
18322 return LHS.getOperand(i: 0);
18323 }
18324 }
18325 }
18326
18327 // Truncate 64-bit setcc to test only upper 32-bits of its operands in the
18328 // following cases where information about the lower 32-bits of its operands
18329 // is known:
18330 //
18331 // If LHS.lo32 == RHS.lo32:
18332 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, eq/ne
18333 // If LHS.lo32 != RHS.lo32:
18334 // setcc LHS, RHS, eq/ne => setcc LHS.hi32, RHS.hi32, false/true
18335 // If LHS.lo32 >= RHS.lo32 (unsigned):
18336 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]lt/ge
18337 // If LHS.lo32 > RHS.lo32 (unsigned):
18338 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]lt/ge
18339 // If LHS.lo32 <= RHS.lo32 (unsigned):
18340 // setcc LHS, RHS, [u]le/gt => LHS.hi32, RHS.hi32, [u]le/gt
18341 // If LHS.lo32 < RHS.lo32 (unsigned):
18342 // setcc LHS, RHS, [u]lt/ge => LHS.hi32, RHS.hi32, [u]le/gt
18343 if (VT == MVT::i64) {
18344 const KnownBits LHSKnownLo32 = DAG.computeKnownBits(Op: LHS).trunc(BitWidth: 32);
18345 const KnownBits RHSKnownLo32 = DAG.computeKnownBits(Op: RHS).trunc(BitWidth: 32);
18346
18347 // NewCC is valid iff we can truncate the setcc to only test the upper 32
18348 // bits
18349 ISD::CondCode NewCC = ISD::SETCC_INVALID;
18350
18351 switch (CC) {
18352 default:
18353 break;
18354 case ISD::SETEQ: {
18355 const std::optional<bool> KnownEq =
18356 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18357 if (KnownEq)
18358 NewCC = *KnownEq ? ISD::SETEQ : ISD::SETFALSE;
18359
18360 break;
18361 }
18362 case ISD::SETNE: {
18363 const std::optional<bool> KnownEq =
18364 KnownBits::eq(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18365 if (KnownEq)
18366 NewCC = *KnownEq ? ISD::SETNE : ISD::SETTRUE;
18367
18368 break;
18369 }
18370 case ISD::SETULT:
18371 case ISD::SETUGE:
18372 case ISD::SETLT:
18373 case ISD::SETGE: {
18374 const std::optional<bool> KnownUge =
18375 KnownBits::uge(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18376 if (KnownUge) {
18377 if (*KnownUge) {
18378 // LHS.lo32 uge RHS.lo32, so LHS >= RHS iff LHS.hi32 >= RHS.hi32
18379 NewCC = CC;
18380 } else {
18381 // LHS.lo32 ult RHS.lo32, so LHS >= RHS iff LHS.hi32 > RHS.hi32
18382 NewCC = CC == ISD::SETULT ? ISD::SETULE
18383 : CC == ISD::SETUGE ? ISD::SETUGT
18384 : CC == ISD::SETLT ? ISD::SETLE
18385 : ISD::SETGT;
18386 }
18387 }
18388 break;
18389 }
18390 case ISD::SETULE:
18391 case ISD::SETUGT:
18392 case ISD::SETLE:
18393 case ISD::SETGT: {
18394 const std::optional<bool> KnownUle =
18395 KnownBits::ule(LHS: LHSKnownLo32, RHS: RHSKnownLo32);
18396 if (KnownUle) {
18397 if (*KnownUle) {
18398 // LHS.lo32 ule RHS.lo32, so LHS <= RHS iff LHS.hi32 <= RHS.hi32
18399 NewCC = CC;
18400 } else {
18401 // LHS.lo32 ugt RHS.lo32, so LHS <= RHS iff LHS.hi32 < RHS.hi32
18402 NewCC = CC == ISD::SETULE ? ISD::SETULT
18403 : CC == ISD::SETUGT ? ISD::SETUGE
18404 : CC == ISD::SETLE ? ISD::SETLT
18405 : ISD::SETGE;
18406 }
18407 }
18408 break;
18409 }
18410 }
18411
18412 if (NewCC != ISD::SETCC_INVALID)
18413 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
18414 RHS: getHiHalf64(Op: RHS, DAG), Cond: NewCC);
18415 }
18416
18417 // Eliminate setcc by using carryout from add/sub instruction
18418
18419 // LHS = ADD i64 RHS, Z LHSlo = UADDO i32 RHSlo, Zlo
18420 // setcc LHS ult RHS -> LHSHi = UADDO_CARRY i32 RHShi, Zhi
18421 // similarly for subtraction
18422
18423 // LHS = ADD i64 Y, 1 LHSlo = UADDO i32 Ylo, 1
18424 // setcc LHS eq 0 -> LHSHi = UADDO_CARRY i32 Yhi, 0
18425
18426 if (VT == MVT::i64 && ((CC == ISD::SETULT &&
18427 sd_match(N: LHS, P: m_Add(L: m_Specific(N: RHS), R: m_Value()))) ||
18428 (CC == ISD::SETUGT &&
18429 sd_match(N: LHS, P: m_Sub(L: m_Specific(N: RHS), R: m_Value()))) ||
18430 (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
18431 sd_match(N: LHS, P: m_Add(L: m_Value(), R: m_One()))))) {
18432 bool IsAdd = LHS.getOpcode() == ISD::ADD;
18433
18434 SDValue Op0 = LHS.getOperand(i: 0);
18435 SDValue Op1 = LHS.getOperand(i: 1);
18436
18437 SDValue Op0Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op0);
18438 SDValue Op1Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Op1);
18439
18440 SDValue Op0Hi = getHiHalf64(Op: Op0, DAG);
18441 SDValue Op1Hi = getHiHalf64(Op: Op1, DAG);
18442
18443 SDValue NodeLo =
18444 DAG.getNode(Opcode: IsAdd ? ISD::UADDO : ISD::USUBO, DL: SL,
18445 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1), Ops: {Op0Lo, Op1Lo});
18446
18447 SDValue CarryInHi = NodeLo.getValue(R: 1);
18448 SDValue NodeHi = DAG.getNode(Opcode: IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
18449 DL: SL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i1),
18450 Ops: {Op0Hi, Op1Hi, CarryInHi});
18451
18452 SDValue ResultLo = NodeLo.getValue(R: 0);
18453 SDValue ResultHi = NodeHi.getValue(R: 0);
18454
18455 SDValue JoinedResult =
18456 DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {ResultLo, ResultHi});
18457
18458 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: JoinedResult);
18459 SDValue Overflow = NodeHi.getValue(R: 1);
18460 DCI.CombineTo(N: LHS.getNode(), Res: Result);
18461 return Overflow;
18462 }
18463
18464 if (VT != MVT::f32 && VT != MVT::f64 &&
18465 (!Subtarget->has16BitInsts() || VT != MVT::f16))
18466 return SDValue();
18467
18468 // Match isinf/isfinite pattern
18469 // (fcmp oeq (fabs x), inf) -> (fp_class x, (p_infinity | n_infinity))
18470 // (fcmp one (fabs x), inf) -> (fp_class x,
18471 // (p_normal | n_normal | p_subnormal | n_subnormal | p_zero | n_zero)
18472 if ((CC == ISD::SETOEQ || CC == ISD::SETONE) &&
18473 LHS.getOpcode() == ISD::FABS) {
18474 const ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
18475 if (!CRHS)
18476 return SDValue();
18477
18478 const APFloat &APF = CRHS->getValueAPF();
18479 if (APF.isInfinity() && !APF.isNegative()) {
18480 const unsigned IsInfMask =
18481 SIInstrFlags::P_INFINITY | SIInstrFlags::N_INFINITY;
18482 const unsigned IsFiniteMask =
18483 SIInstrFlags::N_ZERO | SIInstrFlags::P_ZERO | SIInstrFlags::N_NORMAL |
18484 SIInstrFlags::P_NORMAL | SIInstrFlags::N_SUBNORMAL |
18485 SIInstrFlags::P_SUBNORMAL;
18486 unsigned Mask = CC == ISD::SETOEQ ? IsInfMask : IsFiniteMask;
18487 return DAG.getNode(Opcode: AMDGPUISD::FP_CLASS, DL: SL, VT: MVT::i1, N1: LHS.getOperand(i: 0),
18488 N2: DAG.getConstant(Val: Mask, DL: SL, VT: MVT::i32));
18489 }
18490 }
18491
18492 if (VT == MVT::f64) {
18493 ISD::CondCode HiHalfCC = tryReduceF64CompareToHiHalf(CC, LHS, RHS, DAG);
18494 if (HiHalfCC != ISD::SETCC_INVALID)
18495 return DAG.getSetCC(DL: SL, VT: N->getValueType(ResNo: 0), LHS: getHiHalf64(Op: LHS, DAG),
18496 RHS: getHiHalf64(Op: RHS, DAG), Cond: HiHalfCC);
18497 }
18498
18499 return SDValue();
18500}
18501
18502SDValue
18503SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
18504 DAGCombinerInfo &DCI) const {
18505 SelectionDAG &DAG = DCI.DAG;
18506 SDLoc SL(N);
18507 unsigned Offset = N->getOpcode() - AMDGPUISD::CVT_F32_UBYTE0;
18508
18509 SDValue Src = N->getOperand(Num: 0);
18510 SDValue Shift = N->getOperand(Num: 0);
18511
18512 // TODO: Extend type shouldn't matter (assuming legal types).
18513 if (Shift.getOpcode() == ISD::ZERO_EXTEND)
18514 Shift = Shift.getOperand(i: 0);
18515
18516 if (Shift.getOpcode() == ISD::SRL || Shift.getOpcode() == ISD::SHL) {
18517 // cvt_f32_ubyte1 (shl x, 8) -> cvt_f32_ubyte0 x
18518 // cvt_f32_ubyte3 (shl x, 16) -> cvt_f32_ubyte1 x
18519 // cvt_f32_ubyte0 (srl x, 16) -> cvt_f32_ubyte2 x
18520 // cvt_f32_ubyte1 (srl x, 16) -> cvt_f32_ubyte3 x
18521 // cvt_f32_ubyte0 (srl x, 8) -> cvt_f32_ubyte1 x
18522 if (auto *C = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1))) {
18523 SDValue Shifted = DAG.getZExtOrTrunc(
18524 Op: Shift.getOperand(i: 0), DL: SDLoc(Shift.getOperand(i: 0)), VT: MVT::i32);
18525
18526 unsigned ShiftOffset = 8 * Offset;
18527 if (Shift.getOpcode() == ISD::SHL)
18528 ShiftOffset -= C->getZExtValue();
18529 else
18530 ShiftOffset += C->getZExtValue();
18531
18532 if (ShiftOffset < 32 && (ShiftOffset % 8) == 0) {
18533 return DAG.getNode(Opcode: AMDGPUISD::CVT_F32_UBYTE0 + ShiftOffset / 8, DL: SL,
18534 VT: MVT::f32, Operand: Shifted);
18535 }
18536 }
18537 }
18538
18539 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18540 APInt DemandedBits = APInt::getBitsSet(numBits: 32, loBit: 8 * Offset, hiBit: 8 * Offset + 8);
18541 if (TLI.SimplifyDemandedBits(Op: Src, DemandedBits, DCI)) {
18542 // We simplified Src. If this node is not dead, visit it again so it is
18543 // folded properly.
18544 if (N->getOpcode() != ISD::DELETED_NODE)
18545 DCI.AddToWorklist(N);
18546 return SDValue(N, 0);
18547 }
18548
18549 // Handle (or x, (srl y, 8)) pattern when known bits are zero.
18550 if (SDValue DemandedSrc =
18551 TLI.SimplifyMultipleUseDemandedBits(Op: Src, DemandedBits, DAG))
18552 return DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: MVT::f32, Operand: DemandedSrc);
18553
18554 return SDValue();
18555}
18556
18557SDValue SITargetLowering::performClampCombine(SDNode *N,
18558 DAGCombinerInfo &DCI) const {
18559 ConstantFPSDNode *CSrc = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
18560 if (!CSrc)
18561 return SDValue();
18562
18563 const MachineFunction &MF = DCI.DAG.getMachineFunction();
18564 const APFloat &F = CSrc->getValueAPF();
18565 APFloat Zero = APFloat::getZero(Sem: F.getSemantics());
18566 if (F < Zero ||
18567 (F.isNaN() && MF.getInfo<SIMachineFunctionInfo>()->getMode().DX10Clamp)) {
18568 return DCI.DAG.getConstantFP(Val: Zero, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
18569 }
18570
18571 APFloat One = APFloat::getOne(Sem: F.getSemantics());
18572 if (F > One)
18573 return DCI.DAG.getConstantFP(Val: One, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
18574
18575 return SDValue(CSrc, 0);
18576}
18577
18578SDValue SITargetLowering::performSelectCombine(SDNode *N,
18579 DAGCombinerInfo &DCI) const {
18580
18581 // Try to fold CMP + SELECT patterns with shared constants (both FP and
18582 // integer).
18583 // Detect when CMP and SELECT use the same constant and fold them to avoid
18584 // loading the constant twice. Specifically handles patterns like:
18585 // %cmp = icmp eq i32 %val, 4242
18586 // %sel = select i1 %cmp, i32 4242, i32 %other
18587 // It can be optimized to reuse %val instead of 4242 in select.
18588 SDValue Cond = N->getOperand(Num: 0);
18589 SDValue TrueVal = N->getOperand(Num: 1);
18590 SDValue FalseVal = N->getOperand(Num: 2);
18591
18592 // Check if condition is a comparison.
18593 if (Cond.getOpcode() != ISD::SETCC)
18594 return SDValue();
18595
18596 SDValue LHS = Cond.getOperand(i: 0);
18597 SDValue RHS = Cond.getOperand(i: 1);
18598 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
18599
18600 bool isFloatingPoint = LHS.getValueType().isFloatingPoint();
18601 bool isInteger = LHS.getValueType().isInteger();
18602
18603 // Handle simple floating-point and integer types only.
18604 if (!isFloatingPoint && !isInteger)
18605 return SDValue();
18606
18607 bool isEquality = CC == (isFloatingPoint ? ISD::SETOEQ : ISD::SETEQ);
18608 bool isNonEquality = CC == (isFloatingPoint ? ISD::SETONE : ISD::SETNE);
18609 if (!isEquality && !isNonEquality)
18610 return SDValue();
18611
18612 SDValue ArgVal, ConstVal;
18613 if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: RHS)) ||
18614 (isInteger && isa<ConstantSDNode>(Val: RHS))) {
18615 ConstVal = RHS;
18616 ArgVal = LHS;
18617 } else if ((isFloatingPoint && isa<ConstantFPSDNode>(Val: LHS)) ||
18618 (isInteger && isa<ConstantSDNode>(Val: LHS))) {
18619 ConstVal = LHS;
18620 ArgVal = RHS;
18621 } else {
18622 return SDValue();
18623 }
18624
18625 // Skip optimization for inlinable immediates.
18626 if (isFloatingPoint) {
18627 const APFloat &Val = cast<ConstantFPSDNode>(Val&: ConstVal)->getValueAPF();
18628 if (!Val.isNormal() || Subtarget->getInstrInfo()->isInlineConstant(Imm: Val))
18629 return SDValue();
18630 } else {
18631 const std::optional<int64_t> Val =
18632 cast<ConstantSDNode>(Val&: ConstVal)->getAPIntValue().trySExtValue();
18633 if (Val && AMDGPU::isInlinableIntLiteral(Literal: *Val))
18634 return SDValue();
18635 }
18636
18637 // For equality and non-equality comparisons, patterns:
18638 // select (setcc x, const), const, y -> select (setcc x, const), x, y
18639 // select (setccinv x, const), y, const -> select (setccinv x, const), y, x
18640 if (!(isEquality && TrueVal == ConstVal) &&
18641 !(isNonEquality && FalseVal == ConstVal))
18642 return SDValue();
18643
18644 SDValue SelectLHS = (isEquality && TrueVal == ConstVal) ? ArgVal : TrueVal;
18645 SDValue SelectRHS =
18646 (isNonEquality && FalseVal == ConstVal) ? ArgVal : FalseVal;
18647 return DCI.DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Cond,
18648 N2: SelectLHS, N3: SelectRHS);
18649}
18650
18651SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
18652 DAGCombinerInfo &DCI) const {
18653 switch (N->getOpcode()) {
18654 case ISD::ABS:
18655 if (SDValue Res = promoteUniformUnaryOpToI32(Op: SDValue(N, 0), DCI))
18656 return Res;
18657 break;
18658 case ISD::ADD:
18659 case ISD::SUB:
18660 case ISD::SHL:
18661 case ISD::SRL:
18662 case ISD::SRA:
18663 case ISD::AND:
18664 case ISD::OR:
18665 case ISD::XOR:
18666 case ISD::MUL:
18667 case ISD::SETCC:
18668 case ISD::SELECT:
18669 case ISD::SMIN:
18670 case ISD::SMAX:
18671 case ISD::UMIN:
18672 case ISD::UMAX:
18673 case ISD::USUBSAT:
18674 if (auto Res = promoteUniformOpToI32(Op: SDValue(N, 0), DCI))
18675 return Res;
18676 break;
18677 default:
18678 break;
18679 }
18680
18681 if (getTargetMachine().getOptLevel() == CodeGenOptLevel::None)
18682 return SDValue();
18683
18684 switch (N->getOpcode()) {
18685 case ISD::ADD:
18686 return performAddCombine(N, DCI);
18687 case ISD::PTRADD:
18688 return performPtrAddCombine(N, DCI);
18689 case ISD::SUB:
18690 return performSubCombine(N, DCI);
18691 case ISD::FADD:
18692 return performFAddCombine(N, DCI);
18693 case ISD::FSUB:
18694 return performFSubCombine(N, DCI);
18695 case ISD::FDIV:
18696 return performFDivCombine(N, DCI);
18697 case ISD::FMUL:
18698 return performFMulCombine(N, DCI);
18699 case ISD::SETCC:
18700 return performSetCCCombine(N, DCI);
18701 case ISD::SELECT:
18702 if (auto Res = performSelectCombine(N, DCI))
18703 return Res;
18704 break;
18705 case ISD::FMAXNUM:
18706 case ISD::FMINNUM:
18707 case ISD::FMAXNUM_IEEE:
18708 case ISD::FMINNUM_IEEE:
18709 case ISD::FMAXIMUM:
18710 case ISD::FMINIMUM:
18711 case ISD::FMAXIMUMNUM:
18712 case ISD::FMINIMUMNUM:
18713 case ISD::SMAX:
18714 case ISD::SMIN:
18715 case ISD::UMAX:
18716 case ISD::UMIN:
18717 case AMDGPUISD::FMIN_LEGACY:
18718 case AMDGPUISD::FMAX_LEGACY:
18719 return performMinMaxCombine(N, DCI);
18720 case ISD::FMA:
18721 return performFMACombine(N, DCI);
18722 case ISD::AND:
18723 return performAndCombine(N, DCI);
18724 case ISD::OR:
18725 return performOrCombine(N, DCI);
18726 case ISD::FSHR: {
18727 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
18728 if (N->getValueType(ResNo: 0) == MVT::i32 && N->isDivergent() &&
18729 TII->pseudoToMCOpcode(Opcode: AMDGPU::V_PERM_B32_e64) != -1) {
18730 return matchPERM(N, DCI);
18731 }
18732 break;
18733 }
18734 case ISD::XOR:
18735 return performXorCombine(N, DCI);
18736 case ISD::ANY_EXTEND:
18737 case ISD::ZERO_EXTEND:
18738 return performZeroOrAnyExtendCombine(N, DCI);
18739 case ISD::SIGN_EXTEND_INREG:
18740 return performSignExtendInRegCombine(N, DCI);
18741 case AMDGPUISD::FP_CLASS:
18742 return performClassCombine(N, DCI);
18743 case ISD::FCANONICALIZE:
18744 return performFCanonicalizeCombine(N, DCI);
18745 case AMDGPUISD::RCP:
18746 return performRcpCombine(N, DCI);
18747 case ISD::FLDEXP:
18748 case AMDGPUISD::FRACT:
18749 case AMDGPUISD::RSQ:
18750 case AMDGPUISD::RCP_LEGACY:
18751 case AMDGPUISD::RCP_IFLAG:
18752 case AMDGPUISD::RSQ_CLAMP: {
18753 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
18754 SDValue Src = N->getOperand(Num: 0);
18755 if (Src.isUndef())
18756 return Src;
18757 break;
18758 }
18759 case ISD::SINT_TO_FP:
18760 case ISD::UINT_TO_FP:
18761 return performUCharToFloatCombine(N, DCI);
18762 case ISD::FCOPYSIGN:
18763 return performFCopySignCombine(N, DCI);
18764 case AMDGPUISD::CVT_F32_UBYTE0:
18765 case AMDGPUISD::CVT_F32_UBYTE1:
18766 case AMDGPUISD::CVT_F32_UBYTE2:
18767 case AMDGPUISD::CVT_F32_UBYTE3:
18768 return performCvtF32UByteNCombine(N, DCI);
18769 case AMDGPUISD::FMED3:
18770 return performFMed3Combine(N, DCI);
18771 case AMDGPUISD::CVT_PKRTZ_F16_F32:
18772 return performCvtPkRTZCombine(N, DCI);
18773 case AMDGPUISD::CLAMP:
18774 return performClampCombine(N, DCI);
18775 case ISD::SCALAR_TO_VECTOR: {
18776 SelectionDAG &DAG = DCI.DAG;
18777 EVT VT = N->getValueType(ResNo: 0);
18778
18779 // v2i16 (scalar_to_vector i16:x) -> v2i16 (bitcast (any_extend i16:x))
18780 if (VT == MVT::v2i16 || VT == MVT::v2f16 || VT == MVT::v2bf16) {
18781 SDLoc SL(N);
18782 SDValue Src = N->getOperand(Num: 0);
18783 EVT EltVT = Src.getValueType();
18784 if (EltVT != MVT::i16)
18785 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i16, Operand: Src);
18786
18787 SDValue Ext = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Src);
18788 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Ext);
18789 }
18790
18791 break;
18792 }
18793 case ISD::EXTRACT_VECTOR_ELT:
18794 return performExtractVectorEltCombine(N, DCI);
18795 case ISD::INSERT_VECTOR_ELT:
18796 return performInsertVectorEltCombine(N, DCI);
18797 case ISD::FP_ROUND:
18798 return performFPRoundCombine(N, DCI);
18799 case ISD::LOAD: {
18800 if (SDValue Widened = widenLoad(Ld: cast<LoadSDNode>(Val: N), DCI))
18801 return Widened;
18802 [[fallthrough]];
18803 }
18804 default: {
18805 if (!DCI.isBeforeLegalize()) {
18806 if (MemSDNode *MemNode = dyn_cast<MemSDNode>(Val: N))
18807 return performMemSDNodeCombine(N: MemNode, DCI);
18808 }
18809
18810 break;
18811 }
18812 }
18813
18814 return AMDGPUTargetLowering::PerformDAGCombine(N, DCI);
18815}
18816
18817/// Helper function for adjustWritemask
18818static unsigned SubIdx2Lane(unsigned Idx) {
18819 switch (Idx) {
18820 default:
18821 return ~0u;
18822 case AMDGPU::sub0:
18823 return 0;
18824 case AMDGPU::sub1:
18825 return 1;
18826 case AMDGPU::sub2:
18827 return 2;
18828 case AMDGPU::sub3:
18829 return 3;
18830 case AMDGPU::sub4:
18831 return 4; // Possible with TFE/LWE
18832 }
18833}
18834
18835/// Adjust the writemask of MIMG, VIMAGE or VSAMPLE instructions
18836SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
18837 SelectionDAG &DAG) const {
18838 unsigned Opcode = Node->getMachineOpcode();
18839
18840 // Subtract 1 because the vdata output is not a MachineSDNode operand.
18841 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::d16) - 1;
18842 if (D16Idx >= 0 && Node->getConstantOperandVal(Num: D16Idx))
18843 return Node; // not implemented for D16
18844
18845 SDNode *Users[5] = {nullptr};
18846 unsigned Lane = 0;
18847 unsigned DmaskIdx =
18848 AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::dmask) - 1;
18849 unsigned OldDmask = Node->getConstantOperandVal(Num: DmaskIdx);
18850 unsigned NewDmask = 0;
18851 unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::tfe) - 1;
18852 unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::lwe) - 1;
18853 bool UsesTFC = (int(TFEIdx) >= 0 && Node->getConstantOperandVal(Num: TFEIdx)) ||
18854 (int(LWEIdx) >= 0 && Node->getConstantOperandVal(Num: LWEIdx));
18855 unsigned TFCLane = 0;
18856 bool HasChain = Node->getNumValues() > 1;
18857
18858 if (OldDmask == 0) {
18859 // These are folded out, but on the chance it happens don't assert.
18860 return Node;
18861 }
18862
18863 unsigned OldBitsSet = llvm::popcount(Value: OldDmask);
18864 // Work out which is the TFE/LWE lane if that is enabled.
18865 if (UsesTFC) {
18866 TFCLane = OldBitsSet;
18867 }
18868
18869 // Try to figure out the used register components
18870 for (SDUse &Use : Node->uses()) {
18871
18872 // Don't look at users of the chain.
18873 if (Use.getResNo() != 0)
18874 continue;
18875
18876 SDNode *User = Use.getUser();
18877
18878 // Abort if we can't understand the usage
18879 if (!User->isMachineOpcode() ||
18880 User->getMachineOpcode() != TargetOpcode::EXTRACT_SUBREG)
18881 return Node;
18882
18883 // Lane means which subreg of %vgpra_vgprb_vgprc_vgprd is used.
18884 // Note that subregs are packed, i.e. Lane==0 is the first bit set
18885 // in OldDmask, so it can be any of X,Y,Z,W; Lane==1 is the second bit
18886 // set, etc.
18887 Lane = SubIdx2Lane(Idx: User->getConstantOperandVal(Num: 1));
18888 if (Lane == ~0u)
18889 return Node;
18890
18891 // Check if the use is for the TFE/LWE generated result at VGPRn+1.
18892 if (UsesTFC && Lane == TFCLane) {
18893 Users[Lane] = User;
18894 } else {
18895 // Set which texture component corresponds to the lane.
18896 unsigned Comp;
18897 for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
18898 Comp = llvm::countr_zero(Val: Dmask);
18899 Dmask &= ~(1 << Comp);
18900 }
18901
18902 // Abort if we have more than one user per component.
18903 if (Users[Lane])
18904 return Node;
18905
18906 Users[Lane] = User;
18907 NewDmask |= 1 << Comp;
18908 }
18909 }
18910
18911 // Don't allow 0 dmask, as hardware assumes one channel enabled.
18912 bool NoChannels = !NewDmask;
18913 if (NoChannels) {
18914 if (!UsesTFC) {
18915 // No uses of the result and not using TFC. Then do nothing.
18916 return Node;
18917 }
18918 // If the original dmask has one channel - then nothing to do
18919 if (OldBitsSet == 1)
18920 return Node;
18921 // Use an arbitrary dmask - required for the instruction to work
18922 NewDmask = 1;
18923 }
18924 // Abort if there's no change
18925 if (NewDmask == OldDmask)
18926 return Node;
18927
18928 unsigned BitsSet = llvm::popcount(Value: NewDmask);
18929
18930 // Check for TFE or LWE - increase the number of channels by one to account
18931 // for the extra return value
18932 // This will need adjustment for D16 if this is also included in
18933 // adjustWriteMask (this function) but at present D16 are excluded.
18934 unsigned NewChannels = BitsSet + UsesTFC;
18935
18936 int NewOpcode =
18937 AMDGPU::getMaskedMIMGOp(Opc: Node->getMachineOpcode(), NewChannels);
18938 assert(NewOpcode != -1 &&
18939 NewOpcode != static_cast<int>(Node->getMachineOpcode()) &&
18940 "failed to find equivalent MIMG op");
18941
18942 // Adjust the writemask in the node
18943 SmallVector<SDValue, 12> Ops;
18944 llvm::append_range(C&: Ops, R: Node->ops().take_front(N: DmaskIdx));
18945 Ops.push_back(Elt: DAG.getTargetConstant(Val: NewDmask, DL: SDLoc(Node), VT: MVT::i32));
18946 llvm::append_range(C&: Ops, R: Node->ops().drop_front(N: DmaskIdx + 1));
18947
18948 MVT SVT = Node->getValueType(ResNo: 0).getVectorElementType().getSimpleVT();
18949
18950 MVT ResultVT = NewChannels == 1
18951 ? SVT
18952 : MVT::getVectorVT(VT: SVT, NumElements: NewChannels == 3 ? 4
18953 : NewChannels == 5 ? 8
18954 : NewChannels);
18955 SDVTList NewVTList =
18956 HasChain ? DAG.getVTList(VT1: ResultVT, VT2: MVT::Other) : DAG.getVTList(VT: ResultVT);
18957
18958 MachineSDNode *NewNode =
18959 DAG.getMachineNode(Opcode: NewOpcode, dl: SDLoc(Node), VTs: NewVTList, Ops);
18960
18961 if (HasChain) {
18962 // Update chain.
18963 DAG.setNodeMemRefs(N: NewNode, NewMemRefs: Node->memoperands());
18964 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: SDValue(NewNode, 1));
18965 }
18966
18967 if (NewChannels == 1) {
18968 assert(Node->hasNUsesOfValue(1, 0));
18969 SDNode *Copy =
18970 DAG.getMachineNode(Opcode: TargetOpcode::COPY, dl: SDLoc(Node),
18971 VT: Users[Lane]->getValueType(ResNo: 0), Op1: SDValue(NewNode, 0));
18972 DAG.ReplaceAllUsesWith(From: Users[Lane], To: Copy);
18973 return nullptr;
18974 }
18975
18976 // Update the users of the node with the new indices
18977 for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
18978 SDNode *User = Users[i];
18979 if (!User) {
18980 // Handle the special case of NoChannels. We set NewDmask to 1 above, but
18981 // Users[0] is still nullptr because channel 0 doesn't really have a use.
18982 if (i || !NoChannels)
18983 continue;
18984 } else {
18985 SDValue Op = DAG.getTargetConstant(Val: Idx, DL: SDLoc(User), VT: MVT::i32);
18986 SDNode *NewUser = DAG.UpdateNodeOperands(N: User, Op1: SDValue(NewNode, 0), Op2: Op);
18987 if (NewUser != User) {
18988 DAG.ReplaceAllUsesWith(From: SDValue(User, 0), To: SDValue(NewUser, 0));
18989 DAG.RemoveDeadNode(N: User);
18990 }
18991 }
18992
18993 switch (Idx) {
18994 default:
18995 break;
18996 case AMDGPU::sub0:
18997 Idx = AMDGPU::sub1;
18998 break;
18999 case AMDGPU::sub1:
19000 Idx = AMDGPU::sub2;
19001 break;
19002 case AMDGPU::sub2:
19003 Idx = AMDGPU::sub3;
19004 break;
19005 case AMDGPU::sub3:
19006 Idx = AMDGPU::sub4;
19007 break;
19008 }
19009 }
19010
19011 DAG.RemoveDeadNode(N: Node);
19012 return nullptr;
19013}
19014
19015static bool isFrameIndexOp(SDValue Op) {
19016 if (Op.getOpcode() == ISD::AssertZext)
19017 Op = Op.getOperand(i: 0);
19018
19019 return isa<FrameIndexSDNode>(Val: Op);
19020}
19021
19022/// Legalize target independent instructions (e.g. INSERT_SUBREG)
19023/// with frame index operands.
19024/// LLVM assumes that inputs are to these instructions are registers.
19025SDNode *
19026SITargetLowering::legalizeTargetIndependentNode(SDNode *Node,
19027 SelectionDAG &DAG) const {
19028 if (Node->getOpcode() == ISD::CopyToReg) {
19029 RegisterSDNode *DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1));
19030 SDValue SrcVal = Node->getOperand(Num: 2);
19031
19032 // Insert a copy to a VReg_1 virtual register so LowerI1Copies doesn't have
19033 // to try understanding copies to physical registers.
19034 if (SrcVal.getValueType() == MVT::i1 && DestReg->getReg().isPhysical()) {
19035 SDLoc SL(Node);
19036 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19037 SDValue VReg = DAG.getRegister(
19038 Reg: MRI.createVirtualRegister(RegClass: &AMDGPU::VReg_1RegClass), VT: MVT::i1);
19039
19040 SDNode *Glued = Node->getGluedNode();
19041 SDValue ToVReg = DAG.getCopyToReg(
19042 Chain: Node->getOperand(Num: 0), dl: SL, Reg: VReg, N: SrcVal,
19043 Glue: SDValue(Glued, Glued ? Glued->getNumValues() - 1 : 0));
19044 SDValue ToResultReg = DAG.getCopyToReg(Chain: ToVReg, dl: SL, Reg: SDValue(DestReg, 0),
19045 N: VReg, Glue: ToVReg.getValue(R: 1));
19046 DAG.ReplaceAllUsesWith(From: Node, To: ToResultReg.getNode());
19047 DAG.RemoveDeadNode(N: Node);
19048 return ToResultReg.getNode();
19049 }
19050 }
19051
19052 SmallVector<SDValue, 8> Ops;
19053 for (unsigned i = 0; i < Node->getNumOperands(); ++i) {
19054 if (!isFrameIndexOp(Op: Node->getOperand(Num: i))) {
19055 Ops.push_back(Elt: Node->getOperand(Num: i));
19056 continue;
19057 }
19058
19059 SDLoc DL(Node);
19060 Ops.push_back(Elt: SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL,
19061 VT: Node->getOperand(Num: i).getValueType(),
19062 Op1: Node->getOperand(Num: i)),
19063 0));
19064 }
19065
19066 return DAG.UpdateNodeOperands(N: Node, Ops);
19067}
19068
19069/// Fold the instructions after selecting them.
19070/// Returns null if users were already updated.
19071SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
19072 SelectionDAG &DAG) const {
19073 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19074 unsigned Opcode = Node->getMachineOpcode();
19075
19076 if (TII->isImage(Opcode) && !TII->get(Opcode).mayStore() &&
19077 !TII->isGather4(Opcode) &&
19078 AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::dmask)) {
19079 return adjustWritemask(Node, DAG);
19080 }
19081
19082 if (Opcode == AMDGPU::INSERT_SUBREG || Opcode == AMDGPU::REG_SEQUENCE) {
19083 legalizeTargetIndependentNode(Node, DAG);
19084 return Node;
19085 }
19086
19087 switch (Opcode) {
19088 case AMDGPU::V_DIV_SCALE_F32_e64:
19089 case AMDGPU::V_DIV_SCALE_F64_e64: {
19090 // Satisfy the operand register constraint when one of the inputs is
19091 // undefined. Ordinarily each undef value will have its own implicit_def of
19092 // a vreg, so force these to use a single register.
19093 SDValue Src0 = Node->getOperand(Num: 1);
19094 SDValue Src1 = Node->getOperand(Num: 3);
19095 SDValue Src2 = Node->getOperand(Num: 5);
19096
19097 if ((Src0.isMachineOpcode() &&
19098 Src0.getMachineOpcode() != AMDGPU::IMPLICIT_DEF) &&
19099 (Src0 == Src1 || Src0 == Src2))
19100 break;
19101
19102 MVT VT = Src0.getValueType().getSimpleVT();
19103 const TargetRegisterClass *RC =
19104 getRegClassFor(VT, isDivergent: Src0.getNode()->isDivergent());
19105
19106 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
19107 SDValue UndefReg = DAG.getRegister(Reg: MRI.createVirtualRegister(RegClass: RC), VT);
19108
19109 SDValue ImpDef = DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: SDLoc(Node), Reg: UndefReg,
19110 N: Src0, Glue: SDValue());
19111
19112 // src0 must be the same register as src1 or src2, even if the value is
19113 // undefined, so make sure we don't violate this constraint.
19114 if (Src0.isMachineOpcode() &&
19115 Src0.getMachineOpcode() == AMDGPU::IMPLICIT_DEF) {
19116 if (Src1.isMachineOpcode() &&
19117 Src1.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19118 Src0 = Src1;
19119 else if (Src2.isMachineOpcode() &&
19120 Src2.getMachineOpcode() != AMDGPU::IMPLICIT_DEF)
19121 Src0 = Src2;
19122 else {
19123 assert(Src1.getMachineOpcode() == AMDGPU::IMPLICIT_DEF);
19124 Src0 = UndefReg;
19125 Src1 = UndefReg;
19126 }
19127 } else
19128 break;
19129
19130 SmallVector<SDValue, 9> Ops(Node->ops());
19131 Ops[1] = Src0;
19132 Ops[3] = Src1;
19133 Ops[5] = Src2;
19134 Ops.push_back(Elt: ImpDef.getValue(R: 1));
19135 return DAG.getMachineNode(Opcode, dl: SDLoc(Node), VTs: Node->getVTList(), Ops);
19136 }
19137 default:
19138 break;
19139 }
19140
19141 return Node;
19142}
19143
19144// Any MIMG instructions that use tfe or lwe require an initialization of the
19145// result register that will be written in the case of a memory access failure.
19146// The required code is also added to tie this init code to the result of the
19147// img instruction.
19148void SITargetLowering::AddMemOpInit(MachineInstr &MI) const {
19149 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19150 const SIRegisterInfo &TRI = TII->getRegisterInfo();
19151 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
19152 MachineBasicBlock &MBB = *MI.getParent();
19153
19154 int DstIdx =
19155 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdata);
19156 unsigned InitIdx = 0;
19157
19158 if (TII->isImage(MI)) {
19159 MachineOperand *TFE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::tfe);
19160 MachineOperand *LWE = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::lwe);
19161 MachineOperand *D16 = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::d16);
19162
19163 if (!TFE && !LWE) // intersect_ray
19164 return;
19165
19166 unsigned TFEVal = TFE ? TFE->getImm() : 0;
19167 unsigned LWEVal = LWE ? LWE->getImm() : 0;
19168 unsigned D16Val = D16 ? D16->getImm() : 0;
19169
19170 if (!TFEVal && !LWEVal)
19171 return;
19172
19173 // At least one of TFE or LWE are non-zero
19174 // We have to insert a suitable initialization of the result value and
19175 // tie this to the dest of the image instruction.
19176
19177 // Calculate which dword we have to initialize to 0.
19178 MachineOperand *MO_Dmask = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::dmask);
19179
19180 // check that dmask operand is found.
19181 assert(MO_Dmask && "Expected dmask operand in instruction");
19182
19183 unsigned dmask = MO_Dmask->getImm();
19184 // Determine the number of active lanes taking into account the
19185 // Gather4 special case
19186 unsigned ActiveLanes = TII->isGather4(MI) ? 4 : llvm::popcount(Value: dmask);
19187
19188 bool Packed = !Subtarget->hasUnpackedD16VMem();
19189
19190 InitIdx = D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
19191
19192 // Abandon attempt if the dst size isn't large enough
19193 // - this is in fact an error but this is picked up elsewhere and
19194 // reported correctly.
19195 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19196
19197 uint32_t DstSize = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19198 if (DstSize < InitIdx)
19199 return;
19200 } else if (TII->isMUBUF(MI) && AMDGPU::getMUBUFTfe(Opc: MI.getOpcode())) {
19201 const TargetRegisterClass *DstRC = TII->getRegClass(MCID: MI.getDesc(), OpNum: DstIdx);
19202 InitIdx = TRI.getRegSizeInBits(RC: *DstRC) / 32;
19203 } else {
19204 return;
19205 }
19206
19207 const DebugLoc &DL = MI.getDebugLoc();
19208
19209 // Create a register for the initialization value.
19210 Register PrevDst = MRI.cloneVirtualRegister(VReg: MI.getOperand(i: DstIdx).getReg());
19211 unsigned NewDst = 0; // Final initialized value will be in here
19212
19213 // If PRTStrictNull feature is enabled (the default) then initialize
19214 // all the result registers to 0, otherwise just the error indication
19215 // register (VGPRn+1)
19216 unsigned SizeLeft = Subtarget->usePRTStrictNull() ? InitIdx : 1;
19217 unsigned CurrIdx = Subtarget->usePRTStrictNull() ? 0 : (InitIdx - 1);
19218
19219 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: PrevDst);
19220 for (; SizeLeft; SizeLeft--, CurrIdx++) {
19221 NewDst = MRI.createVirtualRegister(RegClass: TII->getOpRegClass(MI, OpNo: DstIdx));
19222 // Initialize dword
19223 Register SubReg = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
19224 // clang-format off
19225 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: SubReg)
19226 .addImm(Val: 0);
19227 // clang-format on
19228 // Insert into the super-reg
19229 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::INSERT_SUBREG), DestReg: NewDst)
19230 .addReg(RegNo: PrevDst)
19231 .addReg(RegNo: SubReg)
19232 .addImm(Val: SIRegisterInfo::getSubRegFromChannel(Channel: CurrIdx));
19233
19234 PrevDst = NewDst;
19235 }
19236
19237 // Add as an implicit operand
19238 MI.addOperand(Op: MachineOperand::CreateReg(Reg: NewDst, isDef: false, isImp: true));
19239
19240 // Tie the just added implicit operand to the dst
19241 MI.tieOperands(DefIdx: DstIdx, UseIdx: MI.getNumOperands() - 1);
19242}
19243
19244/// Assign the register class depending on the number of
19245/// bits set in the writemask
19246void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
19247 SDNode *Node) const {
19248 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19249
19250 MachineFunction *MF = MI.getMF();
19251 MachineRegisterInfo &MRI = MF->getRegInfo();
19252
19253 if (TII->isVOP3(Opcode: MI.getOpcode())) {
19254 // Make sure constant bus requirements are respected.
19255 TII->legalizeOperandsVOP3(MRI, MI);
19256
19257 if (TII->isMAI(MI)) {
19258 // The ordinary src0, src1, src2 were legalized above.
19259 //
19260 // We have to also legalize the appended v_mfma_ld_scale_b32 operands,
19261 // as a separate instruction.
19262 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19263 Name: AMDGPU::OpName::scale_src0);
19264 if (Src0Idx != -1) {
19265 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
19266 Name: AMDGPU::OpName::scale_src1);
19267 if (TII->usesConstantBus(MRI, MI, OpIdx: Src0Idx) &&
19268 TII->usesConstantBus(MRI, MI, OpIdx: Src1Idx))
19269 TII->legalizeOpWithMove(MI, OpIdx: Src1Idx);
19270 }
19271 }
19272
19273 return;
19274 }
19275
19276 if (TII->isImage(MI))
19277 TII->enforceOperandRCAlignment(MI, OpName: AMDGPU::OpName::vaddr);
19278}
19279
19280static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
19281 uint64_t Val) {
19282 SDValue K = DAG.getTargetConstant(Val, DL, VT: MVT::i32);
19283 return SDValue(DAG.getMachineNode(Opcode: AMDGPU::S_MOV_B32, dl: DL, VT: MVT::i32, Op1: K), 0);
19284}
19285
19286MachineSDNode *SITargetLowering::wrapAddr64Rsrc(SelectionDAG &DAG,
19287 const SDLoc &DL,
19288 SDValue Ptr) const {
19289 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
19290
19291 // Build the half of the subregister with the constants before building the
19292 // full 128-bit register. If we are building multiple resource descriptors,
19293 // this will allow CSEing of the 2-component register.
19294 const SDValue Ops0[] = {
19295 DAG.getTargetConstant(Val: AMDGPU::SGPR_64RegClassID, DL, VT: MVT::i32),
19296 buildSMovImm32(DAG, DL, Val: 0),
19297 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
19298 buildSMovImm32(DAG, DL, Val: TII->getDefaultRsrcDataFormat() >> 32),
19299 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32)};
19300
19301 SDValue SubRegHi = SDValue(
19302 DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v2i32, Ops: Ops0), 0);
19303
19304 // Combine the constants and the pointer.
19305 const SDValue Ops1[] = {
19306 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32), Ptr,
19307 DAG.getTargetConstant(Val: AMDGPU::sub0_sub1, DL, VT: MVT::i32), SubRegHi,
19308 DAG.getTargetConstant(Val: AMDGPU::sub2_sub3, DL, VT: MVT::i32)};
19309
19310 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops: Ops1);
19311}
19312
19313/// Return a resource descriptor with the 'Add TID' bit enabled
19314/// The TID (Thread ID) is multiplied by the stride value (bits [61:48]
19315/// of the resource descriptor) to create an offset, which is added to
19316/// the resource pointer.
19317MachineSDNode *SITargetLowering::buildRSRC(SelectionDAG &DAG, const SDLoc &DL,
19318 SDValue Ptr, uint32_t RsrcDword1,
19319 uint64_t RsrcDword2And3) const {
19320 SDValue PtrLo = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub0, DL, VT: MVT::i32, Operand: Ptr);
19321 SDValue PtrHi = DAG.getTargetExtractSubreg(SRIdx: AMDGPU::sub1, DL, VT: MVT::i32, Operand: Ptr);
19322 if (RsrcDword1) {
19323 PtrHi = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: PtrHi,
19324 N2: DAG.getConstant(Val: RsrcDword1, DL, VT: MVT::i32));
19325 }
19326
19327 SDValue DataLo =
19328 buildSMovImm32(DAG, DL, Val: RsrcDword2And3 & UINT64_C(0xFFFFFFFF));
19329 SDValue DataHi = buildSMovImm32(DAG, DL, Val: RsrcDword2And3 >> 32);
19330
19331 const SDValue Ops[] = {
19332 DAG.getTargetConstant(Val: AMDGPU::SGPR_128RegClassID, DL, VT: MVT::i32),
19333 PtrLo,
19334 DAG.getTargetConstant(Val: AMDGPU::sub0, DL, VT: MVT::i32),
19335 PtrHi,
19336 DAG.getTargetConstant(Val: AMDGPU::sub1, DL, VT: MVT::i32),
19337 DataLo,
19338 DAG.getTargetConstant(Val: AMDGPU::sub2, DL, VT: MVT::i32),
19339 DataHi,
19340 DAG.getTargetConstant(Val: AMDGPU::sub3, DL, VT: MVT::i32)};
19341
19342 return DAG.getMachineNode(Opcode: AMDGPU::REG_SEQUENCE, dl: DL, VT: MVT::v4i32, Ops);
19343}
19344
19345//===----------------------------------------------------------------------===//
19346// SI Inline Assembly Support
19347//===----------------------------------------------------------------------===//
19348
19349std::pair<unsigned, const TargetRegisterClass *>
19350SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
19351 StringRef Constraint,
19352 MVT VT) const {
19353 const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(TRI_);
19354
19355 const TargetRegisterClass *RC = nullptr;
19356 if (Constraint.size() == 1) {
19357 // Check if we cannot determine the bit size of the given value type. This
19358 // can happen, for example, in this situation where we have an empty struct
19359 // (size 0): `call void asm "", "v"({} poison)`-
19360 if (VT == MVT::Other)
19361 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19362 const unsigned BitWidth = VT.getSizeInBits();
19363 switch (Constraint[0]) {
19364 default:
19365 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19366 case 's':
19367 case 'r':
19368 switch (BitWidth) {
19369 case 16:
19370 RC = &AMDGPU::SReg_32RegClass;
19371 break;
19372 case 64:
19373 RC = &AMDGPU::SGPR_64RegClass;
19374 break;
19375 default:
19376 RC = SIRegisterInfo::getSGPRClassForBitWidth(BitWidth);
19377 if (!RC)
19378 return std::pair(0U, nullptr);
19379 break;
19380 }
19381 break;
19382 case 'v':
19383 switch (BitWidth) {
19384 case 1:
19385 return std::pair(0U, nullptr);
19386 case 16:
19387 RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass
19388 : &AMDGPU::VGPR_32_Lo256RegClass;
19389 break;
19390 default:
19391 RC = Subtarget->has1024AddressableVGPRs()
19392 ? TRI->getAlignedLo256VGPRClassForBitWidth(BitWidth)
19393 : TRI->getVGPRClassForBitWidth(BitWidth);
19394 if (!RC)
19395 return std::pair(0U, nullptr);
19396 break;
19397 }
19398 break;
19399 case 'a':
19400 if (!Subtarget->hasMAIInsts())
19401 break;
19402 switch (BitWidth) {
19403 case 1:
19404 return std::pair(0U, nullptr);
19405 case 16:
19406 RC = &AMDGPU::AGPR_32RegClass;
19407 break;
19408 default:
19409 RC = TRI->getAGPRClassForBitWidth(BitWidth);
19410 if (!RC)
19411 return std::pair(0U, nullptr);
19412 break;
19413 }
19414 break;
19415 }
19416 } else if (Constraint == "VA" && Subtarget->hasGFX90AInsts()) {
19417 const unsigned BitWidth = VT.getSizeInBits();
19418 switch (BitWidth) {
19419 case 16:
19420 RC = &AMDGPU::AV_32RegClass;
19421 break;
19422 default:
19423 RC = TRI->getVectorSuperClassForBitWidth(BitWidth);
19424 if (!RC)
19425 return std::pair(0U, nullptr);
19426 break;
19427 }
19428 }
19429
19430 // We actually support i128, i16 and f16 as inline parameters
19431 // even if they are not reported as legal
19432 if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
19433 VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
19434 return std::pair(0U, RC);
19435
19436 auto [Kind, Idx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint);
19437 if (Kind != '\0') {
19438 if (Kind == 'v') {
19439 RC = &AMDGPU::VGPR_32_Lo256RegClass;
19440 } else if (Kind == 's') {
19441 RC = &AMDGPU::SGPR_32RegClass;
19442 } else if (Kind == 'a') {
19443 RC = &AMDGPU::AGPR_32RegClass;
19444 }
19445
19446 if (RC) {
19447 if (NumRegs > 1) {
19448 if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 >= RC->getNumRegs())
19449 return std::pair(0U, nullptr);
19450
19451 uint32_t Width = NumRegs * 32;
19452 // Prohibit constraints for register ranges with a width that does not
19453 // match the required type.
19454 if (VT.SimpleTy != MVT::Other && Width != VT.getSizeInBits())
19455 return std::pair(0U, nullptr);
19456
19457 MCRegister Reg = RC->getRegister(i: Idx);
19458 if (SIRegisterInfo::isVGPRClass(RC))
19459 RC = TRI->getVGPRClassForBitWidth(BitWidth: Width);
19460 else if (SIRegisterInfo::isSGPRClass(RC))
19461 RC = TRI->getSGPRClassForBitWidth(BitWidth: Width);
19462 else if (SIRegisterInfo::isAGPRClass(RC))
19463 RC = TRI->getAGPRClassForBitWidth(BitWidth: Width);
19464 if (RC) {
19465 Reg = TRI->getMatchingSuperReg(Reg, SubIdx: AMDGPU::sub0, RC);
19466 if (!Reg) {
19467 // The register class does not contain the requested register,
19468 // e.g., because it is an SGPR pair that would violate alignment
19469 // requirements.
19470 return std::pair(0U, nullptr);
19471 }
19472 return std::pair(Reg, RC);
19473 }
19474 }
19475
19476 // Reject types that do not fit a single 32-bit register: any scalar wider
19477 // than 32 bits, or a vector that is not exactly 32 bits.
19478 if (VT.SimpleTy != MVT::Other &&
19479 (VT.getSizeInBits() > 32 ||
19480 (VT.isVector() && VT.getSizeInBits() != 32)))
19481 return std::pair(0U, nullptr);
19482 if (RC && Idx < RC->getNumRegs())
19483 return std::pair(RC->getRegister(i: Idx), RC);
19484 return std::pair(0U, nullptr);
19485 }
19486 }
19487
19488 auto Ret = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19489 if (Ret.first)
19490 Ret.second = TRI->getPhysRegBaseClass(Reg: Ret.first);
19491
19492 return Ret;
19493}
19494
19495static bool isImmConstraint(StringRef Constraint) {
19496 if (Constraint.size() == 1) {
19497 switch (Constraint[0]) {
19498 default:
19499 break;
19500 case 'I':
19501 case 'J':
19502 case 'A':
19503 case 'B':
19504 case 'C':
19505 return true;
19506 }
19507 } else if (Constraint == "DA" || Constraint == "DB") {
19508 return true;
19509 }
19510 return false;
19511}
19512
19513SITargetLowering::ConstraintType
19514SITargetLowering::getConstraintType(StringRef Constraint) const {
19515 if (Constraint.size() == 1) {
19516 switch (Constraint[0]) {
19517 default:
19518 break;
19519 case 's':
19520 case 'v':
19521 case 'a':
19522 return C_RegisterClass;
19523 }
19524 } else if (Constraint.size() == 2) {
19525 if (Constraint == "VA")
19526 return C_RegisterClass;
19527 }
19528 if (isImmConstraint(Constraint)) {
19529 return C_Other;
19530 }
19531 return TargetLowering::getConstraintType(Constraint);
19532}
19533
19534static uint64_t clearUnusedBits(uint64_t Val, unsigned Size) {
19535 if (!AMDGPU::isInlinableIntLiteral(Literal: Val)) {
19536 Val = Val & maskTrailingOnes<uint64_t>(N: Size);
19537 }
19538 return Val;
19539}
19540
19541void SITargetLowering::LowerAsmOperandForConstraint(SDValue Op,
19542 StringRef Constraint,
19543 std::vector<SDValue> &Ops,
19544 SelectionDAG &DAG) const {
19545 if (isImmConstraint(Constraint)) {
19546 uint64_t Val;
19547 if (getAsmOperandConstVal(Op, Val) &&
19548 checkAsmConstraintVal(Op, Constraint, Val)) {
19549 Val = clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits());
19550 Ops.push_back(x: DAG.getTargetConstant(Val, DL: SDLoc(Op), VT: MVT::i64));
19551 }
19552 } else {
19553 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
19554 }
19555}
19556
19557bool SITargetLowering::getAsmOperandConstVal(SDValue Op, uint64_t &Val) const {
19558 unsigned Size = Op.getScalarValueSizeInBits();
19559 if (Size > 64)
19560 return false;
19561
19562 if (Size == 16 && !Subtarget->has16BitInsts())
19563 return false;
19564
19565 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
19566 Val = C->getSExtValue();
19567 return true;
19568 }
19569 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
19570 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
19571 return true;
19572 }
19573 if (BuildVectorSDNode *V = dyn_cast<BuildVectorSDNode>(Val&: Op)) {
19574 if (Size != 16 || Op.getNumOperands() != 2)
19575 return false;
19576 if (Op.getOperand(i: 0).isUndef() || Op.getOperand(i: 1).isUndef())
19577 return false;
19578 if (ConstantSDNode *C = V->getConstantSplatNode()) {
19579 Val = C->getSExtValue();
19580 return true;
19581 }
19582 if (ConstantFPSDNode *C = V->getConstantFPSplatNode()) {
19583 Val = C->getValueAPF().bitcastToAPInt().getSExtValue();
19584 return true;
19585 }
19586 }
19587
19588 return false;
19589}
19590
19591bool SITargetLowering::checkAsmConstraintVal(SDValue Op, StringRef Constraint,
19592 uint64_t Val) const {
19593 if (Constraint.size() == 1) {
19594 switch (Constraint[0]) {
19595 case 'I':
19596 return AMDGPU::isInlinableIntLiteral(Literal: Val);
19597 case 'J':
19598 return isInt<16>(x: Val);
19599 case 'A':
19600 return checkAsmConstraintValA(Op, Val);
19601 case 'B':
19602 return isInt<32>(x: Val);
19603 case 'C':
19604 return isUInt<32>(x: clearUnusedBits(Val, Size: Op.getScalarValueSizeInBits())) ||
19605 AMDGPU::isInlinableIntLiteral(Literal: Val);
19606 default:
19607 break;
19608 }
19609 } else if (Constraint.size() == 2) {
19610 if (Constraint == "DA") {
19611 int64_t HiBits = static_cast<int32_t>(Val >> 32);
19612 int64_t LoBits = static_cast<int32_t>(Val);
19613 return checkAsmConstraintValA(Op, Val: HiBits, MaxSize: 32) &&
19614 checkAsmConstraintValA(Op, Val: LoBits, MaxSize: 32);
19615 }
19616 if (Constraint == "DB") {
19617 return true;
19618 }
19619 }
19620 llvm_unreachable("Invalid asm constraint");
19621}
19622
19623bool SITargetLowering::checkAsmConstraintValA(SDValue Op, uint64_t Val,
19624 unsigned MaxSize) const {
19625 unsigned Size = std::min<unsigned>(a: Op.getScalarValueSizeInBits(), b: MaxSize);
19626 bool HasInv2Pi = Subtarget->hasInv2PiInlineImm();
19627 if (Size == 16) {
19628 MVT VT = Op.getSimpleValueType();
19629 switch (VT.SimpleTy) {
19630 default:
19631 return false;
19632 case MVT::i16:
19633 return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi);
19634 case MVT::f16:
19635 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi);
19636 case MVT::bf16:
19637 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi);
19638 case MVT::v2i16:
19639 return AMDGPU::getInlineEncodingV2I16(Literal: Val).has_value();
19640 case MVT::v2f16:
19641 return AMDGPU::getInlineEncodingV2F16(Literal: Val).has_value();
19642 case MVT::v2bf16:
19643 return AMDGPU::getInlineEncodingV2BF16(Literal: Val).has_value();
19644 }
19645 }
19646 if ((Size == 32 && AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi)) ||
19647 (Size == 64 && AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi)))
19648 return true;
19649 return false;
19650}
19651
19652static int getAlignedAGPRClassID(unsigned UnalignedClassID) {
19653 switch (UnalignedClassID) {
19654 case AMDGPU::VReg_64RegClassID:
19655 return AMDGPU::VReg_64_Align2RegClassID;
19656 case AMDGPU::VReg_96RegClassID:
19657 return AMDGPU::VReg_96_Align2RegClassID;
19658 case AMDGPU::VReg_128RegClassID:
19659 return AMDGPU::VReg_128_Align2RegClassID;
19660 case AMDGPU::VReg_160RegClassID:
19661 return AMDGPU::VReg_160_Align2RegClassID;
19662 case AMDGPU::VReg_192RegClassID:
19663 return AMDGPU::VReg_192_Align2RegClassID;
19664 case AMDGPU::VReg_224RegClassID:
19665 return AMDGPU::VReg_224_Align2RegClassID;
19666 case AMDGPU::VReg_256RegClassID:
19667 return AMDGPU::VReg_256_Align2RegClassID;
19668 case AMDGPU::VReg_288RegClassID:
19669 return AMDGPU::VReg_288_Align2RegClassID;
19670 case AMDGPU::VReg_320RegClassID:
19671 return AMDGPU::VReg_320_Align2RegClassID;
19672 case AMDGPU::VReg_352RegClassID:
19673 return AMDGPU::VReg_352_Align2RegClassID;
19674 case AMDGPU::VReg_384RegClassID:
19675 return AMDGPU::VReg_384_Align2RegClassID;
19676 case AMDGPU::VReg_512RegClassID:
19677 return AMDGPU::VReg_512_Align2RegClassID;
19678 case AMDGPU::VReg_1024RegClassID:
19679 return AMDGPU::VReg_1024_Align2RegClassID;
19680 case AMDGPU::AReg_64RegClassID:
19681 return AMDGPU::AReg_64_Align2RegClassID;
19682 case AMDGPU::AReg_96RegClassID:
19683 return AMDGPU::AReg_96_Align2RegClassID;
19684 case AMDGPU::AReg_128RegClassID:
19685 return AMDGPU::AReg_128_Align2RegClassID;
19686 case AMDGPU::AReg_160RegClassID:
19687 return AMDGPU::AReg_160_Align2RegClassID;
19688 case AMDGPU::AReg_192RegClassID:
19689 return AMDGPU::AReg_192_Align2RegClassID;
19690 case AMDGPU::AReg_256RegClassID:
19691 return AMDGPU::AReg_256_Align2RegClassID;
19692 case AMDGPU::AReg_512RegClassID:
19693 return AMDGPU::AReg_512_Align2RegClassID;
19694 case AMDGPU::AReg_1024RegClassID:
19695 return AMDGPU::AReg_1024_Align2RegClassID;
19696 default:
19697 return -1;
19698 }
19699}
19700
19701// Figure out which registers should be reserved for stack access. Only after
19702// the function is legalized do we know all of the non-spill stack objects or if
19703// calls are present.
19704void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
19705 MachineRegisterInfo &MRI = MF.getRegInfo();
19706 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
19707 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
19708 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
19709 const SIInstrInfo *TII = ST.getInstrInfo();
19710
19711 if (Info->isEntryFunction()) {
19712 // Callable functions have fixed registers used for stack access.
19713 reservePrivateMemoryRegs(TM: getTargetMachine(), MF, TRI: *TRI, Info&: *Info);
19714 }
19715
19716 // TODO: Move this logic to getReservedRegs()
19717 // Reserve the SGPR(s) to save/restore EXEC for WWM spill/copy handling.
19718 unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
19719 Register SReg = ST.isWave32()
19720 ? AMDGPU::SGPR_32RegClass.getRegister(i: MaxNumSGPRs - 1)
19721 : TRI->getAlignedHighSGPRForRC(MF, /*Align=*/2,
19722 RC: &AMDGPU::SGPR_64RegClass);
19723 Info->setSGPRForEXECCopy(SReg);
19724
19725 assert(!TRI->isSubRegister(Info->getScratchRSrcReg(),
19726 Info->getStackPtrOffsetReg()));
19727 if (Info->getStackPtrOffsetReg() != AMDGPU::SP_REG)
19728 MRI.replaceRegWith(FromReg: AMDGPU::SP_REG, ToReg: Info->getStackPtrOffsetReg());
19729
19730 // We need to worry about replacing the default register with itself in case
19731 // of MIR testcases missing the MFI.
19732 if (Info->getScratchRSrcReg() != AMDGPU::PRIVATE_RSRC_REG)
19733 MRI.replaceRegWith(FromReg: AMDGPU::PRIVATE_RSRC_REG, ToReg: Info->getScratchRSrcReg());
19734
19735 if (Info->getFrameOffsetReg() != AMDGPU::FP_REG)
19736 MRI.replaceRegWith(FromReg: AMDGPU::FP_REG, ToReg: Info->getFrameOffsetReg());
19737
19738 Info->limitOccupancy(MF);
19739
19740 if (ST.isWave32() && !MF.empty()) {
19741 for (auto &MBB : MF) {
19742 for (auto &MI : MBB) {
19743 TII->fixImplicitOperands(MI);
19744 }
19745 }
19746 }
19747
19748 // FIXME: This is a hack to fixup AGPR classes to use the properly aligned
19749 // classes if required. Ideally the register class constraints would differ
19750 // per-subtarget, but there's no easy way to achieve that right now. This is
19751 // not a problem for VGPRs because the correctly aligned VGPR class is implied
19752 // from using them as the register class for legal types.
19753 if (ST.needsAlignedVGPRs()) {
19754 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
19755 const Register Reg = Register::index2VirtReg(Index: I);
19756 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);
19757 if (!RC)
19758 continue;
19759 int NewClassID = getAlignedAGPRClassID(UnalignedClassID: RC->getID());
19760 if (NewClassID != -1)
19761 MRI.setRegClass(Reg, RC: TRI->getRegClass(i: NewClassID));
19762 }
19763 }
19764
19765 TargetLoweringBase::finalizeLowering(MF);
19766}
19767
19768void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
19769 KnownBits &Known,
19770 const APInt &DemandedElts,
19771 const SelectionDAG &DAG,
19772 unsigned Depth) const {
19773 Known.resetAll();
19774 unsigned Opc = Op.getOpcode();
19775 switch (Opc) {
19776 case ISD::INTRINSIC_WO_CHAIN: {
19777 unsigned IID = Op.getConstantOperandVal(i: 0);
19778 switch (IID) {
19779 case Intrinsic::amdgcn_mbcnt_lo:
19780 case Intrinsic::amdgcn_mbcnt_hi: {
19781 const GCNSubtarget &ST =
19782 DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
19783 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
19784 // most 31 + src1.
19785 Known.Zero.setBitsFrom(
19786 IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
19787 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
19788 Known = KnownBits::add(LHS: Known, RHS: Known2);
19789 return;
19790 }
19791 }
19792 break;
19793 }
19794 }
19795 return AMDGPUTargetLowering::computeKnownBitsForTargetNode(
19796 Op, Known, DemandedElts, DAG, Depth);
19797}
19798
19799void SITargetLowering::computeKnownBitsForStackObjectPointer(
19800 KnownBits &Known, const MachineFunction &MF, Align Alignment) const {
19801 TargetLowering::computeKnownBitsForStackObjectPointer(Known, MF, Alignment);
19802
19803 // Set the high bits to zero based on the maximum allowed scratch size per
19804 // wave. We can't use vaddr in MUBUF instructions if we don't know the address
19805 // calculation won't overflow, so assume the sign bit is never set.
19806 Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex());
19807}
19808
19809static void knownBitsForWorkitemID(const GCNSubtarget &ST,
19810 GISelValueTracking &VT, KnownBits &Known,
19811 unsigned Dim) {
19812 unsigned MaxValue =
19813 ST.getMaxWorkitemID(Kernel: VT.getMachineFunction().getFunction(), Dimension: Dim);
19814 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
19815}
19816
19817static void knownBitsForSBFE(const MachineInstr &MI, GISelValueTracking &VT,
19818 KnownBits &Known, const APInt &DemandedElts,
19819 unsigned BFEWidth, bool SExt, unsigned Depth) {
19820 const MachineRegisterInfo &MRI = VT.getMachineFunction().getRegInfo();
19821 const MachineOperand &Src1 = MI.getOperand(i: 2);
19822
19823 unsigned Src1Cst = 0;
19824 if (Src1.isImm()) {
19825 Src1Cst = Src1.getImm();
19826 } else if (Src1.isReg()) {
19827 auto Cst = getIConstantVRegValWithLookThrough(VReg: Src1.getReg(), MRI);
19828 if (!Cst)
19829 return;
19830 Src1Cst = Cst->Value.getZExtValue();
19831 } else {
19832 return;
19833 }
19834
19835 // Offset is at bits [4:0] for 32 bit, [5:0] for 64 bit.
19836 // Width is always [22:16].
19837 const unsigned Offset =
19838 Src1Cst & maskTrailingOnes<unsigned>(N: (BFEWidth == 32) ? 5 : 6);
19839 const unsigned Width = (Src1Cst >> 16) & maskTrailingOnes<unsigned>(N: 6);
19840
19841 if (Width >= BFEWidth) // Ill-formed.
19842 return;
19843
19844 VT.computeKnownBitsImpl(R: MI.getOperand(i: 1).getReg(), Known, DemandedElts,
19845 Depth: Depth + 1);
19846
19847 Known = Known.extractBits(NumBits: Width, BitPosition: Offset);
19848
19849 if (SExt)
19850 Known = Known.sext(BitWidth: BFEWidth);
19851 else
19852 Known = Known.zext(BitWidth: BFEWidth);
19853}
19854
19855void SITargetLowering::computeKnownBitsForTargetInstr(
19856 GISelValueTracking &VT, Register R, KnownBits &Known,
19857 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
19858 unsigned Depth) const {
19859 Known.resetAll();
19860 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
19861 switch (MI->getOpcode()) {
19862 case AMDGPU::S_BFE_I32:
19863 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
19864 /*SExt=*/true, Depth);
19865 case AMDGPU::S_BFE_U32:
19866 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 32,
19867 /*SExt=*/false, Depth);
19868 case AMDGPU::S_BFE_I64:
19869 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
19870 /*SExt=*/true, Depth);
19871 case AMDGPU::S_BFE_U64:
19872 return knownBitsForSBFE(MI: *MI, VT, Known, DemandedElts, /*Width=*/BFEWidth: 64,
19873 /*SExt=*/false, Depth);
19874 case AMDGPU::G_INTRINSIC:
19875 case AMDGPU::G_INTRINSIC_CONVERGENT: {
19876 Intrinsic::ID IID = cast<GIntrinsic>(Val: MI)->getIntrinsicID();
19877 switch (IID) {
19878 case Intrinsic::amdgcn_workitem_id_x:
19879 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 0);
19880 break;
19881 case Intrinsic::amdgcn_workitem_id_y:
19882 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 1);
19883 break;
19884 case Intrinsic::amdgcn_workitem_id_z:
19885 knownBitsForWorkitemID(ST: *getSubtarget(), VT, Known, Dim: 2);
19886 break;
19887 case Intrinsic::amdgcn_mbcnt_lo:
19888 case Intrinsic::amdgcn_mbcnt_hi: {
19889 // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
19890 // most 31 + src1.
19891 Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
19892 ? getSubtarget()->getWavefrontSizeLog2()
19893 : 5);
19894 KnownBits Known2;
19895 VT.computeKnownBitsImpl(R: MI->getOperand(i: 3).getReg(), Known&: Known2, DemandedElts,
19896 Depth: Depth + 1);
19897 Known = KnownBits::add(LHS: Known, RHS: Known2);
19898 break;
19899 }
19900 case Intrinsic::amdgcn_groupstaticsize: {
19901 // We can report everything over the maximum size as 0. We can't report
19902 // based on the actual size because we don't know if it's accurate or not
19903 // at any given point.
19904 Known.Zero.setHighBits(
19905 llvm::countl_zero(Val: getSubtarget()->getAddressableLocalMemorySize()));
19906 break;
19907 }
19908 }
19909 break;
19910 }
19911 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
19912 Known.Zero.setHighBits(24);
19913 break;
19914 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
19915 Known.Zero.setHighBits(16);
19916 break;
19917 case AMDGPU::G_AMDGPU_COPY_SCC_VCC:
19918 // G_AMDGPU_COPY_SCC_VCC converts a uniform boolean in VCC to SGPR s32,
19919 // producing exactly 0 or 1.
19920 Known.Zero.setHighBits(Known.getBitWidth() - 1);
19921 break;
19922 case AMDGPU::G_AMDGPU_SMED3:
19923 case AMDGPU::G_AMDGPU_UMED3: {
19924 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
19925
19926 KnownBits Known2;
19927 VT.computeKnownBitsImpl(R: Src2, Known&: Known2, DemandedElts, Depth: Depth + 1);
19928 if (Known2.isUnknown())
19929 break;
19930
19931 KnownBits Known1;
19932 VT.computeKnownBitsImpl(R: Src1, Known&: Known1, DemandedElts, Depth: Depth + 1);
19933 if (Known1.isUnknown())
19934 break;
19935
19936 KnownBits Known0;
19937 VT.computeKnownBitsImpl(R: Src0, Known&: Known0, DemandedElts, Depth: Depth + 1);
19938 if (Known0.isUnknown())
19939 break;
19940
19941 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
19942 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
19943 Known.One = Known0.One & Known1.One & Known2.One;
19944 break;
19945 }
19946 }
19947}
19948
19949Align SITargetLowering::computeKnownAlignForTargetInstr(
19950 GISelValueTracking &VT, Register R, const MachineRegisterInfo &MRI,
19951 unsigned Depth) const {
19952 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
19953 if (auto *GI = dyn_cast<GIntrinsic>(Val: MI)) {
19954 // FIXME: Can this move to generic code? What about the case where the call
19955 // site specifies a lower alignment?
19956 Intrinsic::ID IID = GI->getIntrinsicID();
19957 LLVMContext &Ctx = VT.getMachineFunction().getFunction().getContext();
19958 AttributeList Attrs =
19959 Intrinsic::getAttributes(C&: Ctx, id: IID, FT: Intrinsic::getType(Context&: Ctx, id: IID));
19960 if (MaybeAlign RetAlign = Attrs.getRetAlignment())
19961 return *RetAlign;
19962 }
19963 return Align(1);
19964}
19965
19966Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
19967 const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
19968 const Align CacheLineAlign = Align(64);
19969
19970 // GFX950: Prevent an 8-byte instruction at loop header from being split by
19971 // the 32-byte instruction fetch window boundary. This avoids a significant
19972 // fetch delay after backward branch. We use 32-byte alignment with max
19973 // padding of 4 bytes (one s_nop), see getMaxPermittedBytesForAlignment().
19974 if (ML && !DisableLoopAlignment &&
19975 getSubtarget()->hasLoopHeadInstSplitSensitivity()) {
19976 const MachineBasicBlock *Header = ML->getHeader();
19977 // Respect user-specified or previously set alignment.
19978 if (Header->getAlignment() != PrefAlign)
19979 return Header->getAlignment();
19980 if (needsFetchWindowAlignment(MBB: *Header))
19981 return Align(32);
19982 }
19983
19984 // Pre-GFX10 target did not benefit from loop alignment
19985 if (!ML || DisableLoopAlignment || !getSubtarget()->hasInstPrefetch() ||
19986 getSubtarget()->hasInstFwdPrefetchBug())
19987 return PrefAlign;
19988
19989 // On GFX10 I$ is 4 x 64 bytes cache lines.
19990 // By default prefetcher keeps one cache line behind and reads two ahead.
19991 // We can modify it with S_INST_PREFETCH for larger loops to have two lines
19992 // behind and one ahead.
19993 // Therefor we can benefit from aligning loop headers if loop fits 192 bytes.
19994 // If loop fits 64 bytes it always spans no more than two cache lines and
19995 // does not need an alignment.
19996 // Else if loop is less or equal 128 bytes we do not need to modify prefetch,
19997 // Else if loop is less or equal 192 bytes we need two lines behind.
19998
19999 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20000 const MachineBasicBlock *Header = ML->getHeader();
20001 if (Header->getAlignment() != PrefAlign)
20002 return Header->getAlignment(); // Already processed.
20003
20004 unsigned LoopSize = 0;
20005 for (const MachineBasicBlock *MBB : ML->blocks()) {
20006 // If inner loop block is aligned assume in average half of the alignment
20007 // size to be added as nops.
20008 if (MBB != Header)
20009 LoopSize += MBB->getAlignment().value() / 2;
20010
20011 for (const MachineInstr &MI : *MBB) {
20012 LoopSize += TII->getInstSizeInBytes(MI);
20013 if (LoopSize > 192)
20014 return PrefAlign;
20015 }
20016 }
20017
20018 if (LoopSize <= 64)
20019 return PrefAlign;
20020
20021 if (LoopSize <= 128)
20022 return CacheLineAlign;
20023
20024 // If any of parent loops is surrounded by prefetch instructions do not
20025 // insert new for inner loop, which would reset parent's settings.
20026 for (MachineLoop *P = ML->getParentLoop(); P; P = P->getParentLoop()) {
20027 if (MachineBasicBlock *Exit = P->getExitBlock()) {
20028 auto I = Exit->getFirstNonDebugInstr();
20029 if (I != Exit->end() && I->getOpcode() == AMDGPU::S_INST_PREFETCH)
20030 return CacheLineAlign;
20031 }
20032 }
20033
20034 MachineBasicBlock *Pre = ML->getLoopPreheader();
20035 MachineBasicBlock *Exit = ML->getExitBlock();
20036
20037 if (Pre && Exit) {
20038 auto PreTerm = Pre->getFirstTerminator();
20039 if (PreTerm == Pre->begin() ||
20040 std::prev(x: PreTerm)->getOpcode() != AMDGPU::S_INST_PREFETCH)
20041 BuildMI(BB&: *Pre, I: PreTerm, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20042 .addImm(Val: 1); // prefetch 2 lines behind PC
20043
20044 auto ExitHead = Exit->getFirstNonDebugInstr();
20045 if (ExitHead == Exit->end() ||
20046 ExitHead->getOpcode() != AMDGPU::S_INST_PREFETCH)
20047 BuildMI(BB&: *Exit, I: ExitHead, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::S_INST_PREFETCH))
20048 .addImm(Val: 2); // prefetch 1 line behind PC
20049 }
20050
20051 return CacheLineAlign;
20052}
20053
20054unsigned SITargetLowering::getMaxPermittedBytesForAlignment(
20055 MachineBasicBlock *MBB) const {
20056 // GFX950: Limit padding to 4 bytes (one s_nop) for blocks where an 8-byte
20057 // instruction could be split by the 32-byte fetch window boundary.
20058 // See getPrefLoopAlignment() for context.
20059 if (needsFetchWindowAlignment(MBB: *MBB))
20060 return 4;
20061 return TargetLowering::getMaxPermittedBytesForAlignment(MBB);
20062}
20063
20064bool SITargetLowering::needsFetchWindowAlignment(
20065 const MachineBasicBlock &MBB) const {
20066 if (!getSubtarget()->hasLoopHeadInstSplitSensitivity())
20067 return false;
20068 const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
20069 for (const MachineInstr &MI : MBB) {
20070 if (MI.isMetaInstruction())
20071 continue;
20072 // Instructions larger than 4 bytes can be split by a 32-byte boundary.
20073 return TII->getInstSizeInBytes(MI) > 4;
20074 }
20075 return false;
20076}
20077
20078[[maybe_unused]]
20079static bool isCopyFromRegOfInlineAsm(const SDNode *N) {
20080 assert(N->getOpcode() == ISD::CopyFromReg);
20081 do {
20082 // Follow the chain until we find an INLINEASM node.
20083 N = N->getOperand(Num: 0).getNode();
20084 if (N->getOpcode() == ISD::INLINEASM || N->getOpcode() == ISD::INLINEASM_BR)
20085 return true;
20086 } while (N->getOpcode() == ISD::CopyFromReg);
20087 return false;
20088}
20089
20090bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode *N,
20091 FunctionLoweringInfo *FLI,
20092 UniformityInfo *UA) const {
20093 switch (N->getOpcode()) {
20094 case ISD::CopyFromReg: {
20095 const RegisterSDNode *R = cast<RegisterSDNode>(Val: N->getOperand(Num: 1));
20096 const MachineRegisterInfo &MRI = FLI->MF->getRegInfo();
20097 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20098 Register Reg = R->getReg();
20099
20100 // FIXME: Why does this need to consider isLiveIn?
20101 if (Reg.isPhysical() || MRI.isLiveIn(Reg))
20102 return !TRI->isSGPRReg(MRI, Reg);
20103
20104 if (const Value *V = FLI->getValueFromVirtualReg(Vreg: R->getReg()))
20105 return UA->isDivergentAtDef(V);
20106
20107 assert(Reg == FLI->DemoteRegister || isCopyFromRegOfInlineAsm(N));
20108 return !TRI->isSGPRReg(MRI, Reg);
20109 }
20110 case ISD::LOAD: {
20111 const LoadSDNode *L = cast<LoadSDNode>(Val: N);
20112 unsigned AS = L->getAddressSpace();
20113 // A flat load may access private memory.
20114 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
20115 }
20116 case ISD::CALLSEQ_END:
20117 return true;
20118 case ISD::INTRINSIC_WO_CHAIN:
20119 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 0));
20120 case ISD::INTRINSIC_W_CHAIN:
20121 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: N->getConstantOperandVal(Num: 1));
20122 case AMDGPUISD::ATOMIC_CMP_SWAP:
20123 case AMDGPUISD::BUFFER_ATOMIC_SWAP:
20124 case AMDGPUISD::BUFFER_ATOMIC_ADD:
20125 case AMDGPUISD::BUFFER_ATOMIC_SUB:
20126 case AMDGPUISD::BUFFER_ATOMIC_SMIN:
20127 case AMDGPUISD::BUFFER_ATOMIC_UMIN:
20128 case AMDGPUISD::BUFFER_ATOMIC_SMAX:
20129 case AMDGPUISD::BUFFER_ATOMIC_UMAX:
20130 case AMDGPUISD::BUFFER_ATOMIC_AND:
20131 case AMDGPUISD::BUFFER_ATOMIC_OR:
20132 case AMDGPUISD::BUFFER_ATOMIC_XOR:
20133 case AMDGPUISD::BUFFER_ATOMIC_INC:
20134 case AMDGPUISD::BUFFER_ATOMIC_DEC:
20135 case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP:
20136 case AMDGPUISD::BUFFER_ATOMIC_FADD:
20137 case AMDGPUISD::BUFFER_ATOMIC_FMIN:
20138 case AMDGPUISD::BUFFER_ATOMIC_FMAX:
20139 // Target-specific read-modify-write atomics are sources of divergence.
20140 return true;
20141 default:
20142 if (auto *A = dyn_cast<AtomicSDNode>(Val: N)) {
20143 // Generic read-modify-write atomics are sources of divergence.
20144 return A->readMem() && A->writeMem();
20145 }
20146 return false;
20147 }
20148}
20149
20150bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG,
20151 EVT VT) const {
20152 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
20153 case MVT::f32:
20154 return !denormalModeIsFlushAllF32(MF: DAG.getMachineFunction());
20155 case MVT::f64:
20156 case MVT::f16:
20157 return !denormalModeIsFlushAllF64F16(MF: DAG.getMachineFunction());
20158 default:
20159 return false;
20160 }
20161}
20162
20163bool SITargetLowering::denormalsEnabledForType(
20164 LLT Ty, const MachineFunction &MF) const {
20165 switch (Ty.getScalarSizeInBits()) {
20166 case 32:
20167 return !denormalModeIsFlushAllF32(MF);
20168 case 64:
20169 case 16:
20170 return !denormalModeIsFlushAllF64F16(MF);
20171 default:
20172 return false;
20173 }
20174}
20175
20176bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
20177 const APInt &DemandedElts,
20178 const SelectionDAG &DAG,
20179 bool SNaN,
20180 unsigned Depth) const {
20181 if (Op.getOpcode() == AMDGPUISD::CLAMP) {
20182 const MachineFunction &MF = DAG.getMachineFunction();
20183 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
20184
20185 if (Info->getMode().DX10Clamp)
20186 return true; // Clamped to 0.
20187 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
20188 }
20189
20190 return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
20191 DAG, SNaN, Depth);
20192}
20193
20194// On older subtargets, global FP atomic instructions have a hardcoded FP mode
20195// and do not support FP32 denormals, and only support v2f16/f64 denormals.
20196static bool atomicIgnoresDenormalModeOrFPModeIsFTZ(const AtomicRMWInst *RMW) {
20197 if (RMW->hasMetadata(Kind: "amdgpu.ignore.denormal.mode"))
20198 return true;
20199
20200 const fltSemantics &Flt = RMW->getType()->getScalarType()->getFltSemantics();
20201 auto DenormMode = RMW->getFunction()->getDenormalMode(FPType: Flt);
20202 if (DenormMode == DenormalMode::getPreserveSign())
20203 return true;
20204
20205 // TODO: Remove this.
20206 return RMW->getFunction()
20207 ->getFnAttribute(Kind: "amdgpu-unsafe-fp-atomics")
20208 .getValueAsBool();
20209}
20210
20211static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW) {
20212 LLVMContext &Ctx = RMW->getContext();
20213 StringRef MemScope =
20214 Ctx.getSyncScopeName(Id: RMW->getSyncScopeID()).value_or(u: "system");
20215
20216 return OptimizationRemark(DEBUG_TYPE, "Passed", RMW)
20217 << "Hardware instruction generated for atomic "
20218 << RMW->getOperationName(Op: RMW->getOperation())
20219 << " operation at memory scope " << MemScope;
20220}
20221
20222static bool isV2F16OrV2BF16(Type *Ty) {
20223 if (auto *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
20224 Type *EltTy = VT->getElementType();
20225 return VT->getNumElements() == 2 &&
20226 (EltTy->isHalfTy() || EltTy->isBFloatTy());
20227 }
20228
20229 return false;
20230}
20231
20232static bool isV2F16(Type *Ty) {
20233 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
20234 return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
20235}
20236
20237static bool isV2BF16(Type *Ty) {
20238 FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty);
20239 return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
20240}
20241
20242/// \return true if atomicrmw integer ops work for the type.
20243static bool isAtomicRMWLegalIntTy(Type *Ty) {
20244 if (auto *IT = dyn_cast<IntegerType>(Val: Ty)) {
20245 unsigned BW = IT->getBitWidth();
20246 return BW == 32 || BW == 64;
20247 }
20248
20249 return false;
20250}
20251
20252/// \return true if this atomicrmw xchg type can be selected.
20253static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
20254 Type *Ty = RMW->getType();
20255 if (isAtomicRMWLegalIntTy(Ty))
20256 return true;
20257
20258 if (PointerType *PT = dyn_cast<PointerType>(Val: Ty)) {
20259 const DataLayout &DL = RMW->getFunction()->getParent()->getDataLayout();
20260 unsigned BW = DL.getPointerSizeInBits(AS: PT->getAddressSpace());
20261 return BW == 32 || BW == 64;
20262 }
20263
20264 if (Ty->isFloatTy() || Ty->isDoubleTy())
20265 return true;
20266
20267 if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Val: Ty)) {
20268 return VT->getNumElements() == 2 &&
20269 VT->getElementType()->getPrimitiveSizeInBits() == 16;
20270 }
20271
20272 return false;
20273}
20274
20275/// \returns true if it's valid to emit a native instruction for \p RMW, based
20276/// on the properties of the target memory.
20277static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
20278 const AtomicRMWInst *RMW,
20279 bool HasSystemScope) {
20280 // The remote/fine-grained access logic is different from the integer
20281 // atomics. Without AgentScopeFineGrainedRemoteMemoryAtomics support,
20282 // fine-grained access does not work, even for a device local allocation.
20283 //
20284 // With AgentScopeFineGrainedRemoteMemoryAtomics, system scoped device local
20285 // allocations work.
20286 if (HasSystemScope) {
20287 if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics() &&
20288 RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
20289 return true;
20290 if (Subtarget.hasEmulatedSystemScopeAtomics())
20291 return true;
20292 } else if (Subtarget.hasAgentScopeFineGrainedRemoteMemoryAtomics())
20293 return true;
20294
20295 return RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory");
20296}
20297
20298/// \return Action to perform on AtomicRMWInsts for integer operations.
20299static TargetLowering::AtomicExpansionKind
20300atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
20301 return isAtomicRMWLegalIntTy(Ty: RMW->getType())
20302 ? TargetLowering::AtomicExpansionKind::None
20303 : TargetLowering::AtomicExpansionKind::CmpXChg;
20304}
20305
20306/// Return if a flat address space atomicrmw can access private memory.
20307static bool flatInstrMayAccessPrivate(const Instruction *I) {
20308 const MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_noalias_addrspace);
20309 return !MD ||
20310 !AMDGPU::hasValueInRangeLikeMetadata(MD: *MD, Val: AMDGPUAS::PRIVATE_ADDRESS);
20311}
20312
20313static TargetLowering::AtomicExpansionKind
20314getPrivateAtomicExpansionKind(const GCNSubtarget &STI) {
20315 // For GAS, lower to flat atomic.
20316 return STI.hasGloballyAddressableScratch()
20317 ? TargetLowering::AtomicExpansionKind::CustomExpand
20318 : TargetLowering::AtomicExpansionKind::NotAtomic;
20319}
20320
20321TargetLowering::AtomicExpansionKind
20322SITargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
20323 unsigned AS = RMW->getPointerAddressSpace();
20324 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
20325 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
20326
20327 // 64-bit flat atomics that dynamically reside in private memory will silently
20328 // be dropped.
20329 //
20330 // Note that we will emit a new copy of the original atomic in the expansion,
20331 // which will be incrementally relegalized.
20332 const DataLayout &DL = RMW->getFunction()->getDataLayout();
20333 if (AS == AMDGPUAS::FLAT_ADDRESS &&
20334 DL.getTypeSizeInBits(Ty: RMW->getType()) == 64 &&
20335 flatInstrMayAccessPrivate(I: RMW))
20336 return AtomicExpansionKind::CustomExpand;
20337
20338 auto ReportUnsafeHWInst = [=](TargetLowering::AtomicExpansionKind Kind) {
20339 OptimizationRemarkEmitter ORE(RMW->getFunction());
20340 ORE.emit(RemarkBuilder: [=]() {
20341 return emitAtomicRMWLegalRemark(RMW) << " due to an unsafe request.";
20342 });
20343 return Kind;
20344 };
20345
20346 auto SSID = RMW->getSyncScopeID();
20347 bool HasSystemScope =
20348 SSID == SyncScope::System ||
20349 SSID == RMW->getContext().getOrInsertSyncScopeID(SSN: "one-as");
20350
20351 auto Op = RMW->getOperation();
20352 switch (Op) {
20353 case AtomicRMWInst::Xchg:
20354 // PCIe supports add and xchg for system atomics.
20355 return isAtomicRMWLegalXChgTy(RMW)
20356 ? TargetLowering::AtomicExpansionKind::None
20357 : TargetLowering::AtomicExpansionKind::CmpXChg;
20358 case AtomicRMWInst::Add:
20359 // PCIe supports add and xchg for system atomics.
20360 return atomicSupportedIfLegalIntType(RMW);
20361 case AtomicRMWInst::Sub:
20362 case AtomicRMWInst::And:
20363 case AtomicRMWInst::Or:
20364 case AtomicRMWInst::Xor:
20365 case AtomicRMWInst::Max:
20366 case AtomicRMWInst::Min:
20367 case AtomicRMWInst::UMax:
20368 case AtomicRMWInst::UMin:
20369 case AtomicRMWInst::UIncWrap:
20370 case AtomicRMWInst::UDecWrap:
20371 case AtomicRMWInst::USubCond:
20372 case AtomicRMWInst::USubSat: {
20373 if (Op == AtomicRMWInst::USubCond && !Subtarget->hasCondSubInsts())
20374 return AtomicExpansionKind::CmpXChg;
20375 if (Op == AtomicRMWInst::USubSat && !Subtarget->hasSubClampInsts())
20376 return AtomicExpansionKind::CmpXChg;
20377 if (Op == AtomicRMWInst::USubCond || Op == AtomicRMWInst::USubSat) {
20378 auto *IT = dyn_cast<IntegerType>(Val: RMW->getType());
20379 if (!IT || IT->getBitWidth() != 32)
20380 return AtomicExpansionKind::CmpXChg;
20381 }
20382
20383 if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
20384 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
20385 if (Subtarget->hasEmulatedSystemScopeAtomics())
20386 return atomicSupportedIfLegalIntType(RMW);
20387
20388 // On most subtargets, for atomicrmw operations other than add/xchg,
20389 // whether or not the instructions will behave correctly depends on where
20390 // the address physically resides and what interconnect is used in the
20391 // system configuration. On some some targets the instruction will nop,
20392 // and in others synchronization will only occur at degraded device scope.
20393 //
20394 // If the allocation is known local to the device, the instructions should
20395 // work correctly.
20396 if (RMW->hasMetadata(Kind: "amdgpu.no.remote.memory"))
20397 return atomicSupportedIfLegalIntType(RMW);
20398
20399 // If fine-grained remote memory works at device scope, we don't need to
20400 // do anything.
20401 if (!HasSystemScope &&
20402 Subtarget->hasAgentScopeFineGrainedRemoteMemoryAtomics())
20403 return atomicSupportedIfLegalIntType(RMW);
20404
20405 // If we are targeting a remote allocated address, it depends what kind of
20406 // allocation the address belongs to.
20407 //
20408 // If the allocation is fine-grained (in host memory, or in PCIe peer
20409 // device memory), the operation will fail depending on the target.
20410 //
20411 // Note fine-grained host memory access does work on APUs or if XGMI is
20412 // used, but we do not know if we are targeting an APU or the system
20413 // configuration from the ISA version/target-cpu.
20414 if (RMW->hasMetadata(Kind: "amdgpu.no.fine.grained.memory"))
20415 return atomicSupportedIfLegalIntType(RMW);
20416
20417 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
20418 Op == AtomicRMWInst::Xor) {
20419 // Atomic sub/or/xor do not work over PCI express, but atomic add
20420 // does. InstCombine transforms these with 0 to or, so undo that.
20421 if (const Constant *ConstVal = dyn_cast<Constant>(Val: RMW->getValOperand());
20422 ConstVal && ConstVal->isNullValue())
20423 return AtomicExpansionKind::CustomExpand;
20424 }
20425
20426 // If the allocation could be in remote, fine-grained memory, the rmw
20427 // instructions may fail. cmpxchg should work, so emit that. On some
20428 // system configurations, PCIe atomics aren't supported so cmpxchg won't
20429 // even work, so you're out of luck anyway.
20430
20431 // In summary:
20432 //
20433 // Cases that may fail:
20434 // - fine-grained pinned host memory
20435 // - fine-grained migratable host memory
20436 // - fine-grained PCIe peer device
20437 //
20438 // Cases that should work, but may be treated overly conservatively.
20439 // - fine-grained host memory on an APU
20440 // - fine-grained XGMI peer device
20441 return AtomicExpansionKind::CmpXChg;
20442 }
20443
20444 return atomicSupportedIfLegalIntType(RMW);
20445 }
20446 case AtomicRMWInst::FAdd: {
20447 Type *Ty = RMW->getType();
20448
20449 // TODO: Handle REGION_ADDRESS
20450 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
20451 // DS F32 FP atomics do respect the denormal mode, but the rounding mode
20452 // is fixed to round-to-nearest-even.
20453 //
20454 // F64 / PK_F16 / PK_BF16 never flush and are also fixed to
20455 // round-to-nearest-even.
20456 //
20457 // We ignore the rounding mode problem, even in strictfp. The C++ standard
20458 // suggests it is OK if the floating-point mode may not match the calling
20459 // thread.
20460 if (Ty->isFloatTy()) {
20461 return Subtarget->hasLDSFPAtomicAddF32() ? AtomicExpansionKind::None
20462 : AtomicExpansionKind::CmpXChg;
20463 }
20464
20465 if (Ty->isDoubleTy()) {
20466 // Ignores denormal mode, but we don't consider flushing mandatory.
20467 return Subtarget->hasLDSFPAtomicAddF64() ? AtomicExpansionKind::None
20468 : AtomicExpansionKind::CmpXChg;
20469 }
20470
20471 if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
20472 return AtomicExpansionKind::None;
20473
20474 return AtomicExpansionKind::CmpXChg;
20475 }
20476
20477 // LDS atomics respect the denormal mode from the mode register.
20478 //
20479 // Traditionally f32 global/buffer memory atomics would unconditionally
20480 // flush denormals, but newer targets do not flush. f64/f16/bf16 cases never
20481 // flush.
20482 //
20483 // On targets with flat atomic fadd, denormals would flush depending on
20484 // whether the target address resides in LDS or global memory. We consider
20485 // this flat-maybe-flush as will-flush.
20486 if (Ty->isFloatTy() &&
20487 !Subtarget->hasMemoryAtomicFaddF32DenormalSupport() &&
20488 !atomicIgnoresDenormalModeOrFPModeIsFTZ(RMW))
20489 return AtomicExpansionKind::CmpXChg;
20490
20491 // FIXME: These ReportUnsafeHWInsts are imprecise. Some of these cases are
20492 // safe. The message phrasing also should be better.
20493 if (globalMemoryFPAtomicIsLegal(Subtarget: *Subtarget, RMW, HasSystemScope)) {
20494 if (AS == AMDGPUAS::FLAT_ADDRESS) {
20495 // gfx942, gfx12
20496 if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
20497 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20498 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
20499 // gfx90a, gfx942, gfx12
20500 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
20501 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20502
20503 // gfx942, gfx12
20504 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
20505 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20506 } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
20507 // gfx90a, gfx942, gfx12
20508 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
20509 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20510
20511 // While gfx90a/gfx942 supports v2bf16 for global/flat, it does not for
20512 // buffer. gfx12 does have the buffer version.
20513 if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
20514 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20515 }
20516
20517 // global and flat atomic fadd f64: gfx90a, gfx942.
20518 if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy())
20519 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20520
20521 if (AS != AMDGPUAS::FLAT_ADDRESS) {
20522 if (Ty->isFloatTy()) {
20523 // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx942,
20524 // gfx11+.
20525 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
20526 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20527 // global/buffer atomic fadd f32 rtn: gfx90a, gfx942, gfx11+.
20528 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
20529 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20530 } else {
20531 // gfx908
20532 if (RMW->use_empty() &&
20533 Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
20534 isV2F16(Ty))
20535 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20536 }
20537 }
20538
20539 // flat atomic fadd f32: gfx942, gfx11+.
20540 if (AS == AMDGPUAS::FLAT_ADDRESS && Ty->isFloatTy()) {
20541 if (Subtarget->hasFlatAtomicFaddF32Inst())
20542 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20543
20544 // If it is in flat address space, and the type is float, we will try to
20545 // expand it, if the target supports global and lds atomic fadd. The
20546 // reason we need that is, in the expansion, we emit the check of
20547 // address space. If it is in global address space, we emit the global
20548 // atomic fadd; if it is in shared address space, we emit the LDS atomic
20549 // fadd.
20550 if (Subtarget->hasLDSFPAtomicAddF32()) {
20551 if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts())
20552 return AtomicExpansionKind::CustomExpand;
20553 if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts())
20554 return AtomicExpansionKind::CustomExpand;
20555 }
20556 }
20557 }
20558
20559 return AtomicExpansionKind::CmpXChg;
20560 }
20561 case AtomicRMWInst::FMin:
20562 case AtomicRMWInst::FMax: {
20563 Type *Ty = RMW->getType();
20564
20565 // LDS float and double fmin/fmax were always supported.
20566 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
20567 return Ty->isFloatTy() || Ty->isDoubleTy() ? AtomicExpansionKind::None
20568 : AtomicExpansionKind::CmpXChg;
20569 }
20570
20571 if (globalMemoryFPAtomicIsLegal(Subtarget: *Subtarget, RMW, HasSystemScope)) {
20572 // For flat and global cases:
20573 // float, double in gfx7. Manual claims denormal support.
20574 // Removed in gfx8.
20575 // float, double restored in gfx10.
20576 // double removed again in gfx11, so only f32 for gfx11/gfx12.
20577 //
20578 // For gfx9, gfx90a and gfx942 support f64 for global (same as fadd), but
20579 // no f32.
20580 if (AS == AMDGPUAS::FLAT_ADDRESS) {
20581 if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy())
20582 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20583 if (Subtarget->hasAtomicFMinFMaxF64FlatInsts() && Ty->isDoubleTy())
20584 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20585 } else if (AMDGPU::isExtendedGlobalAddrSpace(AS) ||
20586 AS == AMDGPUAS::BUFFER_FAT_POINTER) {
20587 if (Subtarget->hasAtomicFMinFMaxF32GlobalInsts() && Ty->isFloatTy())
20588 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20589 if (Subtarget->hasAtomicFMinFMaxF64GlobalInsts() && Ty->isDoubleTy())
20590 return ReportUnsafeHWInst(AtomicExpansionKind::None);
20591 }
20592 }
20593
20594 return AtomicExpansionKind::CmpXChg;
20595 }
20596 case AtomicRMWInst::Nand:
20597 case AtomicRMWInst::FSub:
20598 default:
20599 return AtomicExpansionKind::CmpXChg;
20600 }
20601
20602 llvm_unreachable("covered atomicrmw op switch");
20603}
20604
20605TargetLowering::AtomicExpansionKind
20606SITargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
20607 return LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
20608 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
20609 : AtomicExpansionKind::None;
20610}
20611
20612TargetLowering::AtomicExpansionKind
20613SITargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
20614 return SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS
20615 ? getPrivateAtomicExpansionKind(STI: *getSubtarget())
20616 : AtomicExpansionKind::None;
20617}
20618
20619TargetLowering::AtomicExpansionKind
20620SITargetLowering::shouldExpandAtomicCmpXchgInIR(
20621 const AtomicCmpXchgInst *CmpX) const {
20622 unsigned AddrSpace = CmpX->getPointerAddressSpace();
20623 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
20624 return getPrivateAtomicExpansionKind(STI: *getSubtarget());
20625
20626 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS || !flatInstrMayAccessPrivate(I: CmpX))
20627 return AtomicExpansionKind::None;
20628
20629 const DataLayout &DL = CmpX->getDataLayout();
20630
20631 Type *ValTy = CmpX->getNewValOperand()->getType();
20632
20633 // If a 64-bit flat atomic may alias private, we need to avoid using the
20634 // atomic in the private case.
20635 return DL.getTypeSizeInBits(Ty: ValTy) == 64 ? AtomicExpansionKind::CustomExpand
20636 : AtomicExpansionKind::None;
20637}
20638
20639const TargetRegisterClass *
20640SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
20641 const TargetRegisterClass *RC = TargetLoweringBase::getRegClassFor(VT, isDivergent: false);
20642 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
20643 if (RC == &AMDGPU::VReg_1RegClass && !isDivergent)
20644 return Subtarget->isWave64() ? &AMDGPU::SReg_64RegClass
20645 : &AMDGPU::SReg_32RegClass;
20646 if (!TRI->isSGPRClass(RC) && !isDivergent)
20647 return TRI->getEquivalentSGPRClass(VRC: RC);
20648 if (TRI->isSGPRClass(RC) && isDivergent) {
20649 if (Subtarget->hasGFX90AInsts())
20650 return TRI->getEquivalentAVClass(SRC: RC);
20651 return TRI->getEquivalentVGPRClass(SRC: RC);
20652 }
20653
20654 return RC;
20655}
20656
20657// FIXME: This is a workaround for DivergenceAnalysis not understanding always
20658// uniform values (as produced by the mask results of control flow intrinsics)
20659// used outside of divergent blocks. The phi users need to also be treated as
20660// always uniform.
20661//
20662// FIXME: DA is no longer in-use. Does this still apply to UniformityAnalysis?
20663static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited,
20664 unsigned WaveSize) {
20665 // FIXME: We assume we never cast the mask results of a control flow
20666 // intrinsic.
20667 // Early exit if the type won't be consistent as a compile time hack.
20668 IntegerType *IT = dyn_cast<IntegerType>(Val: V->getType());
20669 if (!IT || IT->getBitWidth() != WaveSize)
20670 return false;
20671
20672 if (!isa<Instruction>(Val: V))
20673 return false;
20674 if (!Visited.insert(Ptr: V).second)
20675 return false;
20676 bool Result = false;
20677 for (const auto *U : V->users()) {
20678 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: U)) {
20679 if (V == U->getOperand(i: 1)) {
20680 switch (Intrinsic->getIntrinsicID()) {
20681 default:
20682 Result = false;
20683 break;
20684 case Intrinsic::amdgcn_if_break:
20685 case Intrinsic::amdgcn_if:
20686 case Intrinsic::amdgcn_else:
20687 Result = true;
20688 break;
20689 }
20690 }
20691 if (V == U->getOperand(i: 0)) {
20692 switch (Intrinsic->getIntrinsicID()) {
20693 default:
20694 Result = false;
20695 break;
20696 case Intrinsic::amdgcn_end_cf:
20697 case Intrinsic::amdgcn_loop:
20698 Result = true;
20699 break;
20700 }
20701 }
20702 } else {
20703 Result = hasCFUser(V: U, Visited, WaveSize);
20704 }
20705 if (Result)
20706 break;
20707 }
20708 return Result;
20709}
20710
20711bool SITargetLowering::requiresUniformRegister(MachineFunction &MF,
20712 const Value *V) const {
20713 if (const CallInst *CI = dyn_cast<CallInst>(Val: V)) {
20714 if (CI->isInlineAsm()) {
20715 // FIXME: This cannot give a correct answer. This should only trigger in
20716 // the case where inline asm returns mixed SGPR and VGPR results, used
20717 // outside the defining block. We don't have a specific result to
20718 // consider, so this assumes if any value is SGPR, the overall register
20719 // also needs to be SGPR.
20720 const SIRegisterInfo *SIRI = Subtarget->getRegisterInfo();
20721 TargetLowering::AsmOperandInfoVector TargetConstraints = ParseConstraints(
20722 DL: MF.getDataLayout(), TRI: Subtarget->getRegisterInfo(), Call: *CI);
20723 for (auto &TC : TargetConstraints) {
20724 if (TC.Type == InlineAsm::isOutput) {
20725 ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
20726 const TargetRegisterClass *RC =
20727 getRegForInlineAsmConstraint(TRI_: SIRI, Constraint: TC.ConstraintCode,
20728 VT: TC.ConstraintVT)
20729 .second;
20730 if (RC && SIRI->isSGPRClass(RC))
20731 return true;
20732 }
20733 }
20734 }
20735 }
20736 SmallPtrSet<const Value *, 16> Visited;
20737 return hasCFUser(V, Visited, WaveSize: Subtarget->getWavefrontSize());
20738}
20739
20740bool SITargetLowering::hasMemSDNodeUser(SDNode *N) const {
20741 for (SDUse &Use : N->uses()) {
20742 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: Use.getUser())) {
20743 if (getBasePtrIndex(N: M) == Use.getOperandNo())
20744 return true;
20745 }
20746 }
20747 return false;
20748}
20749
20750bool SITargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
20751 SDValue N1) const {
20752 if (!N0.hasOneUse())
20753 return false;
20754 // Take care of the opportunity to keep N0 uniform
20755 if (N0->isDivergent() || !N1->isDivergent())
20756 return true;
20757 // Check if we have a good chance to form the memory access pattern with the
20758 // base and offset
20759 return (DAG.isBaseWithConstantOffset(Op: N0) &&
20760 hasMemSDNodeUser(N: *N0->user_begin()));
20761}
20762
20763bool SITargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
20764 Register N0, Register N1) const {
20765 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
20766}
20767
20768MachineMemOperand::Flags
20769SITargetLowering::getTargetMMOFlags(const Instruction &I) const {
20770 // Propagate metadata set by AMDGPUAnnotateUniformValues to the MMO of a load.
20771 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
20772 if (I.getMetadata(Kind: "amdgpu.noclobber"))
20773 Flags |= MONoClobber;
20774 if (I.getMetadata(Kind: "amdgpu.last.use"))
20775 Flags |= MOLastUse;
20776 return Flags;
20777}
20778
20779void SITargetLowering::emitExpandAtomicAddrSpacePredicate(
20780 Instruction *AI) const {
20781 // Given: atomicrmw fadd ptr %addr, float %val ordering
20782 //
20783 // With this expansion we produce the following code:
20784 // [...]
20785 // %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %addr)
20786 // br i1 %is.shared, label %atomicrmw.shared, label %atomicrmw.check.private
20787 //
20788 // atomicrmw.shared:
20789 // %cast.shared = addrspacecast ptr %addr to ptr addrspace(3)
20790 // %loaded.shared = atomicrmw fadd ptr addrspace(3) %cast.shared,
20791 // float %val ordering
20792 // br label %atomicrmw.phi
20793 //
20794 // atomicrmw.check.private:
20795 // %is.private = call i1 @llvm.amdgcn.is.private(ptr %int8ptr)
20796 // br i1 %is.private, label %atomicrmw.private, label %atomicrmw.global
20797 //
20798 // atomicrmw.private:
20799 // %cast.private = addrspacecast ptr %addr to ptr addrspace(5)
20800 // %loaded.private = load float, ptr addrspace(5) %cast.private
20801 // %val.new = fadd float %loaded.private, %val
20802 // store float %val.new, ptr addrspace(5) %cast.private
20803 // br label %atomicrmw.phi
20804 //
20805 // atomicrmw.global:
20806 // %cast.global = addrspacecast ptr %addr to ptr addrspace(1)
20807 // %loaded.global = atomicrmw fadd ptr addrspace(1) %cast.global,
20808 // float %val ordering
20809 // br label %atomicrmw.phi
20810 //
20811 // atomicrmw.phi:
20812 // %loaded.phi = phi float [ %loaded.shared, %atomicrmw.shared ],
20813 // [ %loaded.private, %atomicrmw.private ],
20814 // [ %loaded.global, %atomicrmw.global ]
20815 // br label %atomicrmw.end
20816 //
20817 // atomicrmw.end:
20818 // [...]
20819 //
20820 //
20821 // For 64-bit atomics which may reside in private memory, we perform a simpler
20822 // version that only inserts the private check, and uses the flat operation.
20823
20824 IRBuilder<> Builder(AI);
20825 LLVMContext &Ctx = Builder.getContext();
20826
20827 auto *RMW = dyn_cast<AtomicRMWInst>(Val: AI);
20828 const unsigned PtrOpIdx = RMW ? AtomicRMWInst::getPointerOperandIndex()
20829 : AtomicCmpXchgInst::getPointerOperandIndex();
20830 Value *Addr = AI->getOperand(i: PtrOpIdx);
20831
20832 /// TODO: Only need to check private, then emit flat-known-not private (no
20833 /// need for shared block, or cast to global).
20834 AtomicCmpXchgInst *CX = dyn_cast<AtomicCmpXchgInst>(Val: AI);
20835
20836 Align Alignment;
20837 if (RMW)
20838 Alignment = RMW->getAlign();
20839 else if (CX)
20840 Alignment = CX->getAlign();
20841 else
20842 llvm_unreachable("unhandled atomic operation");
20843
20844 // FullFlatEmulation is true if we need to issue the private, shared, and
20845 // global cases.
20846 //
20847 // If this is false, we are only dealing with the flat-targeting-private case,
20848 // where we only insert a check for private and still use the flat instruction
20849 // for global and shared.
20850
20851 bool FullFlatEmulation =
20852 RMW && RMW->getOperation() == AtomicRMWInst::FAdd &&
20853 ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) ||
20854 (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() &&
20855 RMW->getType()->isDoubleTy()));
20856
20857 // If the return value isn't used, do not introduce a false use in the phi.
20858 bool ReturnValueIsUsed = !AI->use_empty();
20859
20860 BasicBlock *BB = Builder.GetInsertBlock();
20861 Function *F = BB->getParent();
20862 BasicBlock *ExitBB =
20863 BB->splitBasicBlock(I: Builder.GetInsertPoint(), BBName: "atomicrmw.end");
20864 BasicBlock *SharedBB = nullptr;
20865
20866 BasicBlock *CheckPrivateBB = BB;
20867 if (FullFlatEmulation) {
20868 SharedBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.shared", Parent: F, InsertBefore: ExitBB);
20869 CheckPrivateBB =
20870 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.check.private", Parent: F, InsertBefore: ExitBB);
20871 }
20872
20873 BasicBlock *PrivateBB =
20874 BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.private", Parent: F, InsertBefore: ExitBB);
20875 BasicBlock *GlobalBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.global", Parent: F, InsertBefore: ExitBB);
20876 BasicBlock *PhiBB = BasicBlock::Create(Context&: Ctx, Name: "atomicrmw.phi", Parent: F, InsertBefore: ExitBB);
20877
20878 std::prev(x: BB->end())->eraseFromParent();
20879 Builder.SetInsertPoint(BB);
20880
20881 Value *LoadedShared = nullptr;
20882 if (FullFlatEmulation) {
20883 Value *IsShared = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_shared,
20884 Args: {Addr}, FMFSource: nullptr, Name: "is.shared");
20885 Builder.CreateCondBr(Cond: IsShared, True: SharedBB, False: CheckPrivateBB);
20886 Builder.SetInsertPoint(SharedBB);
20887 Value *CastToLocal = Builder.CreateAddrSpaceCast(
20888 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS));
20889
20890 Instruction *Clone = AI->clone();
20891 Clone->insertInto(ParentBB: SharedBB, It: SharedBB->end());
20892 Clone->getOperandUse(i: PtrOpIdx).set(CastToLocal);
20893 LoadedShared = Clone;
20894
20895 Builder.CreateBr(Dest: PhiBB);
20896 Builder.SetInsertPoint(CheckPrivateBB);
20897 }
20898
20899 Value *IsPrivate = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_is_private,
20900 Args: {Addr}, FMFSource: nullptr, Name: "is.private");
20901 Builder.CreateCondBr(Cond: IsPrivate, True: PrivateBB, False: GlobalBB);
20902
20903 Builder.SetInsertPoint(PrivateBB);
20904
20905 Value *CastToPrivate = Builder.CreateAddrSpaceCast(
20906 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::PRIVATE_ADDRESS));
20907
20908 Value *LoadedPrivate;
20909 if (RMW) {
20910 LoadedPrivate = Builder.CreateAlignedLoad(
20911 Ty: RMW->getType(), Ptr: CastToPrivate, Align: RMW->getAlign(), Name: "loaded.private");
20912
20913 Value *NewVal = buildAtomicRMWValue(Op: RMW->getOperation(), Builder,
20914 Loaded: LoadedPrivate, Val: RMW->getValOperand());
20915
20916 Builder.CreateAlignedStore(Val: NewVal, Ptr: CastToPrivate, Align: RMW->getAlign());
20917 } else {
20918 auto [ResultLoad, Equal] =
20919 buildCmpXchgValue(Builder, Ptr: CastToPrivate, Cmp: CX->getCompareOperand(),
20920 Val: CX->getNewValOperand(), Alignment: CX->getAlign());
20921
20922 Value *Insert = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CX->getType()),
20923 Val: ResultLoad, Idxs: 0);
20924 LoadedPrivate = Builder.CreateInsertValue(Agg: Insert, Val: Equal, Idxs: 1);
20925 }
20926
20927 Builder.CreateBr(Dest: PhiBB);
20928
20929 Builder.SetInsertPoint(GlobalBB);
20930
20931 // Continue using a flat instruction if we only emitted the check for private.
20932 Instruction *LoadedGlobal = AI;
20933 if (FullFlatEmulation) {
20934 Value *CastToGlobal = Builder.CreateAddrSpaceCast(
20935 V: Addr, DestTy: PointerType::get(C&: Ctx, AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
20936 AI->getOperandUse(i: PtrOpIdx).set(CastToGlobal);
20937 }
20938
20939 AI->removeFromParent();
20940 AI->insertInto(ParentBB: GlobalBB, It: GlobalBB->end());
20941
20942 // The new atomicrmw may go through another round of legalization later.
20943 if (!FullFlatEmulation) {
20944 // We inserted the runtime check already, make sure we do not try to
20945 // re-expand this.
20946 // TODO: Should union with any existing metadata.
20947 MDBuilder MDB(F->getContext());
20948 MDNode *RangeNotPrivate =
20949 MDB.createRange(Lo: APInt(32, AMDGPUAS::PRIVATE_ADDRESS),
20950 Hi: APInt(32, AMDGPUAS::PRIVATE_ADDRESS + 1));
20951 LoadedGlobal->setMetadata(KindID: LLVMContext::MD_noalias_addrspace,
20952 Node: RangeNotPrivate);
20953 }
20954
20955 Builder.CreateBr(Dest: PhiBB);
20956
20957 Builder.SetInsertPoint(PhiBB);
20958
20959 if (ReturnValueIsUsed) {
20960 PHINode *Loaded = Builder.CreatePHI(Ty: AI->getType(), NumReservedValues: 3);
20961 AI->replaceAllUsesWith(V: Loaded);
20962 if (FullFlatEmulation)
20963 Loaded->addIncoming(V: LoadedShared, BB: SharedBB);
20964 Loaded->addIncoming(V: LoadedPrivate, BB: PrivateBB);
20965 Loaded->addIncoming(V: LoadedGlobal, BB: GlobalBB);
20966 Loaded->takeName(V: AI);
20967 }
20968
20969 Builder.CreateBr(Dest: ExitBB);
20970}
20971
20972static void convertScratchAtomicToFlatAtomic(Instruction *I,
20973 unsigned PtrOpIdx) {
20974 Value *PtrOp = I->getOperand(i: PtrOpIdx);
20975 assert(PtrOp->getType()->getPointerAddressSpace() ==
20976 AMDGPUAS::PRIVATE_ADDRESS);
20977
20978 Type *FlatPtr = PointerType::get(C&: I->getContext(), AddressSpace: AMDGPUAS::FLAT_ADDRESS);
20979 Value *ASCast = CastInst::CreatePointerCast(S: PtrOp, Ty: FlatPtr, Name: "scratch.ascast",
20980 InsertBefore: I->getIterator());
20981 I->setOperand(i: PtrOpIdx, Val: ASCast);
20982}
20983
20984void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
20985 AtomicRMWInst::BinOp Op = AI->getOperation();
20986
20987 if (AI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
20988 return convertScratchAtomicToFlatAtomic(I: AI, PtrOpIdx: AI->getPointerOperandIndex());
20989
20990 if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
20991 Op == AtomicRMWInst::Xor) {
20992 if (const auto *ConstVal = dyn_cast<Constant>(Val: AI->getValOperand());
20993 ConstVal && ConstVal->isNullValue()) {
20994 // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
20995 AI->setOperation(AtomicRMWInst::Add);
20996
20997 // We may still need the private-alias-flat handling below.
20998
20999 // TODO: Skip this for cases where we cannot access remote memory.
21000 }
21001 }
21002
21003 // The non-flat expansions should only perform the de-canonicalization of
21004 // identity values.
21005 if (AI->getPointerAddressSpace() != AMDGPUAS::FLAT_ADDRESS)
21006 return;
21007
21008 emitExpandAtomicAddrSpacePredicate(AI);
21009}
21010
21011void SITargetLowering::emitExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) const {
21012 if (CI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21013 return convertScratchAtomicToFlatAtomic(I: CI, PtrOpIdx: CI->getPointerOperandIndex());
21014
21015 emitExpandAtomicAddrSpacePredicate(AI: CI);
21016}
21017
21018void SITargetLowering::emitExpandAtomicLoad(LoadInst *LI) const {
21019 if (LI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21020 return convertScratchAtomicToFlatAtomic(I: LI, PtrOpIdx: LI->getPointerOperandIndex());
21021
21022 llvm_unreachable(
21023 "Expand Atomic Load only handles SCRATCH -> FLAT conversion");
21024}
21025
21026void SITargetLowering::emitExpandAtomicStore(StoreInst *SI) const {
21027 if (SI->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS)
21028 return convertScratchAtomicToFlatAtomic(I: SI, PtrOpIdx: SI->getPointerOperandIndex());
21029
21030 llvm_unreachable(
21031 "Expand Atomic Store only handles SCRATCH -> FLAT conversion");
21032}
21033
21034LoadInst *
21035SITargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
21036 IRBuilder<> Builder(AI);
21037 auto Order = AI->getOrdering();
21038
21039 // The optimization removes store aspect of the atomicrmw. Therefore, cache
21040 // must be flushed if the atomic ordering had a release semantics. This is
21041 // not necessary a fence, a release fence just coincides to do that flush.
21042 // Avoid replacing of an atomicrmw with a release semantics.
21043 if (isReleaseOrStronger(AO: Order))
21044 return nullptr;
21045
21046 LoadInst *LI = Builder.CreateAlignedLoad(
21047 Ty: AI->getType(), Ptr: AI->getPointerOperand(), Align: AI->getAlign());
21048 LI->setAtomic(Ordering: Order, SSID: AI->getSyncScopeID());
21049 LI->copyMetadata(SrcInst: *AI);
21050 LI->takeName(V: AI);
21051 AI->replaceAllUsesWith(V: LI);
21052 AI->eraseFromParent();
21053 return LI;
21054}
21055