1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This is the parent TargetLowering class for hardware code gen
11/// targets.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUISelLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUMachineFunctionInfo.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUSelectionDAGInfo.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/Analysis.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Support/KnownBits.h"
29#include "llvm/Target/TargetMachine.h"
30
31using namespace llvm;
32
33#define GET_CALLING_CONV_IMPL
34#include "AMDGPUGenCallingConv.inc"
35
36static cl::opt<bool> AMDGPUBypassSlowDiv(
37 "amdgpu-bypass-slow-div",
38 cl::desc("Skip 64-bit divide for dynamic 32-bit values"),
39 cl::init(Val: true));
40
41// Find a larger type to do a load / store of a vector with.
42EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
43 unsigned StoreSize = VT.getStoreSizeInBits();
44 if (StoreSize <= 32)
45 return EVT::getIntegerVT(Context&: Ctx, BitWidth: StoreSize);
46
47 if (StoreSize % 32 == 0)
48 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i32, NumElements: StoreSize / 32);
49
50 return VT;
51}
52
53unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
54 return DAG.computeKnownBits(Op).countMaxActiveBits();
55}
56
57unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
58 // In order for this to be a signed 24-bit value, bit 23, must
59 // be a sign bit.
60 return DAG.ComputeMaxSignificantBits(Op);
61}
62
63AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
64 const TargetSubtargetInfo &STI,
65 const AMDGPUSubtarget &AMDGPUSTI)
66 : TargetLowering(TM, STI), Subtarget(&AMDGPUSTI) {
67 // Always lower memset, memcpy, and memmove intrinsics to load/store
68 // instructions, rather then generating calls to memset, mempcy or memmove.
69 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U;
70 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U;
71 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U;
72
73 // Enable ganging up loads and stores in the memcpy DAG lowering.
74 MaxGluedStoresPerMemcpy = 16;
75
76 // Lower floating point store/load to integer store/load to reduce the number
77 // of patterns in tablegen.
78 setOperationAction(Op: ISD::LOAD, VT: MVT::f32, Action: Promote);
79 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
80
81 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Promote);
82 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
83
84 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f32, Action: Promote);
85 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
86
87 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f32, Action: Promote);
88 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
89
90 setOperationAction(Op: ISD::LOAD, VT: MVT::v5f32, Action: Promote);
91 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
92
93 setOperationAction(Op: ISD::LOAD, VT: MVT::v6f32, Action: Promote);
94 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
95
96 setOperationAction(Op: ISD::LOAD, VT: MVT::v7f32, Action: Promote);
97 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
98
99 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f32, Action: Promote);
100 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
101
102 setOperationAction(Op: ISD::LOAD, VT: MVT::v9f32, Action: Promote);
103 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
104
105 setOperationAction(Op: ISD::LOAD, VT: MVT::v10f32, Action: Promote);
106 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
107
108 setOperationAction(Op: ISD::LOAD, VT: MVT::v11f32, Action: Promote);
109 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
110
111 setOperationAction(Op: ISD::LOAD, VT: MVT::v12f32, Action: Promote);
112 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
113
114 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f32, Action: Promote);
115 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
116
117 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f32, Action: Promote);
118 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
119
120 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Promote);
121 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i64, DestVT: MVT::v2i32);
122
123 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
124 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
125
126 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Promote);
127 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f64, DestVT: MVT::v2i32);
128
129 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Promote);
130 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
131
132 setOperationAction(Op: ISD::LOAD, VT: MVT::v3i64, Action: Promote);
133 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
134
135 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i64, Action: Promote);
136 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
137
138 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f64, Action: Promote);
139 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
140
141 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f64, Action: Promote);
142 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
143
144 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i64, Action: Promote);
145 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
146
147 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f64, Action: Promote);
148 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
149
150 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i64, Action: Promote);
151 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
152
153 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f64, Action: Promote);
154 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
155
156 setOperationAction(Op: ISD::LOAD, VT: MVT::i128, Action: Promote);
157 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i128, DestVT: MVT::v4i32);
158
159 // TODO: Would be better to consume as directly legal
160 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f32, Action: Promote);
161 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
162
163 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f64, Action: Promote);
164 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f64, DestVT: MVT::i64);
165
166 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f16, Action: Promote);
167 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
168
169 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::bf16, Action: Promote);
170 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
171
172 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2f32, Action: Promote);
173 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2f32, DestVT: MVT::i64);
174
175 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f32, Action: Promote);
176 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
177
178 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f64, Action: Promote);
179 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f64, DestVT: MVT::i64);
180
181 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f16, Action: Promote);
182 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
183
184 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::bf16, Action: Promote);
185 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
186
187 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2f32, Action: Promote);
188 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2f32, DestVT: MVT::i64);
189
190 // There are no 64-bit extloads. These should be done as a 32-bit extload and
191 // an extension to 64-bit.
192 for (MVT VT : MVT::integer_valuetypes())
193 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64, MemVT: VT,
194 Action: Expand);
195
196 for (MVT VT : MVT::integer_valuetypes()) {
197 if (VT == MVT::i64)
198 continue;
199
200 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) {
201 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i1, Action: Promote);
202 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i8, Action: Legal);
203 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i16, Action: Legal);
204 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i32, Action: Expand);
205 }
206 }
207
208 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
209 for (auto MemVT :
210 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
211 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT,
212 Action: Expand);
213
214 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
215 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
216 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
218 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
220 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
221 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
222 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
223 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
224 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
225 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
226 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
227 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
228
229 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
230 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
231 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
232 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
233 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
234 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
235
236 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
237 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
238 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
239 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
240 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
241 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
242 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
243 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
244 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
245 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
246 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
247 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
248
249 setOperationAction(Op: ISD::STORE, VT: MVT::f32, Action: Promote);
250 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
251
252 setOperationAction(Op: ISD::STORE, VT: MVT::v2f32, Action: Promote);
253 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
254
255 setOperationAction(Op: ISD::STORE, VT: MVT::v3f32, Action: Promote);
256 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
257
258 setOperationAction(Op: ISD::STORE, VT: MVT::v4f32, Action: Promote);
259 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
260
261 setOperationAction(Op: ISD::STORE, VT: MVT::v5f32, Action: Promote);
262 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
263
264 setOperationAction(Op: ISD::STORE, VT: MVT::v6f32, Action: Promote);
265 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
266
267 setOperationAction(Op: ISD::STORE, VT: MVT::v7f32, Action: Promote);
268 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
269
270 setOperationAction(Op: ISD::STORE, VT: MVT::v8f32, Action: Promote);
271 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
272
273 setOperationAction(Op: ISD::STORE, VT: MVT::v9f32, Action: Promote);
274 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
275
276 setOperationAction(Op: ISD::STORE, VT: MVT::v10f32, Action: Promote);
277 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
278
279 setOperationAction(Op: ISD::STORE, VT: MVT::v11f32, Action: Promote);
280 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
281
282 setOperationAction(Op: ISD::STORE, VT: MVT::v12f32, Action: Promote);
283 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
284
285 setOperationAction(Op: ISD::STORE, VT: MVT::v16f32, Action: Promote);
286 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
287
288 setOperationAction(Op: ISD::STORE, VT: MVT::v32f32, Action: Promote);
289 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
290
291 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Promote);
292 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i64, DestVT: MVT::v2i32);
293
294 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
295 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
296
297 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Promote);
298 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f64, DestVT: MVT::v2i32);
299
300 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Promote);
301 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
302
303 setOperationAction(Op: ISD::STORE, VT: MVT::v3i64, Action: Promote);
304 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
305
306 setOperationAction(Op: ISD::STORE, VT: MVT::v3f64, Action: Promote);
307 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
308
309 setOperationAction(Op: ISD::STORE, VT: MVT::v4i64, Action: Promote);
310 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
311
312 setOperationAction(Op: ISD::STORE, VT: MVT::v4f64, Action: Promote);
313 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
314
315 setOperationAction(Op: ISD::STORE, VT: MVT::v8i64, Action: Promote);
316 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
317
318 setOperationAction(Op: ISD::STORE, VT: MVT::v8f64, Action: Promote);
319 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
320
321 setOperationAction(Op: ISD::STORE, VT: MVT::v16i64, Action: Promote);
322 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
323
324 setOperationAction(Op: ISD::STORE, VT: MVT::v16f64, Action: Promote);
325 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
326
327 setOperationAction(Op: ISD::STORE, VT: MVT::i128, Action: Promote);
328 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i128, DestVT: MVT::v4i32);
329
330 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i1, Action: Expand);
331 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i8, Action: Expand);
332 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
333 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i32, Action: Expand);
334
335 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i1, Action: Expand);
336 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i8, Action: Expand);
337 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i16, Action: Expand);
338 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i32, Action: Expand);
339
340 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
341 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
342 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
343 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
344 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
345 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
346 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
347 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
348 setTruncStoreAction(ValVT: MVT::v6f32, MemVT: MVT::v6f16, Action: Expand);
349 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
350 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
351 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
352 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
353 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
354 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
355
356 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
357 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
358 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
359
360 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
361 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
362 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
363
364 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i8, Action: Expand);
365
366 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
367 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
368 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i8, Action: Expand);
369 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i1, Action: Expand);
370 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
371 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
372 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
373
374 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i32, Action: Expand);
375 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i16, Action: Expand);
376 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
377 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
378 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
379
380 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i1, Action: Expand);
381 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i8, Action: Expand);
382 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i16, Action: Expand);
383
384 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i1, Action: Expand);
385 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i8, Action: Expand);
386 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i16, Action: Expand);
387
388 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i1, Action: Expand);
389 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i8, Action: Expand);
390 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i16, Action: Expand);
391
392 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
393 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
394 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
395
396 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
397 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
398 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
399 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i16, Action: Expand);
400 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
401 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
402 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i1, Action: Expand);
403
404 setOperationAction(Ops: ISD::Constant, VTs: {MVT::i32, MVT::i64}, Action: Legal);
405 setOperationAction(Ops: ISD::ConstantFP, VTs: {MVT::f32, MVT::f64}, Action: Legal);
406
407 setOperationAction(Ops: {ISD::BR_JT, ISD::BRIND}, VT: MVT::Other, Action: Expand);
408
409 // For R600, this is totally unsupported, just custom lower to produce an
410 // error.
411 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
412
413 // Library functions. These default to Expand, but we have instructions
414 // for them.
415 setOperationAction(Ops: {ISD::FCEIL, ISD::FPOW, ISD::FABS, ISD::FFLOOR,
416 ISD::FROUNDEVEN, ISD::FTRUNC},
417 VTs: {MVT::f16, MVT::f32}, Action: Legal);
418 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM}, VT: MVT::f32, Action: Legal);
419
420 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Custom);
421 setOperationAction(Ops: ISD::FROUND, VTs: {MVT::f32, MVT::f64}, Action: Custom);
422 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND},
423 VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
424
425 setOperationAction(
426 Ops: {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, VT: MVT::f32,
427 Action: Custom);
428 setOperationAction(Ops: {ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, VT: MVT::f64, Action: Custom);
429
430 setOperationAction(Ops: ISD::FNEARBYINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
431
432 setOperationAction(Ops: ISD::FRINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
433
434 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VTs: {MVT::f16, MVT::f32, MVT::f64},
435 Action: Expand);
436
437 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
438 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
439 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Custom);
440
441 setOperationAction(Ops: {ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, VT: MVT::f16,
442 Action: Custom);
443
444 setOperationAction(Ops: ISD::FCANONICALIZE, VTs: {MVT::f32, MVT::f64}, Action: Legal);
445
446 // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
447 // scalarization code. Can be removed when IS_FPCLASS expand isn't called by
448 // default unless marked custom/legal.
449 setOperationAction(Ops: ISD::IS_FPCLASS,
450 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
451 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
452 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64,
453 MVT::v16f64},
454 Action: Custom);
455
456 // Expand to fneg + fadd.
457 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
458
459 setOperationAction(Ops: ISD::CONCAT_VECTORS,
460 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
461 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
462 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
463 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
464 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
465 Action: Custom);
466
467 setOperationAction(
468 Ops: ISD::EXTRACT_SUBVECTOR,
469 VTs: {MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32,
470 MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32,
471 MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32,
472 MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32,
473 MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
474 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
475 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64},
476 Action: Custom);
477
478 setOperationAction(Ops: {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP}, VT: MVT::f64,
479 Action: Expand);
480 setOperationAction(Ops: ISD::FP_TO_FP16, VTs: {MVT::f64, MVT::f32}, Action: Custom);
481
482 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
483 for (MVT VT : ScalarIntVTs) {
484 // These should use [SU]DIVREM, so set them to expand
485 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT,
486 Action: Expand);
487
488 // GPU does not have divrem function for signed or unsigned.
489 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT, Action: Custom);
490
491 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
492 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
493
494 setOperationAction(Ops: {ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Action: Expand);
495
496 setOperationAction(Ops: {ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT,
497 Action: Expand);
498 }
499
500 // The hardware supports 32-bit FSHR, but not FSHL.
501 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
502
503 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VTs: {MVT::i32, MVT::i64}, Action: Expand);
504
505 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT: MVT::i16, Action: Expand);
506
507 setOperationAction(Ops: {ISD::MUL, ISD::MULHU, ISD::MULHS}, VT: MVT::i64, Action: Expand);
508 setOperationAction(Ops: {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT,
509 ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
510 ISD::FP_TO_UINT_SAT},
511 VT: MVT::i64, Action: Custom);
512 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: Expand);
513
514 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i32,
515 Action: Legal);
516
517 setOperationAction(
518 Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON},
519 VT: MVT::i64, Action: Custom);
520
521 for (auto VT : {MVT::i8, MVT::i16})
522 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT, Action: Custom);
523
524 static const MVT::SimpleValueType VectorIntTypes[] = {
525 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
526 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
527
528 for (MVT VT : VectorIntTypes) {
529 // Expand the following operations for the current type by default.
530 // clang-format off
531 setOperationAction(Ops: {ISD::ADD, ISD::AND,
532 ISD::FP_TO_SINT, ISD::FP_TO_UINT,
533 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
534 ISD::MUL, ISD::MULHU,
535 ISD::MULHS, ISD::OR,
536 ISD::SHL, ISD::SRA,
537 ISD::SRL, ISD::ROTL,
538 ISD::ROTR, ISD::SUB,
539 ISD::SINT_TO_FP, ISD::UINT_TO_FP,
540 ISD::SDIV, ISD::UDIV,
541 ISD::SREM, ISD::UREM,
542 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
543 ISD::SDIVREM, ISD::UDIVREM,
544 ISD::SELECT, ISD::VSELECT,
545 ISD::SELECT_CC, ISD::XOR,
546 ISD::BSWAP, ISD::CTPOP,
547 ISD::CTTZ, ISD::CTLZ,
548 ISD::VECTOR_SHUFFLE, ISD::SETCC,
549 ISD::ADDRSPACECAST},
550 VT, Action: Expand);
551 // clang-format on
552 }
553
554 static const MVT::SimpleValueType FloatVectorTypes[] = {
555 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
556 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
557
558 for (MVT VT : FloatVectorTypes) {
559 setOperationAction(
560 Ops: {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM,
561 ISD::FADD, ISD::FCEIL, ISD::FCOS,
562 ISD::FDIV, ISD::FEXP2, ISD::FEXP,
563 ISD::FEXP10, ISD::FLOG2, ISD::FREM,
564 ISD::FLOG, ISD::FLOG10, ISD::FPOW,
565 ISD::FFLOOR, ISD::FTRUNC, ISD::FMUL,
566 ISD::FMA, ISD::FRINT, ISD::FNEARBYINT,
567 ISD::FSQRT, ISD::FSIN, ISD::FSUB,
568 ISD::FNEG, ISD::VSELECT, ISD::SELECT_CC,
569 ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, ISD::SETCC,
570 ISD::FCANONICALIZE, ISD::FROUNDEVEN},
571 VT, Action: Expand);
572 }
573
574 // This causes using an unrolled select operation rather than expansion with
575 // bit operations. This is in general better, but the alternative using BFI
576 // instructions may be better if the select sources are SGPRs.
577 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f32, Action: Promote);
578 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
579
580 setOperationAction(Op: ISD::SELECT, VT: MVT::v3f32, Action: Promote);
581 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
582
583 setOperationAction(Op: ISD::SELECT, VT: MVT::v4f32, Action: Promote);
584 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
585
586 setOperationAction(Op: ISD::SELECT, VT: MVT::v5f32, Action: Promote);
587 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
588
589 setOperationAction(Op: ISD::SELECT, VT: MVT::v6f32, Action: Promote);
590 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
591
592 setOperationAction(Op: ISD::SELECT, VT: MVT::v7f32, Action: Promote);
593 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
594
595 setOperationAction(Op: ISD::SELECT, VT: MVT::v9f32, Action: Promote);
596 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
597
598 setOperationAction(Op: ISD::SELECT, VT: MVT::v10f32, Action: Promote);
599 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
600
601 setOperationAction(Op: ISD::SELECT, VT: MVT::v11f32, Action: Promote);
602 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
603
604 setOperationAction(Op: ISD::SELECT, VT: MVT::v12f32, Action: Promote);
605 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
606
607 setSchedulingPreference(Sched::RegPressure);
608 setJumpIsExpensive(true);
609
610 setMinCmpXchgSizeInBits(32);
611 setSupportsUnalignedAtomics(false);
612
613 PredictableSelectIsExpensive = false;
614
615 // We want to find all load dependencies for long chains of stores to enable
616 // merging into very wide vectors. The problem is with vectors with > 4
617 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
618 // vectors are a legal type, even though we have to split the loads
619 // usually. When we can more precisely specify load legality per address
620 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
621 // smarter so that they can figure out what to do in 2 iterations without all
622 // N > 4 stores on the same chain.
623 GatherAllAliasesMaxDepth = 16;
624
625 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
626 // about these during lowering.
627 MaxStoresPerMemcpy = 0xffffffff;
628 MaxStoresPerMemmove = 0xffffffff;
629 MaxStoresPerMemset = 0xffffffff;
630
631 // The expansion for 64-bit division is enormous.
632 if (AMDGPUBypassSlowDiv)
633 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
634
635 setTargetDAGCombine({ISD::BITCAST, ISD::SHL,
636 ISD::SRA, ISD::SRL,
637 ISD::TRUNCATE, ISD::MUL,
638 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
639 ISD::MULHU, ISD::MULHS,
640 ISD::SELECT, ISD::SELECT_CC,
641 ISD::STORE, ISD::FADD,
642 ISD::FSUB, ISD::FNEG,
643 ISD::FABS, ISD::AssertZext,
644 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN});
645
646 setMaxAtomicSizeInBitsSupported(64);
647 setMaxDivRemBitWidthSupported(64);
648 setMaxLargeFPConvertBitWidthSupported(64);
649}
650
651//===----------------------------------------------------------------------===//
652// Target Information
653//===----------------------------------------------------------------------===//
654
655LLVM_READNONE
656static bool fnegFoldsIntoOpcode(unsigned Opc) {
657 switch (Opc) {
658 case ISD::FADD:
659 case ISD::FSUB:
660 case ISD::FMUL:
661 case ISD::FMA:
662 case ISD::FMAD:
663 case ISD::FMINNUM:
664 case ISD::FMAXNUM:
665 case ISD::FMINNUM_IEEE:
666 case ISD::FMAXNUM_IEEE:
667 case ISD::FMINIMUM:
668 case ISD::FMAXIMUM:
669 case ISD::FMINIMUMNUM:
670 case ISD::FMAXIMUMNUM:
671 case ISD::SELECT:
672 case ISD::FSIN:
673 case ISD::FTRUNC:
674 case ISD::FRINT:
675 case ISD::FNEARBYINT:
676 case ISD::FROUNDEVEN:
677 case ISD::FCANONICALIZE:
678 case AMDGPUISD::RCP:
679 case AMDGPUISD::RCP_LEGACY:
680 case AMDGPUISD::RCP_IFLAG:
681 case AMDGPUISD::SIN_HW:
682 case AMDGPUISD::FMUL_LEGACY:
683 case AMDGPUISD::FMIN_LEGACY:
684 case AMDGPUISD::FMAX_LEGACY:
685 case AMDGPUISD::FMED3:
686 // TODO: handle llvm.amdgcn.fma.legacy
687 return true;
688 case ISD::BITCAST:
689 llvm_unreachable("bitcast is special cased");
690 default:
691 return false;
692 }
693}
694
695static bool fnegFoldsIntoOp(const SDNode *N) {
696 unsigned Opc = N->getOpcode();
697 if (Opc == ISD::BITCAST) {
698 // TODO: Is there a benefit to checking the conditions performFNegCombine
699 // does? We don't for the other cases.
700 SDValue BCSrc = N->getOperand(Num: 0);
701 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
702 return BCSrc.getNumOperands() == 2 &&
703 BCSrc.getOperand(i: 1).getValueSizeInBits() == 32;
704 }
705
706 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32;
707 }
708
709 return fnegFoldsIntoOpcode(Opc);
710}
711
712/// \p returns true if the operation will definitely need to use a 64-bit
713/// encoding, and thus will use a VOP3 encoding regardless of the source
714/// modifiers.
715LLVM_READONLY
716static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
717 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) ||
718 VT == MVT::f64;
719}
720
721/// Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the
722/// type for ISD::SELECT.
723LLVM_READONLY
724static bool selectSupportsSourceMods(const SDNode *N) {
725 // TODO: Only applies if select will be vector
726 return N->getValueType(ResNo: 0) == MVT::f32;
727}
728
729// Most FP instructions support source modifiers, but this could be refined
730// slightly.
731LLVM_READONLY
732static bool hasSourceMods(const SDNode *N) {
733 if (isa<MemSDNode>(Val: N))
734 return false;
735
736 switch (N->getOpcode()) {
737 case ISD::CopyToReg:
738 case ISD::FDIV:
739 case ISD::FREM:
740 case ISD::INLINEASM:
741 case ISD::INLINEASM_BR:
742 case AMDGPUISD::DIV_SCALE:
743 case ISD::INTRINSIC_W_CHAIN:
744
745 // TODO: Should really be looking at the users of the bitcast. These are
746 // problematic because bitcasts are used to legalize all stores to integer
747 // types.
748 case ISD::BITCAST:
749 return false;
750 case ISD::INTRINSIC_WO_CHAIN: {
751 switch (N->getConstantOperandVal(Num: 0)) {
752 case Intrinsic::amdgcn_interp_p1:
753 case Intrinsic::amdgcn_interp_p2:
754 case Intrinsic::amdgcn_interp_mov:
755 case Intrinsic::amdgcn_interp_p1_f16:
756 case Intrinsic::amdgcn_interp_p2_f16:
757 return false;
758 default:
759 return true;
760 }
761 }
762 case ISD::SELECT:
763 return selectSupportsSourceMods(N);
764 default:
765 return true;
766 }
767}
768
769bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N,
770 unsigned CostThreshold) {
771 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
772 // it is truly free to use a source modifier in all cases. If there are
773 // multiple users but for each one will necessitate using VOP3, there will be
774 // a code size increase. Try to avoid increasing code size unless we know it
775 // will save on the instruction count.
776 unsigned NumMayIncreaseSize = 0;
777 MVT VT = N->getValueType(ResNo: 0).getScalarType().getSimpleVT();
778
779 assert(!N->use_empty());
780
781 // XXX - Should this limit number of uses to check?
782 for (const SDNode *U : N->users()) {
783 if (!hasSourceMods(N: U))
784 return false;
785
786 if (!opMustUseVOP3Encoding(N: U, VT)) {
787 if (++NumMayIncreaseSize > CostThreshold)
788 return false;
789 }
790 }
791
792 return true;
793}
794
795EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
796 ISD::NodeType ExtendKind) const {
797 assert(!VT.isVector() && "only scalar expected");
798
799 // Round to the next multiple of 32-bits.
800 unsigned Size = VT.getSizeInBits();
801 if (Size <= 32)
802 return MVT::i32;
803 return EVT::getIntegerVT(Context, BitWidth: 32 * ((Size + 31) / 32));
804}
805
806unsigned AMDGPUTargetLowering::getVectorIdxWidth(const DataLayout &) const {
807 return 32;
808}
809
810bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
811 return true;
812}
813
814// The backend supports 32 and 64 bit floating point immediates.
815// FIXME: Why are we reporting vectors of FP immediates as legal?
816bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
817 bool ForCodeSize) const {
818 return isTypeLegal(VT: VT.getScalarType());
819}
820
821// We don't want to shrink f64 / f32 constants.
822bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
823 EVT ScalarVT = VT.getScalarType();
824 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
825}
826
827bool AMDGPUTargetLowering::shouldReduceLoadWidth(
828 SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
829 std::optional<unsigned> ByteOffset) const {
830 // TODO: This may be worth removing. Check regression tests for diffs.
831 if (!TargetLoweringBase::shouldReduceLoadWidth(Load: N, ExtTy, NewVT, ByteOffset))
832 return false;
833
834 unsigned NewSize = NewVT.getStoreSizeInBits();
835
836 // If we are reducing to a 32-bit load or a smaller multi-dword load,
837 // this is always better.
838 if (NewSize >= 32)
839 return true;
840
841 EVT OldVT = N->getValueType(ResNo: 0);
842 unsigned OldSize = OldVT.getStoreSizeInBits();
843
844 MemSDNode *MN = cast<MemSDNode>(Val: N);
845 unsigned AS = MN->getAddressSpace();
846 // Do not shrink an aligned scalar load to sub-dword.
847 // Scalar engine cannot do sub-dword loads.
848 // TODO: Update this for GFX12 which does have scalar sub-dword loads.
849 if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) &&
850 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
851 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
852 (isa<LoadSDNode>(Val: N) && AS == AMDGPUAS::GLOBAL_ADDRESS &&
853 MN->isInvariant())) &&
854 AMDGPU::isUniformMMO(MMO: MN->getMemOperand()))
855 return false;
856
857 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
858 // extloads, so doing one requires using a buffer_load. In cases where we
859 // still couldn't use a scalar load, using the wider load shouldn't really
860 // hurt anything.
861
862 // If the old size already had to be an extload, there's no harm in continuing
863 // to reduce the width.
864 return (OldSize < 32);
865}
866
867bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
868 const SelectionDAG &DAG,
869 const MachineMemOperand &MMO) const {
870
871 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
872
873 if (LoadTy.getScalarType() == MVT::i32)
874 return false;
875
876 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
877 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
878
879 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
880 return false;
881
882 unsigned Fast = 0;
883 return allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
884 VT: CastTy, MMO, Fast: &Fast) &&
885 Fast;
886}
887
888// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
889// profitable with the expansion for 64-bit since it's generally good to
890// speculate things.
891bool AMDGPUTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
892 return true;
893}
894
895bool AMDGPUTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
896 return true;
897}
898
899bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const {
900 switch (N->getOpcode()) {
901 case ISD::EntryToken:
902 case ISD::TokenFactor:
903 return true;
904 case ISD::INTRINSIC_WO_CHAIN: {
905 unsigned IntrID = N->getConstantOperandVal(Num: 0);
906 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
907 }
908 case ISD::INTRINSIC_W_CHAIN: {
909 unsigned IntrID = N->getConstantOperandVal(Num: 1);
910 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
911 }
912 case ISD::LOAD:
913 if (cast<LoadSDNode>(Val: N)->getMemOperand()->getAddrSpace() ==
914 AMDGPUAS::CONSTANT_ADDRESS_32BIT)
915 return true;
916 return false;
917 case AMDGPUISD::SETCC: // ballot-style instruction
918 return true;
919 }
920 return false;
921}
922
923SDValue AMDGPUTargetLowering::getNegatedExpression(
924 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
925 NegatibleCost &Cost, unsigned Depth) const {
926
927 switch (Op.getOpcode()) {
928 case ISD::FMA:
929 case ISD::FMAD: {
930 // Negating a fma is not free if it has users without source mods.
931 if (!allUsesHaveSourceMods(N: Op.getNode()))
932 return SDValue();
933 break;
934 }
935 case AMDGPUISD::RCP: {
936 SDValue Src = Op.getOperand(i: 0);
937 EVT VT = Op.getValueType();
938 SDLoc SL(Op);
939
940 SDValue NegSrc = getNegatedExpression(Op: Src, DAG, LegalOperations,
941 ForCodeSize, Cost, Depth: Depth + 1);
942 if (NegSrc)
943 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: NegSrc, Flags: Op->getFlags());
944 return SDValue();
945 }
946 default:
947 break;
948 }
949
950 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps: LegalOperations,
951 OptForSize: ForCodeSize, Cost, Depth);
952}
953
954//===---------------------------------------------------------------------===//
955// Target Properties
956//===---------------------------------------------------------------------===//
957
958bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
959 assert(VT.isFloatingPoint());
960
961 // Packed operations do not have a fabs modifier.
962 // Report this based on the end legalized type.
963 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
964}
965
966bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
967 assert(VT.isFloatingPoint());
968 // Report this based on the end legalized type.
969 VT = VT.getScalarType();
970 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
971}
972
973bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
974 unsigned NumElem,
975 unsigned AS) const {
976 return true;
977}
978
979bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
980 // There are few operations which truly have vector input operands. Any vector
981 // operation is going to involve operations on each component, and a
982 // build_vector will be a copy per element, so it always makes sense to use a
983 // build_vector input in place of the extracted element to avoid a copy into a
984 // super register.
985 //
986 // We should probably only do this if all users are extracts only, but this
987 // should be the common case.
988 return true;
989}
990
991bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
992 // Truncate is just accessing a subregister.
993
994 unsigned SrcSize = Source.getSizeInBits();
995 unsigned DestSize = Dest.getSizeInBits();
996
997 return DestSize < SrcSize && DestSize % 32 == 0 ;
998}
999
1000bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
1001 // Truncate is just accessing a subregister.
1002
1003 unsigned SrcSize = Source->getScalarSizeInBits();
1004 unsigned DestSize = Dest->getScalarSizeInBits();
1005
1006 if (DestSize== 16 && Subtarget->has16BitInsts())
1007 return SrcSize >= 32;
1008
1009 return DestSize < SrcSize && DestSize % 32 == 0;
1010}
1011
1012bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
1013 unsigned SrcSize = Src->getScalarSizeInBits();
1014 unsigned DestSize = Dest->getScalarSizeInBits();
1015
1016 if (SrcSize == 16 && Subtarget->has16BitInsts())
1017 return DestSize >= 32;
1018
1019 return SrcSize == 32 && DestSize == 64;
1020}
1021
1022bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
1023 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
1024 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
1025 // this will enable reducing 64-bit operations the 32-bit, which is always
1026 // good.
1027
1028 if (Src == MVT::i16)
1029 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1030
1031 return Src == MVT::i32 && Dest == MVT::i64;
1032}
1033
1034bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
1035 EVT DestVT) const {
1036 switch (N->getOpcode()) {
1037 case ISD::ABS:
1038 case ISD::ADD:
1039 case ISD::SUB:
1040 case ISD::SHL:
1041 case ISD::SRL:
1042 case ISD::SRA:
1043 case ISD::AND:
1044 case ISD::OR:
1045 case ISD::XOR:
1046 case ISD::MUL:
1047 case ISD::SETCC:
1048 case ISD::SELECT:
1049 case ISD::SMIN:
1050 case ISD::SMAX:
1051 case ISD::UMIN:
1052 case ISD::UMAX:
1053 case ISD::USUBSAT:
1054 if (isTypeLegal(VT: MVT::i16) &&
1055 (!DestVT.isVector() ||
1056 !isOperationLegal(Op: ISD::ADD, VT: MVT::v2i16))) { // Check if VOP3P
1057 // Don't narrow back down to i16 if promoted to i32 already.
1058 if (!N->isDivergent() && DestVT.isInteger() &&
1059 DestVT.getScalarSizeInBits() > 1 &&
1060 DestVT.getScalarSizeInBits() <= 16 &&
1061 SrcVT.getScalarSizeInBits() > 16) {
1062 return false;
1063 }
1064 }
1065 return true;
1066 default:
1067 break;
1068 }
1069
1070 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
1071 // limited number of native 64-bit operations. Shrinking an operation to fit
1072 // in a single 32-bit register should always be helpful. As currently used,
1073 // this is much less general than the name suggests, and is only used in
1074 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
1075 // not profitable, and may actually be harmful.
1076 if (isa<LoadSDNode>(Val: N))
1077 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
1078
1079 return true;
1080}
1081
1082bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
1083 const SDNode* N, CombineLevel Level) const {
1084 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
1085 N->getOpcode() == ISD::SRL) &&
1086 "Expected shift op");
1087
1088 SDValue ShiftLHS = N->getOperand(Num: 0);
1089 if (!ShiftLHS->hasOneUse())
1090 return false;
1091
1092 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
1093 !ShiftLHS.getOperand(i: 0)->hasOneUse())
1094 return false;
1095
1096 // Always commute pre-type legalization and right shifts.
1097 // We're looking for shl(or(x,y),z) patterns.
1098 if (Level < CombineLevel::AfterLegalizeTypes ||
1099 N->getOpcode() != ISD::SHL || N->getOperand(Num: 0).getOpcode() != ISD::OR)
1100 return true;
1101
1102 // If only user is a i32 right-shift, then don't destroy a BFE pattern.
1103 if (N->getValueType(ResNo: 0) == MVT::i32 && N->hasOneUse() &&
1104 (N->user_begin()->getOpcode() == ISD::SRA ||
1105 N->user_begin()->getOpcode() == ISD::SRL))
1106 return false;
1107
1108 // Don't destroy or(shl(load_zext(),c), load_zext()) patterns.
1109 auto IsShiftAndLoad = [](SDValue LHS, SDValue RHS) {
1110 if (LHS.getOpcode() != ISD::SHL)
1111 return false;
1112 auto *RHSLd = dyn_cast<LoadSDNode>(Val&: RHS);
1113 auto *LHS0 = dyn_cast<LoadSDNode>(Val: LHS.getOperand(i: 0));
1114 auto *LHS1 = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
1115 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD &&
1116 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1117 RHSLd->getExtensionType() == ISD::ZEXTLOAD;
1118 };
1119 SDValue LHS = N->getOperand(Num: 0).getOperand(i: 0);
1120 SDValue RHS = N->getOperand(Num: 0).getOperand(i: 1);
1121 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS));
1122}
1123
1124//===---------------------------------------------------------------------===//
1125// TargetLowering Callbacks
1126//===---------------------------------------------------------------------===//
1127
1128CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
1129 bool IsVarArg) {
1130 switch (CC) {
1131 case CallingConv::AMDGPU_VS:
1132 case CallingConv::AMDGPU_GS:
1133 case CallingConv::AMDGPU_PS:
1134 case CallingConv::AMDGPU_CS:
1135 case CallingConv::AMDGPU_HS:
1136 case CallingConv::AMDGPU_ES:
1137 case CallingConv::AMDGPU_LS:
1138 return CC_AMDGPU;
1139 case CallingConv::AMDGPU_CS_Chain:
1140 case CallingConv::AMDGPU_CS_ChainPreserve:
1141 return CC_AMDGPU_CS_CHAIN;
1142 case CallingConv::C:
1143 case CallingConv::Fast:
1144 case CallingConv::Cold:
1145 return CC_AMDGPU_Func;
1146 case CallingConv::AMDGPU_Gfx:
1147 case CallingConv::AMDGPU_Gfx_WholeWave:
1148 return CC_SI_Gfx;
1149 case CallingConv::AMDGPU_KERNEL:
1150 case CallingConv::SPIR_KERNEL:
1151 default:
1152 reportFatalUsageError(reason: "unsupported calling convention for call");
1153 }
1154}
1155
1156CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
1157 bool IsVarArg) {
1158 switch (CC) {
1159 case CallingConv::AMDGPU_KERNEL:
1160 case CallingConv::SPIR_KERNEL:
1161 llvm_unreachable("kernels should not be handled here");
1162 case CallingConv::AMDGPU_VS:
1163 case CallingConv::AMDGPU_GS:
1164 case CallingConv::AMDGPU_PS:
1165 case CallingConv::AMDGPU_CS:
1166 case CallingConv::AMDGPU_CS_Chain:
1167 case CallingConv::AMDGPU_CS_ChainPreserve:
1168 case CallingConv::AMDGPU_HS:
1169 case CallingConv::AMDGPU_ES:
1170 case CallingConv::AMDGPU_LS:
1171 return RetCC_SI_Shader;
1172 case CallingConv::AMDGPU_Gfx:
1173 case CallingConv::AMDGPU_Gfx_WholeWave:
1174 return RetCC_SI_Gfx;
1175 case CallingConv::C:
1176 case CallingConv::Fast:
1177 case CallingConv::Cold:
1178 return RetCC_AMDGPU_Func;
1179 default:
1180 reportFatalUsageError(reason: "unsupported calling convention");
1181 }
1182}
1183
1184/// The SelectionDAGBuilder will automatically promote function arguments
1185/// with illegal types. However, this does not work for the AMDGPU targets
1186/// since the function arguments are stored in memory as these illegal types.
1187/// In order to handle this properly we need to get the original types sizes
1188/// from the LLVM IR Function and fixup the ISD:InputArg values before
1189/// passing them to AnalyzeFormalArguments()
1190
1191/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
1192/// input values across multiple registers. Each item in the Ins array
1193/// represents a single value that will be stored in registers. Ins[x].VT is
1194/// the value type of the value that will be stored in the register, so
1195/// whatever SDNode we lower the argument to needs to be this type.
1196///
1197/// In order to correctly lower the arguments we need to know the size of each
1198/// argument. Since Ins[x].VT gives us the size of the register that will
1199/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
1200/// for the original function argument so that we can deduce the correct memory
1201/// type to use for Ins[x]. In most cases the correct memory type will be
1202/// Ins[x].ArgVT. However, this will not always be the case. If, for example,
1203/// we have a kernel argument of type v8i8, this argument will be split into
1204/// 8 parts and each part will be represented by its own item in the Ins array.
1205/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
1206/// the argument before it was split. From this, we deduce that the memory type
1207/// for each individual part is i8. We pass the memory type as LocVT to the
1208/// calling convention analysis function and the register type (Ins[x].VT) as
1209/// the ValVT.
1210void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
1211 CCState &State,
1212 const SmallVectorImpl<ISD::InputArg> &Ins) const {
1213 const MachineFunction &MF = State.getMachineFunction();
1214 const Function &Fn = MF.getFunction();
1215 LLVMContext &Ctx = Fn.getContext();
1216 const unsigned ExplicitOffset = Subtarget->getExplicitKernelArgOffset();
1217 CallingConv::ID CC = Fn.getCallingConv();
1218
1219 Align MaxAlign = Align(1);
1220 uint64_t ExplicitArgOffset = 0;
1221 const DataLayout &DL = Fn.getDataLayout();
1222
1223 unsigned InIndex = 0;
1224
1225 for (const Argument &Arg : Fn.args()) {
1226 const bool IsByRef = Arg.hasByRefAttr();
1227 Type *BaseArgTy = Arg.getType();
1228 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1229 Align Alignment = DL.getValueOrABITypeAlignment(
1230 Alignment: IsByRef ? Arg.getParamAlign() : std::nullopt, Ty: MemArgTy);
1231 MaxAlign = std::max(a: Alignment, b: MaxAlign);
1232 uint64_t AllocSize = DL.getTypeAllocSize(Ty: MemArgTy);
1233
1234 uint64_t ArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + ExplicitOffset;
1235 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + AllocSize;
1236
1237 // We're basically throwing away everything passed into us and starting over
1238 // to get accurate in-memory offsets. The "PartOffset" is completely useless
1239 // to us as computed in Ins.
1240 //
1241 // We also need to figure out what type legalization is trying to do to get
1242 // the correct memory offsets.
1243
1244 SmallVector<EVT, 16> ValueVTs;
1245 SmallVector<uint64_t, 16> Offsets;
1246 ComputeValueVTs(TLI: *this, DL, Ty: BaseArgTy, ValueVTs, /*MemVTs=*/nullptr,
1247 FixedOffsets: &Offsets, StartingOffset: ArgOffset);
1248
1249 for (unsigned Value = 0, NumValues = ValueVTs.size();
1250 Value != NumValues; ++Value) {
1251 uint64_t BasePartOffset = Offsets[Value];
1252
1253 EVT ArgVT = ValueVTs[Value];
1254 EVT MemVT = ArgVT;
1255 MVT RegisterVT = getRegisterTypeForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1256 unsigned NumRegs = getNumRegistersForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1257
1258 if (NumRegs == 1) {
1259 // This argument is not split, so the IR type is the memory type.
1260 if (ArgVT.isExtended()) {
1261 // We have an extended type, like i24, so we should just use the
1262 // register type.
1263 MemVT = RegisterVT;
1264 } else {
1265 MemVT = ArgVT;
1266 }
1267 } else if (ArgVT.isVector() && RegisterVT.isVector() &&
1268 ArgVT.getScalarType() == RegisterVT.getScalarType()) {
1269 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
1270 // We have a vector value which has been split into a vector with
1271 // the same scalar type, but fewer elements. This should handle
1272 // all the floating-point vector types.
1273 MemVT = RegisterVT;
1274 } else if (ArgVT.isVector() &&
1275 ArgVT.getVectorNumElements() == NumRegs) {
1276 // This arg has been split so that each element is stored in a separate
1277 // register.
1278 MemVT = ArgVT.getScalarType();
1279 } else if (ArgVT.isExtended()) {
1280 // We have an extended type, like i65.
1281 MemVT = RegisterVT;
1282 } else {
1283 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
1284 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0);
1285 if (RegisterVT.isInteger()) {
1286 MemVT = EVT::getIntegerVT(Context&: State.getContext(), BitWidth: MemoryBits);
1287 } else if (RegisterVT.isVector()) {
1288 assert(!RegisterVT.getScalarType().isFloatingPoint());
1289 unsigned NumElements = RegisterVT.getVectorNumElements();
1290 assert(MemoryBits % NumElements == 0);
1291 // This vector type has been split into another vector type with
1292 // a different elements size.
1293 EVT ScalarVT = EVT::getIntegerVT(Context&: State.getContext(),
1294 BitWidth: MemoryBits / NumElements);
1295 MemVT = EVT::getVectorVT(Context&: State.getContext(), VT: ScalarVT, NumElements);
1296 } else {
1297 llvm_unreachable("cannot deduce memory type.");
1298 }
1299 }
1300
1301 // Convert one element vectors to scalar.
1302 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
1303 MemVT = MemVT.getScalarType();
1304
1305 // Round up vec3/vec5 argument.
1306 if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
1307 MemVT = MemVT.getPow2VectorType(Context&: State.getContext());
1308 } else if (!MemVT.isSimple() && !MemVT.isVector()) {
1309 MemVT = MemVT.getRoundIntegerType(Context&: State.getContext());
1310 }
1311
1312 unsigned PartOffset = 0;
1313 for (unsigned i = 0; i != NumRegs; ++i) {
1314 State.addLoc(V: CCValAssign::getCustomMem(ValNo: InIndex++, ValVT: RegisterVT,
1315 Offset: BasePartOffset + PartOffset,
1316 LocVT: MemVT.getSimpleVT(),
1317 HTP: CCValAssign::Full));
1318 PartOffset += MemVT.getStoreSize();
1319 }
1320 }
1321 }
1322}
1323
1324SDValue AMDGPUTargetLowering::LowerReturn(
1325 SDValue Chain, CallingConv::ID CallConv,
1326 bool isVarArg,
1327 const SmallVectorImpl<ISD::OutputArg> &Outs,
1328 const SmallVectorImpl<SDValue> &OutVals,
1329 const SDLoc &DL, SelectionDAG &DAG) const {
1330 // FIXME: Fails for r600 tests
1331 //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
1332 // "wave terminate should not have return values");
1333 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM, DL, VT: MVT::Other, Operand: Chain);
1334}
1335
1336//===---------------------------------------------------------------------===//
1337// Target specific lowering
1338//===---------------------------------------------------------------------===//
1339
1340/// Selects the correct CCAssignFn for a given CallingConvention value.
1341CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1342 bool IsVarArg) {
1343 return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1344}
1345
1346CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1347 bool IsVarArg) {
1348 return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1349}
1350
1351SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1352 SelectionDAG &DAG,
1353 MachineFrameInfo &MFI,
1354 int ClobberedFI) const {
1355 SmallVector<SDValue, 8> ArgChains;
1356 int64_t FirstByte = MFI.getObjectOffset(ObjectIdx: ClobberedFI);
1357 int64_t LastByte = FirstByte + MFI.getObjectSize(ObjectIdx: ClobberedFI) - 1;
1358
1359 // Include the original chain at the beginning of the list. When this is
1360 // used by target LowerCall hooks, this helps legalize find the
1361 // CALLSEQ_BEGIN node.
1362 ArgChains.push_back(Elt: Chain);
1363
1364 // Add a chain value for each stack argument corresponding
1365 for (SDNode *U : DAG.getEntryNode().getNode()->users()) {
1366 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U)) {
1367 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr())) {
1368 if (FI->getIndex() < 0) {
1369 int64_t InFirstByte = MFI.getObjectOffset(ObjectIdx: FI->getIndex());
1370 int64_t InLastByte = InFirstByte;
1371 InLastByte += MFI.getObjectSize(ObjectIdx: FI->getIndex()) - 1;
1372
1373 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1374 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1375 ArgChains.push_back(Elt: SDValue(L, 1));
1376 }
1377 }
1378 }
1379 }
1380
1381 // Build a tokenfactor for all the chains.
1382 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
1383}
1384
1385SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1386 SmallVectorImpl<SDValue> &InVals,
1387 StringRef Reason) const {
1388 SDValue Callee = CLI.Callee;
1389 SelectionDAG &DAG = CLI.DAG;
1390
1391 const Function &Fn = DAG.getMachineFunction().getFunction();
1392
1393 StringRef FuncName("<unknown>");
1394
1395 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Val&: Callee))
1396 FuncName = G->getSymbol();
1397 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
1398 FuncName = G->getGlobal()->getName();
1399
1400 DAG.getContext()->diagnose(
1401 DI: DiagnosticInfoUnsupported(Fn, Reason + FuncName, CLI.DL.getDebugLoc()));
1402
1403 if (!CLI.IsTailCall) {
1404 for (ISD::InputArg &Arg : CLI.Ins)
1405 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
1406 }
1407
1408 // FIXME: Hack because R600 doesn't handle callseq pseudos yet.
1409 if (getTargetMachine().getTargetTriple().getArch() == Triple::r600)
1410 return CLI.Chain;
1411
1412 SDValue Chain = DAG.getCALLSEQ_START(Chain: CLI.Chain, InSize: 0, OutSize: 0, DL: CLI.DL);
1413 return DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, /*InGlue=*/Glue: SDValue(), DL: CLI.DL);
1414}
1415
1416SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1417 SmallVectorImpl<SDValue> &InVals) const {
1418 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported call to function ");
1419}
1420
1421SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1422 SelectionDAG &DAG) const {
1423 const Function &Fn = DAG.getMachineFunction().getFunction();
1424
1425 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1426 Fn, "unsupported dynamic alloca", SDLoc(Op).getDebugLoc()));
1427 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()), Op.getOperand(i: 0)};
1428 return DAG.getMergeValues(Ops, dl: SDLoc());
1429}
1430
1431SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1432 SelectionDAG &DAG) const {
1433 switch (Op.getOpcode()) {
1434 default:
1435 Op->print(OS&: errs(), G: &DAG);
1436 llvm_unreachable("Custom lowering code for this "
1437 "instruction is not implemented yet!");
1438 break;
1439 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1440 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1441 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1442 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1443 case ISD::SDIVREM:
1444 return LowerSDIVREM(Op, DAG);
1445 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1446 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1447 case ISD::FRINT: return LowerFRINT(Op, DAG);
1448 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1449 case ISD::FROUNDEVEN:
1450 return LowerFROUNDEVEN(Op, DAG);
1451 case ISD::FROUND: return LowerFROUND(Op, DAG);
1452 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1453 case ISD::FLOG2:
1454 return LowerFLOG2(Op, DAG);
1455 case ISD::FLOG:
1456 case ISD::FLOG10:
1457 return LowerFLOGCommon(Op, DAG);
1458 case ISD::FEXP:
1459 case ISD::FEXP10:
1460 return lowerFEXP(Op, DAG);
1461 case ISD::FEXP2:
1462 return lowerFEXP2(Op, DAG);
1463 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1464 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1465 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1466 case ISD::FP_TO_SINT:
1467 case ISD::FP_TO_UINT:
1468 return LowerFP_TO_INT(Op, DAG);
1469 case ISD::FP_TO_SINT_SAT:
1470 case ISD::FP_TO_UINT_SAT:
1471 return LowerFP_TO_INT_SAT(Op, DAG);
1472 case ISD::CTTZ:
1473 case ISD::CTTZ_ZERO_POISON:
1474 case ISD::CTLZ:
1475 case ISD::CTLZ_ZERO_POISON:
1476 return LowerCTLZ_CTTZ(Op, DAG);
1477 case ISD::CTLS:
1478 return LowerCTLS(Op, DAG);
1479 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1480 }
1481 return Op;
1482}
1483
1484void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
1485 SmallVectorImpl<SDValue> &Results,
1486 SelectionDAG &DAG) const {
1487 switch (N->getOpcode()) {
1488 case ISD::SIGN_EXTEND_INREG:
1489 // Different parts of legalization seem to interpret which type of
1490 // sign_extend_inreg is the one to check for custom lowering. The extended
1491 // from type is what really matters, but some places check for custom
1492 // lowering of the result type. This results in trying to use
1493 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1494 // nothing here and let the illegal result integer be handled normally.
1495 return;
1496 case ISD::FLOG2:
1497 if (SDValue Lowered = LowerFLOG2(Op: SDValue(N, 0), DAG))
1498 Results.push_back(Elt: Lowered);
1499 return;
1500 case ISD::FLOG:
1501 case ISD::FLOG10:
1502 if (SDValue Lowered = LowerFLOGCommon(Op: SDValue(N, 0), DAG))
1503 Results.push_back(Elt: Lowered);
1504 return;
1505 case ISD::FEXP2:
1506 if (SDValue Lowered = lowerFEXP2(Op: SDValue(N, 0), DAG))
1507 Results.push_back(Elt: Lowered);
1508 return;
1509 case ISD::FEXP:
1510 case ISD::FEXP10:
1511 if (SDValue Lowered = lowerFEXP(Op: SDValue(N, 0), DAG))
1512 Results.push_back(Elt: Lowered);
1513 return;
1514 case ISD::CTLZ:
1515 case ISD::CTLZ_ZERO_POISON:
1516 if (auto Lowered = lowerCTLZResults(Op: SDValue(N, 0u), DAG))
1517 Results.push_back(Elt: Lowered);
1518 return;
1519 default:
1520 return;
1521 }
1522}
1523
1524SDValue AMDGPUTargetLowering::LowerBlockAddress(SDValue Op,
1525 SelectionDAG &DAG) const {
1526 BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val&: Op);
1527 SDLoc SL(Op);
1528 EVT VT = Op.getValueType();
1529 return DAG.getTargetBlockAddress(BA: BA->getBlockAddress(), VT, Offset: BA->getOffset(),
1530 TargetFlags: BA->getTargetFlags());
1531}
1532
1533SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
1534 SDValue Op,
1535 SelectionDAG &DAG) const {
1536
1537 const DataLayout &DL = DAG.getDataLayout();
1538 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Val&: Op);
1539 const GlobalValue *GV = G->getGlobal();
1540
1541 if (!MFI->isModuleEntryFunction()) {
1542 bool IsNamedBarrier = AMDGPU::isNamedBarrier(GV: *cast<GlobalVariable>(Val: GV));
1543 std::optional<uint32_t> Address =
1544 AMDGPUMachineFunctionInfo::getLDSAbsoluteAddress(GV: *GV);
1545 if (!Address && IsNamedBarrier)
1546 llvm_unreachable("named barrier should have an assigned address");
1547 if (Address) {
1548 if (IsNamedBarrier) {
1549 unsigned BarCnt = cast<GlobalVariable>(Val: GV)->getGlobalSize(DL) / 16;
1550 MFI->recordNumNamedBarriers(GVAddr: Address.value(), BarCnt);
1551 }
1552 // A constant byte offset (e.g. from a GEP into an array of named
1553 // barriers) folds directly into the fixed LDS address.
1554 return DAG.getConstant(Val: *Address + G->getOffset(), DL: SDLoc(Op),
1555 VT: Op.getValueType());
1556 }
1557 }
1558
1559 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1560 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
1561 if (!MFI->isModuleEntryFunction() &&
1562 GV->getName() != "llvm.amdgcn.module.lds" &&
1563 !AMDGPU::isNamedBarrier(GV: *cast<GlobalVariable>(Val: GV))) {
1564 SDLoc DL(Op);
1565 const Function &Fn = DAG.getMachineFunction().getFunction();
1566 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1567 Fn, "local memory global used by non-kernel function",
1568 DL.getDebugLoc(), DS_Warning));
1569
1570 // We currently don't have a way to correctly allocate LDS objects that
1571 // aren't directly associated with a kernel. We do force inlining of
1572 // functions that use local objects. However, if these dead functions are
1573 // not eliminated, we don't want a compile time error. Just emit a warning
1574 // and a trap, since there should be no callable path here.
1575 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL, VT: MVT::Other, Operand: DAG.getEntryNode());
1576 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
1577 N1: Trap, N2: DAG.getRoot());
1578 DAG.setRoot(OutputChain);
1579 return DAG.getPOISON(VT: Op.getValueType());
1580 }
1581
1582 // TODO: We could emit code to handle the initialization somewhere.
1583 // We ignore the initializer for now and legalize it to allow selection.
1584 // The initializer will anyway get errored out during assembly emission.
1585 unsigned Offset = MFI->allocateLDSGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1586 // A constant byte offset (e.g. from a GEP into an array of named barriers)
1587 // folds directly into the allocated LDS address.
1588 return DAG.getConstant(Val: Offset + G->getOffset(), DL: SDLoc(Op),
1589 VT: Op.getValueType());
1590 }
1591 return SDValue();
1592}
1593
1594SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1595 SelectionDAG &DAG) const {
1596 SmallVector<SDValue, 8> Args;
1597 SDLoc SL(Op);
1598
1599 EVT VT = Op.getValueType();
1600 if (VT.getVectorElementType().getSizeInBits() < 32) {
1601 unsigned OpBitSize = Op.getOperand(i: 0).getValueType().getSizeInBits();
1602 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1603 unsigned NewNumElt = OpBitSize / 32;
1604 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1605 : EVT::getVectorVT(Context&: *DAG.getContext(),
1606 VT: MVT::i32, NumElements: NewNumElt);
1607 for (const SDUse &U : Op->ops()) {
1608 SDValue In = U.get();
1609 SDValue NewIn = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewEltVT, Operand: In);
1610 if (NewNumElt > 1)
1611 DAG.ExtractVectorElements(Op: NewIn, Args);
1612 else
1613 Args.push_back(Elt: NewIn);
1614 }
1615
1616 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
1617 NumElements: NewNumElt * Op.getNumOperands());
1618 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1619 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: BV);
1620 }
1621 }
1622
1623 for (const SDUse &U : Op->ops())
1624 DAG.ExtractVectorElements(Op: U.get(), Args);
1625
1626 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1627}
1628
1629SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1630 SelectionDAG &DAG) const {
1631 SDLoc SL(Op);
1632 SmallVector<SDValue, 8> Args;
1633 unsigned Start = Op.getConstantOperandVal(i: 1);
1634 EVT VT = Op.getValueType();
1635 EVT SrcVT = Op.getOperand(i: 0).getValueType();
1636
1637 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
1638 unsigned NumElt = VT.getVectorNumElements();
1639 unsigned NumSrcElt = SrcVT.getVectorNumElements();
1640 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types");
1641
1642 // Extract 32-bit registers at a time.
1643 EVT NewSrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumSrcElt / 2);
1644 EVT NewVT = NumElt == 2
1645 ? MVT::i32
1646 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElt / 2);
1647 SDValue Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewSrcVT, Operand: Op.getOperand(i: 0));
1648
1649 DAG.ExtractVectorElements(Op: Tmp, Args, Start: Start / 2, Count: NumElt / 2);
1650 if (NumElt == 2)
1651 Tmp = Args[0];
1652 else
1653 Tmp = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1654
1655 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Tmp);
1656 }
1657
1658 DAG.ExtractVectorElements(Op: Op.getOperand(i: 0), Args, Start,
1659 Count: VT.getVectorNumElements());
1660
1661 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1662}
1663
1664// TODO: Handle fabs too
1665static SDValue peekFNeg(SDValue Val) {
1666 if (Val.getOpcode() == ISD::FNEG)
1667 return Val.getOperand(i: 0);
1668
1669 return Val;
1670}
1671
1672static SDValue peekFPSignOps(SDValue Val) {
1673 if (Val.getOpcode() == ISD::FNEG)
1674 Val = Val.getOperand(i: 0);
1675 if (Val.getOpcode() == ISD::FABS)
1676 Val = Val.getOperand(i: 0);
1677 if (Val.getOpcode() == ISD::FCOPYSIGN)
1678 Val = Val.getOperand(i: 0);
1679 return Val;
1680}
1681
1682SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl(
1683 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1684 SDValue False, SDValue CC, DAGCombinerInfo &DCI) const {
1685 SelectionDAG &DAG = DCI.DAG;
1686 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
1687 switch (CCOpcode) {
1688 case ISD::SETOEQ:
1689 case ISD::SETONE:
1690 case ISD::SETUNE:
1691 case ISD::SETNE:
1692 case ISD::SETUEQ:
1693 case ISD::SETEQ:
1694 case ISD::SETFALSE:
1695 case ISD::SETFALSE2:
1696 case ISD::SETTRUE:
1697 case ISD::SETTRUE2:
1698 case ISD::SETUO:
1699 case ISD::SETO:
1700 break;
1701 case ISD::SETULE:
1702 case ISD::SETULT: {
1703 if (LHS == True)
1704 return DAG.getNode(Opcode: AMDGPUISD::FMIN_LEGACY, DL, VT, N1: RHS, N2: LHS);
1705 return DAG.getNode(Opcode: AMDGPUISD::FMAX_LEGACY, DL, VT, N1: LHS, N2: RHS);
1706 }
1707 case ISD::SETOLE:
1708 case ISD::SETOLT:
1709 case ISD::SETLE:
1710 case ISD::SETLT: {
1711 // Ordered. Assume ordered for undefined.
1712
1713 // Only do this after legalization to avoid interfering with other combines
1714 // which might occur.
1715 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1716 !DCI.isCalledByLegalizer())
1717 return SDValue();
1718
1719 // We need to permute the operands to get the correct NaN behavior. The
1720 // selected operand is the second one based on the failing compare with NaN,
1721 // so permute it based on the compare type the hardware uses.
1722 if (LHS == True)
1723 return DAG.getNode(Opcode: AMDGPUISD::FMIN_LEGACY, DL, VT, N1: LHS, N2: RHS);
1724 return DAG.getNode(Opcode: AMDGPUISD::FMAX_LEGACY, DL, VT, N1: RHS, N2: LHS);
1725 }
1726 case ISD::SETUGE:
1727 case ISD::SETUGT: {
1728 if (LHS == True)
1729 return DAG.getNode(Opcode: AMDGPUISD::FMAX_LEGACY, DL, VT, N1: RHS, N2: LHS);
1730 return DAG.getNode(Opcode: AMDGPUISD::FMIN_LEGACY, DL, VT, N1: LHS, N2: RHS);
1731 }
1732 case ISD::SETGT:
1733 case ISD::SETGE:
1734 case ISD::SETOGE:
1735 case ISD::SETOGT: {
1736 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1737 !DCI.isCalledByLegalizer())
1738 return SDValue();
1739
1740 if (LHS == True)
1741 return DAG.getNode(Opcode: AMDGPUISD::FMAX_LEGACY, DL, VT, N1: LHS, N2: RHS);
1742 return DAG.getNode(Opcode: AMDGPUISD::FMIN_LEGACY, DL, VT, N1: RHS, N2: LHS);
1743 }
1744 case ISD::SETCC_INVALID:
1745 llvm_unreachable("Invalid setcc condcode!");
1746 }
1747 return SDValue();
1748}
1749
1750/// Generate Min/Max node
1751SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
1752 SDValue LHS, SDValue RHS,
1753 SDValue True, SDValue False,
1754 SDValue CC,
1755 DAGCombinerInfo &DCI) const {
1756 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
1757 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, DCI);
1758
1759 SelectionDAG &DAG = DCI.DAG;
1760
1761 // If we can't directly match this, try to see if we can fold an fneg to
1762 // match.
1763
1764 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
1765 ConstantFPSDNode *CFalse = dyn_cast<ConstantFPSDNode>(Val&: False);
1766 SDValue NegTrue = peekFNeg(Val: True);
1767
1768 // Undo the combine foldFreeOpFromSelect does if it helps us match the
1769 // fmin/fmax.
1770 //
1771 // select (fcmp olt (lhs, K)), (fneg lhs), -K
1772 // -> fneg (fmin_legacy lhs, K)
1773 //
1774 // TODO: Use getNegatedExpression
1775 if (LHS == NegTrue && CFalse && CRHS) {
1776 APFloat NegRHS = neg(X: CRHS->getValueAPF());
1777 if (NegRHS == CFalse->getValueAPF()) {
1778 SDValue Combined =
1779 combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True: NegTrue, False, CC, DCI);
1780 if (Combined)
1781 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Combined);
1782 return SDValue();
1783 }
1784 }
1785
1786 return SDValue();
1787}
1788
1789std::pair<SDValue, SDValue>
1790AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1791 SDLoc SL(Op);
1792
1793 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1794
1795 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1796 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1797
1798 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1799 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1800
1801 return std::pair(Lo, Hi);
1802}
1803
1804SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1805 SDLoc SL(Op);
1806
1807 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1808 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1809 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1810}
1811
1812SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1813 SDLoc SL(Op);
1814
1815 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1816 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1817 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1818}
1819
1820// Split a vector type into two parts. The first part is a power of two vector.
1821// The second part is whatever is left over, and is a scalar if it would
1822// otherwise be a 1-vector.
1823std::pair<EVT, EVT>
1824AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1825 EVT LoVT, HiVT;
1826 EVT EltVT = VT.getVectorElementType();
1827 unsigned NumElts = VT.getVectorNumElements();
1828 unsigned LoNumElts = PowerOf2Ceil(A: (NumElts + 1) / 2);
1829 LoVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: LoNumElts);
1830 HiVT = NumElts - LoNumElts == 1
1831 ? EltVT
1832 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts - LoNumElts);
1833 return std::pair(LoVT, HiVT);
1834}
1835
1836// Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1837// scalar.
1838std::pair<SDValue, SDValue>
1839AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1840 const EVT &LoVT, const EVT &HiVT,
1841 SelectionDAG &DAG) const {
1842 EVT VT = N.getValueType();
1843 assert(LoVT.getVectorNumElements() +
1844 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
1845 VT.getVectorNumElements() &&
1846 "More vector elements requested than available!");
1847 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N,
1848 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1849
1850 unsigned LoNumElts = LoVT.getVectorNumElements();
1851
1852 if (HiVT.isVector()) {
1853 unsigned HiNumElts = HiVT.getVectorNumElements();
1854 if ((VT.getVectorNumElements() % HiNumElts) == 0) {
1855 // Avoid creating an extract_subvector with an index that isn't a multiple
1856 // of the result type.
1857 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
1858 N2: DAG.getConstant(Val: LoNumElts, DL, VT: MVT::i32));
1859 return {Lo, Hi};
1860 }
1861
1862 SmallVector<SDValue, 8> Elts;
1863 DAG.ExtractVectorElements(Op: N, Args&: Elts, /*Start=*/LoNumElts,
1864 /*Count=*/HiNumElts);
1865 SDValue Hi = DAG.getBuildVector(VT: HiVT, DL, Ops: Elts);
1866 return {Lo, Hi};
1867 }
1868
1869 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: HiVT, N1: N,
1870 N2: DAG.getVectorIdxConstant(Val: LoNumElts, DL));
1871 return {Lo, Hi};
1872}
1873
1874SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1875 SelectionDAG &DAG) const {
1876 LoadSDNode *Load = cast<LoadSDNode>(Val: Op);
1877 EVT VT = Op.getValueType();
1878 SDLoc SL(Op);
1879
1880
1881 // If this is a 2 element vector, we really want to scalarize and not create
1882 // weird 1 element vectors.
1883 if (VT.getVectorNumElements() == 2) {
1884 SDValue Ops[2];
1885 std::tie(args&: Ops[0], args&: Ops[1]) = scalarizeVectorLoad(LD: Load, DAG);
1886 return DAG.getMergeValues(Ops, dl: SL);
1887 }
1888
1889 SDValue BasePtr = Load->getBasePtr();
1890 EVT MemVT = Load->getMemoryVT();
1891
1892 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1893
1894 EVT LoVT, HiVT;
1895 EVT LoMemVT, HiMemVT;
1896 SDValue Lo, Hi;
1897
1898 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1899 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1900 std::tie(args&: Lo, args&: Hi) = splitVector(N: Op, DL: SL, LoVT, HiVT, DAG);
1901
1902 unsigned Size = LoMemVT.getStoreSize();
1903 Align BaseAlign = Load->getAlign();
1904 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1905
1906 SDValue LoLoad = DAG.getExtLoad(
1907 ExtType: Load->getExtensionType(), dl: SL, VT: LoVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1908 MemVT: LoMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1909 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Size));
1910 SDValue HiLoad = DAG.getExtLoad(
1911 ExtType: Load->getExtensionType(), dl: SL, VT: HiVT, Chain: Load->getChain(), Ptr: HiPtr,
1912 PtrInfo: SrcValue.getWithOffset(O: LoMemVT.getStoreSize()), MemVT: HiMemVT, Alignment: HiAlign,
1913 MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1914
1915 SDValue Join;
1916 if (LoVT == HiVT) {
1917 // This is the case that the vector is power of two so was evenly split.
1918 Join = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, N1: LoLoad, N2: HiLoad);
1919 } else {
1920 Join = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SL, VT, N1: DAG.getPOISON(VT), N2: LoLoad,
1921 N3: DAG.getVectorIdxConstant(Val: 0, DL: SL));
1922 Join = DAG.getNode(
1923 Opcode: HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, DL: SL,
1924 VT, N1: Join, N2: HiLoad,
1925 N3: DAG.getVectorIdxConstant(Val: LoVT.getVectorNumElements(), DL: SL));
1926 }
1927
1928 SDValue Ops[] = {Join, DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
1929 N1: LoLoad.getValue(R: 1), N2: HiLoad.getValue(R: 1))};
1930
1931 return DAG.getMergeValues(Ops, dl: SL);
1932}
1933
1934SDValue AMDGPUTargetLowering::WidenOrSplitVectorLoad(SDValue Op,
1935 SelectionDAG &DAG) const {
1936 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
1937 EVT VT = Op.getValueType();
1938 SDValue BasePtr = Load->getBasePtr();
1939 EVT MemVT = Load->getMemoryVT();
1940 SDLoc SL(Op);
1941 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1942 Align BaseAlign = Load->getAlign();
1943 unsigned NumElements = MemVT.getVectorNumElements();
1944
1945 // Widen from vec3 to vec4 when the load is at least 8-byte aligned
1946 // or 16-byte fully dereferenceable. Otherwise, split the vector load.
1947 if (NumElements != 3 ||
1948 (BaseAlign < Align(8) &&
1949 !SrcValue.isDereferenceable(Size: 16, C&: *DAG.getContext(), DL: DAG.getDataLayout())))
1950 return SplitVectorLoad(Op, DAG);
1951
1952 assert(NumElements == 3);
1953
1954 EVT WideVT =
1955 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
1956 EVT WideMemVT =
1957 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(), NumElements: 4);
1958 SDValue WideLoad = DAG.getExtLoad(
1959 ExtType: Load->getExtensionType(), dl: SL, VT: WideVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1960 MemVT: WideMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags());
1961 return DAG.getMergeValues(
1962 Ops: {DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT, N1: WideLoad,
1963 N2: DAG.getVectorIdxConstant(Val: 0, DL: SL)),
1964 WideLoad.getValue(R: 1)},
1965 dl: SL);
1966}
1967
1968SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1969 SelectionDAG &DAG) const {
1970 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
1971 SDValue Val = Store->getValue();
1972 EVT VT = Val.getValueType();
1973
1974 // If this is a 2 element vector, we really want to scalarize and not create
1975 // weird 1 element vectors.
1976 if (VT.getVectorNumElements() == 2)
1977 return scalarizeVectorStore(ST: Store, DAG);
1978
1979 EVT MemVT = Store->getMemoryVT();
1980 SDValue Chain = Store->getChain();
1981 SDValue BasePtr = Store->getBasePtr();
1982 SDLoc SL(Op);
1983
1984 EVT LoVT, HiVT;
1985 EVT LoMemVT, HiMemVT;
1986 SDValue Lo, Hi;
1987
1988 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1989 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1990 std::tie(args&: Lo, args&: Hi) = splitVector(N: Val, DL: SL, LoVT, HiVT, DAG);
1991
1992 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: LoMemVT.getStoreSize());
1993
1994 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
1995 Align BaseAlign = Store->getAlign();
1996 unsigned Size = LoMemVT.getStoreSize();
1997 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1998
1999 SDValue LoStore =
2000 DAG.getTruncStore(Chain, dl: SL, Val: Lo, Ptr: BasePtr, PtrInfo: SrcValue, SVT: LoMemVT, Alignment: BaseAlign,
2001 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
2002 SDValue HiStore = DAG.getTruncStore(
2003 Chain, dl: SL, Val: Hi, Ptr: HiPtr, PtrInfo: SrcValue.getWithOffset(O: Size), SVT: HiMemVT, Alignment: HiAlign,
2004 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
2005
2006 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: LoStore, N2: HiStore);
2007}
2008
2009// This is a shortcut for integer division because we have fast i32<->f32
2010// conversions, and fast f32 reciprocal instructions.
2011SDValue AMDGPUTargetLowering::LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG,
2012 bool Sign) const {
2013 SDLoc DL(Op);
2014 EVT VT = Op.getValueType();
2015 assert(VT == MVT::i32 && "LowerDIVREMToFloat expects an i32");
2016
2017 SDValue LHS = Op.getOperand(i: 0);
2018 SDValue RHS = Op.getOperand(i: 1);
2019 MVT IntVT = MVT::i32;
2020 MVT FltVT = MVT::f32;
2021
2022 unsigned LHSSignBits;
2023 unsigned RHSSignBits;
2024 if (Sign) {
2025 LHSSignBits = DAG.ComputeNumSignBits(Op: LHS);
2026 RHSSignBits = DAG.ComputeNumSignBits(Op: RHS);
2027 if (LHSSignBits < 9 || RHSSignBits < 9)
2028 return SDValue();
2029 } else {
2030 KnownBits LHSKnown = DAG.computeKnownBits(Op: LHS);
2031 KnownBits RHSKnown = DAG.computeKnownBits(Op: RHS);
2032
2033 LHSSignBits = LHSKnown.countMinLeadingZeros();
2034 RHSSignBits = RHSKnown.countMinLeadingZeros();
2035 }
2036
2037 unsigned BitSize = VT.getSizeInBits();
2038 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
2039 unsigned DivBits = BitSize - SignBits;
2040 if (Sign)
2041 ++DivBits;
2042
2043 // In order to avoid problems due to 1 ulp accuracy issues with v_rcp_f32,
2044 // limit LowerDIVREMToFloat to:
2045 // [-0x400000,0x3FFFFF] for Sign
2046 // [ 0x000000,0x3FFFFF] for !Sign
2047 // This matches what is done in expandDivRemToFloatImpl.
2048 if (DivBits > (Sign ? 23 : 22))
2049 return SDValue();
2050
2051 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
2052 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
2053
2054 // int ia = (int)LHS;
2055 SDValue ia = LHS;
2056
2057 // int ib, (int)RHS;
2058 SDValue ib = RHS;
2059
2060 // The calculation:
2061 // fq = fa*recip(fb)
2062 // may be too small due to the 1ulp accuracy in the recip
2063 // operation and rounding issues. Since fq is truncated to produce
2064 // an integer value it may be too small by one. This is
2065 // dealt with by incrementing fa by 1ulp:
2066 // fq = (fa+1ulp)*recip(fb)
2067 // This will increase fa's magnitude by at most 0.5
2068 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
2069 // Thus, this method is safe since fa must be incremented by at least 1.0
2070 // for the quotient to increase by one.
2071 SDValue fa = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ia);
2072 SDValue faAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: fa);
2073 SDValue faIncremented = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: faAsInt,
2074 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
2075 fa = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FltVT, Operand: faIncremented);
2076
2077 // float fb = (float)ib;
2078 SDValue fb = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ib);
2079
2080 SDValue fq = DAG.getNode(Opcode: ISD::FMUL, DL, VT: FltVT,
2081 N1: fa, N2: DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: FltVT, Operand: fb));
2082
2083 // fq = trunc(fq);
2084 fq = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: FltVT, Operand: fq);
2085
2086 // int iq = (int)fq;
2087 SDValue Div = DAG.getNode(Opcode: ToInt, DL, VT: IntVT, Operand: fq);
2088
2089 // Rem needs compensation, it's easier to recompute it
2090 SDValue Rem = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Div, N2: RHS);
2091 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Rem);
2092
2093 return DAG.getMergeValues(Ops: { Div, Rem }, dl: DL);
2094}
2095
2096void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
2097 SelectionDAG &DAG,
2098 SmallVectorImpl<SDValue> &Results) const {
2099 SDLoc DL(Op);
2100 EVT VT = Op.getValueType();
2101
2102 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
2103
2104 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2105
2106 SDValue One = DAG.getConstant(Val: 1, DL, VT: HalfVT);
2107 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HalfVT);
2108
2109 //HiLo split
2110 SDValue LHS_Lo, LHS_Hi;
2111 SDValue LHS = Op.getOperand(i: 0);
2112 std::tie(args&: LHS_Lo, args&: LHS_Hi) = DAG.SplitScalar(N: LHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2113
2114 SDValue RHS_Lo, RHS_Hi;
2115 SDValue RHS = Op.getOperand(i: 1);
2116 std::tie(args&: RHS_Lo, args&: RHS_Hi) = DAG.SplitScalar(N: RHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2117
2118 if (DAG.MaskedValueIsZero(Op: RHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32)) &&
2119 DAG.MaskedValueIsZero(Op: LHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32))) {
2120
2121 SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2122 N1: LHS_Lo, N2: RHS_Lo);
2123
2124 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 0), Zero});
2125 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 1), Zero});
2126
2127 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV));
2128 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM));
2129 return;
2130 }
2131
2132 if (isTypeLegal(VT: MVT::i64)) {
2133 // The algorithm here is based on ideas from "Software Integer Division",
2134 // Tom Rodeheffer, August 2008.
2135
2136 MachineFunction &MF = DAG.getMachineFunction();
2137 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2138
2139 // Compute denominator reciprocal.
2140 unsigned FMAD =
2141 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA
2142 : MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign()
2143 ? (unsigned)ISD::FMAD
2144 : (unsigned)AMDGPUISD::FMAD_FTZ;
2145
2146 SDValue Cvt_Lo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Lo);
2147 SDValue Cvt_Hi = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Hi);
2148 SDValue Mad1 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Cvt_Hi,
2149 N2: DAG.getConstantFP(Val: APInt(32, 0x4f800000).bitsToFloat(), DL, VT: MVT::f32),
2150 N3: Cvt_Lo);
2151 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: MVT::f32, Operand: Mad1);
2152 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Rcp,
2153 N2: DAG.getConstantFP(Val: APInt(32, 0x5f7ffffc).bitsToFloat(), DL, VT: MVT::f32));
2154 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Mul1,
2155 N2: DAG.getConstantFP(Val: APInt(32, 0x2f800000).bitsToFloat(), DL, VT: MVT::f32));
2156 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: MVT::f32, Operand: Mul2);
2157 SDValue Mad2 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Trunc,
2158 N2: DAG.getConstantFP(Val: APInt(32, 0xcf800000).bitsToFloat(), DL, VT: MVT::f32),
2159 N3: Mul1);
2160 SDValue Rcp_Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Mad2);
2161 SDValue Rcp_Hi = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Trunc);
2162 SDValue Rcp64 = DAG.getBitcast(VT,
2163 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Rcp_Lo, Rcp_Hi}));
2164
2165 SDValue Zero64 = DAG.getConstant(Val: 0, DL, VT);
2166 SDValue One64 = DAG.getConstant(Val: 1, DL, VT);
2167 SDValue Zero1 = DAG.getConstant(Val: 0, DL, VT: MVT::i1);
2168 SDVTList HalfCarryVT = DAG.getVTList(VT1: HalfVT, VT2: MVT::i1);
2169
2170 // First round of UNR (Unsigned integer Newton-Raphson).
2171 SDValue Neg_RHS = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero64, N2: RHS);
2172 SDValue Mullo1 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Rcp64);
2173 SDValue Mulhi1 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Rcp64, N2: Mullo1);
2174 SDValue Mulhi1_Lo, Mulhi1_Hi;
2175 std::tie(args&: Mulhi1_Lo, args&: Mulhi1_Hi) =
2176 DAG.SplitScalar(N: Mulhi1, DL, LoVT: HalfVT, HiVT: HalfVT);
2177 SDValue Add1_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Lo,
2178 N2: Mulhi1_Lo, N3: Zero1);
2179 SDValue Add1_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Hi,
2180 N2: Mulhi1_Hi, N3: Add1_Lo.getValue(R: 1));
2181 SDValue Add1 = DAG.getBitcast(VT,
2182 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add1_Lo, Add1_Hi}));
2183
2184 // Second round of UNR.
2185 SDValue Mullo2 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Add1);
2186 SDValue Mulhi2 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Add1, N2: Mullo2);
2187 SDValue Mulhi2_Lo, Mulhi2_Hi;
2188 std::tie(args&: Mulhi2_Lo, args&: Mulhi2_Hi) =
2189 DAG.SplitScalar(N: Mulhi2, DL, LoVT: HalfVT, HiVT: HalfVT);
2190 SDValue Add2_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Lo,
2191 N2: Mulhi2_Lo, N3: Zero1);
2192 SDValue Add2_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Hi,
2193 N2: Mulhi2_Hi, N3: Add2_Lo.getValue(R: 1));
2194 SDValue Add2 = DAG.getBitcast(VT,
2195 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add2_Lo, Add2_Hi}));
2196
2197 SDValue Mulhi3 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: LHS, N2: Add2);
2198
2199 SDValue Mul3 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: RHS, N2: Mulhi3);
2200
2201 SDValue Mul3_Lo, Mul3_Hi;
2202 std::tie(args&: Mul3_Lo, args&: Mul3_Hi) = DAG.SplitScalar(N: Mul3, DL, LoVT: HalfVT, HiVT: HalfVT);
2203 SDValue Sub1_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Lo,
2204 N2: Mul3_Lo, N3: Zero1);
2205 SDValue Sub1_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Hi,
2206 N2: Mul3_Hi, N3: Sub1_Lo.getValue(R: 1));
2207 SDValue Sub1_Mi = DAG.getNode(Opcode: ISD::SUB, DL, VT: HalfVT, N1: LHS_Hi, N2: Mul3_Hi);
2208 SDValue Sub1 = DAG.getBitcast(VT,
2209 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub1_Lo, Sub1_Hi}));
2210
2211 SDValue MinusOne = DAG.getConstant(Val: 0xffffffffu, DL, VT: HalfVT);
2212 SDValue C1 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2213 Cond: ISD::SETUGE);
2214 SDValue C2 = DAG.getSelectCC(DL, LHS: Sub1_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2215 Cond: ISD::SETUGE);
2216 SDValue C3 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: C2, False: C1, Cond: ISD::SETEQ);
2217
2218 // TODO: Here and below portions of the code can be enclosed into if/endif.
2219 // Currently control flow is unconditional and we have 4 selects after
2220 // potential endif to substitute PHIs.
2221
2222 // if C3 != 0 ...
2223 SDValue Sub2_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Lo,
2224 N2: RHS_Lo, N3: Zero1);
2225 SDValue Sub2_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Mi,
2226 N2: RHS_Hi, N3: Sub1_Lo.getValue(R: 1));
2227 SDValue Sub2_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2228 N2: Zero, N3: Sub2_Lo.getValue(R: 1));
2229 SDValue Sub2 = DAG.getBitcast(VT,
2230 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub2_Lo, Sub2_Hi}));
2231
2232 SDValue Add3 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mulhi3, N2: One64);
2233
2234 SDValue C4 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2235 Cond: ISD::SETUGE);
2236 SDValue C5 = DAG.getSelectCC(DL, LHS: Sub2_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2237 Cond: ISD::SETUGE);
2238 SDValue C6 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: C5, False: C4, Cond: ISD::SETEQ);
2239
2240 // if (C6 != 0)
2241 SDValue Add4 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Add3, N2: One64);
2242
2243 SDValue Sub3_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Lo,
2244 N2: RHS_Lo, N3: Zero1);
2245 SDValue Sub3_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2246 N2: RHS_Hi, N3: Sub2_Lo.getValue(R: 1));
2247 SDValue Sub3_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub3_Mi,
2248 N2: Zero, N3: Sub3_Lo.getValue(R: 1));
2249 SDValue Sub3 = DAG.getBitcast(VT,
2250 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub3_Lo, Sub3_Hi}));
2251
2252 // endif C6
2253 // endif C3
2254
2255 SDValue Sel1 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Add4, False: Add3, Cond: ISD::SETNE);
2256 SDValue Div = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel1, False: Mulhi3, Cond: ISD::SETNE);
2257
2258 SDValue Sel2 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Sub3, False: Sub2, Cond: ISD::SETNE);
2259 SDValue Rem = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel2, False: Sub1, Cond: ISD::SETNE);
2260
2261 Results.push_back(Elt: Div);
2262 Results.push_back(Elt: Rem);
2263
2264 return;
2265 }
2266
2267 // r600 expandion.
2268 // Get Speculative values
2269 SDValue DIV_Part = DAG.getNode(Opcode: ISD::UDIV, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2270 SDValue REM_Part = DAG.getNode(Opcode: ISD::UREM, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2271
2272 SDValue REM_Lo = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: REM_Part, False: LHS_Hi, Cond: ISD::SETEQ);
2273 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {REM_Lo, Zero});
2274 REM = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM);
2275
2276 SDValue DIV_Hi = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: DIV_Part, False: Zero, Cond: ISD::SETEQ);
2277 SDValue DIV_Lo = Zero;
2278
2279 const unsigned halfBitWidth = HalfVT.getSizeInBits();
2280
2281 for (unsigned i = 0; i < halfBitWidth; ++i) {
2282 const unsigned bitPos = halfBitWidth - i - 1;
2283 SDValue POS = DAG.getConstant(Val: bitPos, DL, VT: HalfVT);
2284 // Get value of high bit
2285 SDValue HBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: HalfVT, N1: LHS_Lo, N2: POS);
2286 HBit = DAG.getNode(Opcode: ISD::AND, DL, VT: HalfVT, N1: HBit, N2: One);
2287 HBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: HBit);
2288
2289 // Shift
2290 REM = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: REM, N2: DAG.getConstant(Val: 1, DL, VT));
2291 // Add LHS high bit
2292 REM = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: REM, N2: HBit);
2293
2294 SDValue BIT = DAG.getConstant(Val: 1ULL << bitPos, DL, VT: HalfVT);
2295 SDValue realBIT = DAG.getSelectCC(DL, LHS: REM, RHS, True: BIT, False: Zero, Cond: ISD::SETUGE);
2296
2297 DIV_Lo = DAG.getNode(Opcode: ISD::OR, DL, VT: HalfVT, N1: DIV_Lo, N2: realBIT);
2298
2299 // Update REM
2300 SDValue REM_sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: REM, N2: RHS);
2301 REM = DAG.getSelectCC(DL, LHS: REM, RHS, True: REM_sub, False: REM, Cond: ISD::SETUGE);
2302 }
2303
2304 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {DIV_Lo, DIV_Hi});
2305 DIV = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV);
2306 Results.push_back(Elt: DIV);
2307 Results.push_back(Elt: REM);
2308}
2309
2310SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
2311 SelectionDAG &DAG) const {
2312 SDLoc DL(Op);
2313 EVT VT = Op.getValueType();
2314
2315 if (VT == MVT::i64) {
2316 SmallVector<SDValue, 2> Results;
2317 LowerUDIVREM64(Op, DAG, Results);
2318 return DAG.getMergeValues(Ops: Results, dl: DL);
2319 }
2320
2321 if (VT == MVT::i32) {
2322 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: false))
2323 return Res;
2324 }
2325
2326 SDValue X = Op.getOperand(i: 0);
2327 SDValue Y = Op.getOperand(i: 1);
2328
2329 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
2330 // algorithm used here.
2331
2332 // Initial estimate of inv(y).
2333 SDValue Z = DAG.getNode(Opcode: AMDGPUISD::URECIP, DL, VT, Operand: Y);
2334
2335 // One round of UNR.
2336 SDValue NegY = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Y);
2337 SDValue NegYZ = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: NegY, N2: Z);
2338 Z = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Z,
2339 N2: DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Z, N2: NegYZ));
2340
2341 // Quotient/remainder estimate.
2342 SDValue Q = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: X, N2: Z);
2343 SDValue R =
2344 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: X, N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Q, N2: Y));
2345
2346 // First quotient/remainder refinement.
2347 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2348 SDValue One = DAG.getConstant(Val: 1, DL, VT);
2349 SDValue Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2350 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2351 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2352 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2353 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2354
2355 // Second quotient/remainder refinement.
2356 Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2357 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2358 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2359 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2360 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2361
2362 return DAG.getMergeValues(Ops: {Q, R}, dl: DL);
2363}
2364
2365SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
2366 SelectionDAG &DAG) const {
2367 SDLoc DL(Op);
2368 EVT VT = Op.getValueType();
2369
2370 SDValue LHS = Op.getOperand(i: 0);
2371 SDValue RHS = Op.getOperand(i: 1);
2372
2373 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
2374 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2375
2376 if (VT == MVT::i32) {
2377 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: true))
2378 return Res;
2379 }
2380
2381 // LHS must have > 33 sign-bits to ensure that LHS != -2147483648
2382 // Otherwise 32-bit division cannot be used safely.
2383 // -2147483648/1 and -2147483648/-1 are not equal,
2384 // but they produce the same lower 32-bit result.
2385 if (VT == MVT::i64 && DAG.ComputeNumSignBits(Op: LHS) > 33 &&
2386 DAG.ComputeNumSignBits(Op: RHS) > 32) {
2387 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2388
2389 //HiLo split
2390 SDValue LHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: LHS, N2: Zero);
2391 SDValue RHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: RHS, N2: Zero);
2392 SDValue DIVREM = DAG.getNode(Opcode: ISD::SDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2393 N1: LHS_Lo, N2: RHS_Lo);
2394 SDValue Res[2] = {
2395 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 0)),
2396 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 1))
2397 };
2398 return DAG.getMergeValues(Ops: Res, dl: DL);
2399 }
2400
2401 SDValue LHSign = DAG.getSelectCC(DL, LHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2402 SDValue RHSign = DAG.getSelectCC(DL, LHS: RHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2403 SDValue DSign = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHSign, N2: RHSign);
2404 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2405
2406 LHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: LHSign);
2407 RHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: RHS, N2: RHSign);
2408
2409 LHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHS, N2: LHSign);
2410 RHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: RHS, N2: RHSign);
2411
2412 SDValue Div = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
2413 SDValue Rem = Div.getValue(R: 1);
2414
2415 Div = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Div, N2: DSign);
2416 Rem = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Rem, N2: RSign);
2417
2418 Div = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Div, N2: DSign);
2419 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Rem, N2: RSign);
2420
2421 SDValue Res[2] = {
2422 Div,
2423 Rem
2424 };
2425 return DAG.getMergeValues(Ops: Res, dl: DL);
2426}
2427
2428SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2429 SDLoc SL(Op);
2430 SDValue Src = Op.getOperand(i: 0);
2431
2432 // result = trunc(src)
2433 // if (src > 0.0 && src != result)
2434 // result += 1.0
2435
2436 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2437
2438 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2439 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
2440
2441 EVT SetCCVT =
2442 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2443
2444 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOGT);
2445 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2446 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2447
2448 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: One, N3: Zero);
2449 // TODO: Should this propagate fast-math-flags?
2450 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2451}
2452
2453static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2454 SelectionDAG &DAG) {
2455 const unsigned FractBits = 52;
2456 const unsigned ExpBits = 11;
2457
2458 SDValue ExpPart = DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32,
2459 N1: Hi,
2460 N2: DAG.getConstant(Val: FractBits - 32, DL: SL, VT: MVT::i32),
2461 N3: DAG.getConstant(Val: ExpBits, DL: SL, VT: MVT::i32));
2462 SDValue Exp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ExpPart,
2463 N2: DAG.getConstant(Val: 1023, DL: SL, VT: MVT::i32));
2464
2465 return Exp;
2466}
2467
2468SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2469 SDLoc SL(Op);
2470 SDValue Src = Op.getOperand(i: 0);
2471
2472 assert(Op.getValueType() == MVT::f64);
2473
2474 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
2475
2476 // Extract the upper half, since this is where we will find the sign and
2477 // exponent.
2478 SDValue Hi = getHiHalf64(Op: Src, DAG);
2479
2480 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2481
2482 const unsigned FractBits = 52;
2483
2484 // Extract the sign bit.
2485 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, DL: SL, VT: MVT::i32);
2486 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Hi, N2: SignBitMask);
2487
2488 // Extend back to 64-bits.
2489 SDValue SignBit64 = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Zero, SignBit});
2490 SignBit64 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: SignBit64);
2491
2492 SDValue BcInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Src);
2493 const SDValue FractMask
2494 = DAG.getConstant(Val: (UINT64_C(1) << FractBits) - 1, DL: SL, VT: MVT::i64);
2495
2496 SDValue Shr = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: FractMask, N2: Exp);
2497 SDValue Not = DAG.getNOT(DL: SL, Val: Shr, VT: MVT::i64);
2498 SDValue Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i64, N1: BcInt, N2: Not);
2499
2500 EVT SetCCVT =
2501 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i32);
2502
2503 const SDValue FiftyOne = DAG.getConstant(Val: FractBits - 1, DL: SL, VT: MVT::i32);
2504
2505 SDValue ExpLt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: Zero, Cond: ISD::SETLT);
2506 SDValue ExpGt51 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: FiftyOne, Cond: ISD::SETGT);
2507
2508 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpLt0, N2: SignBit64, N3: Tmp0);
2509 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpGt51, N2: BcInt, N3: Tmp1);
2510
2511 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f64, Operand: Tmp2);
2512}
2513
2514SDValue AMDGPUTargetLowering::LowerFROUNDEVEN(SDValue Op,
2515 SelectionDAG &DAG) const {
2516 SDLoc SL(Op);
2517 SDValue Src = Op.getOperand(i: 0);
2518
2519 assert(Op.getValueType() == MVT::f64);
2520
2521 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2522 SDValue C1 = DAG.getConstantFP(Val: C1Val, DL: SL, VT: MVT::f64);
2523 SDValue CopySign = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MVT::f64, N1: C1, N2: Src);
2524
2525 // TODO: Should this propagate fast-math-flags?
2526
2527 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Src, N2: CopySign);
2528 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT: MVT::f64, N1: Tmp1, N2: CopySign);
2529
2530 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f64, Operand: Src);
2531
2532 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2533 SDValue C2 = DAG.getConstantFP(Val: C2Val, DL: SL, VT: MVT::f64);
2534
2535 EVT SetCCVT =
2536 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2537 SDValue Cond = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Fabs, RHS: C2, Cond: ISD::SETOGT);
2538
2539 return DAG.getSelect(DL: SL, VT: MVT::f64, Cond, LHS: Src, RHS: Tmp2);
2540}
2541
2542SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op,
2543 SelectionDAG &DAG) const {
2544 // FNEARBYINT and FRINT are the same, except in their handling of FP
2545 // exceptions. Those aren't really meaningful for us, and OpenCL only has
2546 // rint, so just treat them as equivalent.
2547 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT: Op.getValueType(),
2548 Operand: Op.getOperand(i: 0));
2549}
2550
2551SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2552 auto VT = Op.getValueType();
2553 auto Arg = Op.getOperand(i: 0u);
2554 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT, Operand: Arg);
2555}
2556
2557// XXX - May require not supporting f32 denormals?
2558
2559// Don't handle v2f16. The extra instructions to scalarize and repack around the
2560// compare and vselect end up producing worse code than scalarizing the whole
2561// operation.
2562SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2563 SDLoc SL(Op);
2564 SDValue X = Op.getOperand(i: 0);
2565 EVT VT = Op.getValueType();
2566
2567 SDValue T = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: X);
2568
2569 // TODO: Should this propagate fast-math-flags?
2570
2571 SDValue Diff = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: T);
2572
2573 SDValue AbsDiff = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Diff);
2574
2575 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2576 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2577
2578 EVT SetCCVT =
2579 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2580
2581 const SDValue Half = DAG.getConstantFP(Val: 0.5, DL: SL, VT);
2582 SDValue Cmp = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsDiff, RHS: Half, Cond: ISD::SETOGE);
2583 SDValue OneOrZeroFP = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cmp, N2: One, N3: Zero);
2584
2585 SDValue SignedOffset = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: OneOrZeroFP, N2: X);
2586 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: T, N2: SignedOffset);
2587}
2588
2589SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2590 SDLoc SL(Op);
2591 SDValue Src = Op.getOperand(i: 0);
2592
2593 // result = trunc(src);
2594 // if (src < 0.0 && src != result)
2595 // result += -1.0.
2596
2597 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2598
2599 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2600 const SDValue NegOne = DAG.getConstantFP(Val: -1.0, DL: SL, VT: MVT::f64);
2601
2602 EVT SetCCVT =
2603 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2604
2605 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOLT);
2606 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2607 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2608
2609 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: NegOne, N3: Zero);
2610 // TODO: Should this propagate fast-math-flags?
2611 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2612}
2613
2614/// Return true if it's known that \p Src can never be an f32 denormal value.
2615static bool valueIsKnownNeverF32Denorm(SDValue Src) {
2616 switch (Src.getOpcode()) {
2617 case ISD::FP_EXTEND:
2618 return Src.getOperand(i: 0).getValueType() == MVT::f16;
2619 case ISD::FP16_TO_FP:
2620 case ISD::FFREXP:
2621 case ISD::FSQRT:
2622 case AMDGPUISD::LOG:
2623 case AMDGPUISD::EXP:
2624 return true;
2625 case ISD::INTRINSIC_WO_CHAIN: {
2626 unsigned IntrinsicID = Src.getConstantOperandVal(i: 0);
2627 switch (IntrinsicID) {
2628 case Intrinsic::amdgcn_frexp_mant:
2629 case Intrinsic::amdgcn_log:
2630 case Intrinsic::amdgcn_log_clamp:
2631 case Intrinsic::amdgcn_exp2:
2632 case Intrinsic::amdgcn_sqrt:
2633 return true;
2634 default:
2635 return false;
2636 }
2637 }
2638 default:
2639 return false;
2640 }
2641
2642 llvm_unreachable("covered opcode switch");
2643}
2644
2645bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG,
2646 SDNodeFlags Flags) {
2647 return Flags.hasApproximateFuncs();
2648}
2649
2650bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG,
2651 SDValue Src,
2652 SDNodeFlags Flags) {
2653 return !valueIsKnownNeverF32Denorm(Src) &&
2654 DAG.getMachineFunction()
2655 .getDenormalMode(FPType: APFloat::IEEEsingle())
2656 .Input != DenormalMode::PreserveSign;
2657}
2658
2659SDValue AMDGPUTargetLowering::getIsLtSmallestNormal(SelectionDAG &DAG,
2660 SDValue Src,
2661 SDNodeFlags Flags) const {
2662 SDLoc SL(Src);
2663 EVT VT = Src.getValueType();
2664 const fltSemantics &Semantics = VT.getFltSemantics();
2665 SDValue SmallestNormal =
2666 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2667
2668 // Want to scale denormals up, but negatives and 0 work just as well on the
2669 // scaled path.
2670 SDValue IsLtSmallestNormal = DAG.getSetCC(
2671 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2672 RHS: SmallestNormal, Cond: ISD::SETOLT);
2673
2674 return IsLtSmallestNormal;
2675}
2676
2677SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src,
2678 SDNodeFlags Flags) const {
2679 SDLoc SL(Src);
2680 EVT VT = Src.getValueType();
2681 const fltSemantics &Semantics = VT.getFltSemantics();
2682 SDValue Inf = DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL: SL, VT);
2683
2684 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Src, Flags);
2685 SDValue IsFinite = DAG.getSetCC(
2686 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Fabs,
2687 RHS: Inf, Cond: ISD::SETOLT);
2688 return IsFinite;
2689}
2690
2691/// If denormal handling is required return the scaled input to FLOG2, and the
2692/// check for denormal range. Otherwise, return null values.
2693std::pair<SDValue, SDValue>
2694AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL,
2695 SDValue Src, SDNodeFlags Flags) const {
2696 if (!needsDenormHandlingF32(DAG, Src, Flags))
2697 return {};
2698
2699 MVT VT = MVT::f32;
2700 const fltSemantics &Semantics = APFloat::IEEEsingle();
2701 SDValue SmallestNormal =
2702 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2703
2704 SDValue IsLtSmallestNormal = DAG.getSetCC(
2705 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2706 RHS: SmallestNormal, Cond: ISD::SETOLT);
2707
2708 SDValue Scale32 = DAG.getConstantFP(Val: 0x1.0p+32, DL: SL, VT);
2709 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2710 SDValue ScaleFactor =
2711 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: Scale32, N3: One, Flags);
2712
2713 SDValue ScaledInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Src, N2: ScaleFactor, Flags);
2714 return {ScaledInput, IsLtSmallestNormal};
2715}
2716
2717SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const {
2718 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
2719 // If we have to handle denormals, scale up the input and adjust the result.
2720
2721 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
2722 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
2723
2724 SDLoc SL(Op);
2725 EVT VT = Op.getValueType();
2726 SDValue Src = Op.getOperand(i: 0);
2727 SDNodeFlags Flags = Op->getFlags();
2728
2729 if (VT == MVT::f16) {
2730 // Nothing in half is a denormal when promoted to f32.
2731 assert(!isTypeLegal(VT));
2732 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
2733 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
2734 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
2735 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
2736 }
2737
2738 auto [ScaledInput, IsLtSmallestNormal] =
2739 getScaledLogInput(DAG, SL, Src, Flags);
2740 if (!ScaledInput)
2741 return DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: Src, Flags);
2742
2743 SDValue Log2 = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2744
2745 SDValue ThirtyTwo = DAG.getConstantFP(Val: 32.0, DL: SL, VT);
2746 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2747 SDValue ResultOffset =
2748 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: ThirtyTwo, N3: Zero);
2749 return DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: Log2, N2: ResultOffset, Flags);
2750}
2751
2752static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2753 SDValue Y, SDValue C, SDNodeFlags Flags = SDNodeFlags()) {
2754 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Y, Flags);
2755 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: C, Flags);
2756}
2757
2758SDValue AMDGPUTargetLowering::LowerFLOGCommon(SDValue Op,
2759 SelectionDAG &DAG) const {
2760 SDValue X = Op.getOperand(i: 0);
2761 EVT VT = Op.getValueType();
2762 SDNodeFlags Flags = Op->getFlags();
2763 SDLoc DL(Op);
2764 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10;
2765 assert(IsLog10 || Op.getOpcode() == ISD::FLOG);
2766
2767 if (VT == MVT::f16 || Flags.hasApproximateFuncs()) {
2768 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
2769 // depending on !fpmath metadata.
2770
2771 bool PromoteToF32 = VT == MVT::f16 && (!Flags.hasApproximateFuncs() ||
2772 !isTypeLegal(VT: MVT::f16));
2773
2774 if (PromoteToF32) {
2775 // Log and multiply in f32 is always good enough for f16.
2776 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2777 }
2778
2779 SDValue Lowered = LowerFLOGUnsafe(Op: X, SL: DL, DAG, IsLog10, Flags);
2780 if (PromoteToF32) {
2781 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Lowered,
2782 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32), Flags);
2783 }
2784
2785 return Lowered;
2786 }
2787
2788 SDValue ScaledInput, IsScaled;
2789 if (VT == MVT::f16)
2790 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2791 else {
2792 std::tie(args&: ScaledInput, args&: IsScaled) = getScaledLogInput(DAG, SL: DL, Src: X, Flags);
2793 if (ScaledInput)
2794 X = ScaledInput;
2795 }
2796
2797 SDValue Y = DAG.getNode(Opcode: AMDGPUISD::LOG, DL, VT, Operand: X, Flags);
2798
2799 SDValue R;
2800 if (Subtarget->hasFastFMAF32()) {
2801 // c+cc are ln(2)/ln(10) to more than 49 bits
2802 const float c_log10 = 0x1.344134p-2f;
2803 const float cc_log10 = 0x1.09f79ep-26f;
2804
2805 // c + cc is ln(2) to more than 49 bits
2806 const float c_log = 0x1.62e42ep-1f;
2807 const float cc_log = 0x1.efa39ep-25f;
2808
2809 SDValue C = DAG.getConstantFP(Val: IsLog10 ? c_log10 : c_log, DL, VT);
2810 SDValue CC = DAG.getConstantFP(Val: IsLog10 ? cc_log10 : cc_log, DL, VT);
2811 // This adds correction terms for which contraction may lead to an increase
2812 // in the error of the approximation, so disable it.
2813 Flags.setAllowContract(false);
2814 R = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Y, N2: C, Flags);
2815 SDValue NegR = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: R, Flags);
2816 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: C, N3: NegR, Flags);
2817 SDValue FMA1 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: CC, N3: FMA0, Flags);
2818 R = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: R, N2: FMA1, Flags);
2819 } else {
2820 // ch+ct is ln(2)/ln(10) to more than 36 bits
2821 const float ch_log10 = 0x1.344000p-2f;
2822 const float ct_log10 = 0x1.3509f6p-18f;
2823
2824 // ch + ct is ln(2) to more than 36 bits
2825 const float ch_log = 0x1.62e000p-1f;
2826 const float ct_log = 0x1.0bfbe8p-15f;
2827
2828 SDValue CH = DAG.getConstantFP(Val: IsLog10 ? ch_log10 : ch_log, DL, VT);
2829 SDValue CT = DAG.getConstantFP(Val: IsLog10 ? ct_log10 : ct_log, DL, VT);
2830
2831 SDValue YAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Y);
2832 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL, VT: MVT::i32);
2833 SDValue YHInt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: YAsInt, N2: MaskConst);
2834 SDValue YH = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: YHInt);
2835 SDValue YT = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: YH, Flags);
2836 // This adds correction terms for which contraction may lead to an increase
2837 // in the error of the approximation, so disable it.
2838 Flags.setAllowContract(false);
2839 SDValue YTCT = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: YT, N2: CT, Flags);
2840 SDValue Mad0 = getMad(DAG, SL: DL, VT, X: YH, Y: CT, C: YTCT, Flags);
2841 SDValue Mad1 = getMad(DAG, SL: DL, VT, X: YT, Y: CH, C: Mad0, Flags);
2842 R = getMad(DAG, SL: DL, VT, X: YH, Y: CH, C: Mad1);
2843 }
2844
2845 const bool IsFiniteOnly = Flags.hasNoNaNs() && Flags.hasNoInfs();
2846
2847 // TODO: Check if known finite from source value.
2848 if (!IsFiniteOnly) {
2849 SDValue IsFinite = getIsFinite(DAG, Src: Y, Flags);
2850 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsFinite, N2: R, N3: Y, Flags);
2851 }
2852
2853 if (IsScaled) {
2854 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
2855 SDValue ShiftK =
2856 DAG.getConstantFP(Val: IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2857 SDValue Shift =
2858 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsScaled, N2: ShiftK, N3: Zero, Flags);
2859 R = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: R, N2: Shift, Flags);
2860 }
2861
2862 return R;
2863}
2864
2865SDValue AMDGPUTargetLowering::LowerFLOG10(SDValue Op, SelectionDAG &DAG) const {
2866 return LowerFLOGCommon(Op, DAG);
2867}
2868
2869// Do f32 fast math expansion for flog2 or flog10. This is accurate enough for a
2870// promote f16 operation.
2871SDValue AMDGPUTargetLowering::LowerFLOGUnsafe(SDValue Src, const SDLoc &SL,
2872 SelectionDAG &DAG, bool IsLog10,
2873 SDNodeFlags Flags) const {
2874 EVT VT = Src.getValueType();
2875 unsigned LogOp =
2876 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2;
2877
2878 double Log2BaseInverted =
2879 IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2880
2881 if (VT == MVT::f32) {
2882 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags);
2883 if (ScaledInput) {
2884 SDValue LogSrc = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2885 SDValue ScaledResultOffset =
2886 DAG.getConstantFP(Val: -32.0 * Log2BaseInverted, DL: SL, VT);
2887
2888 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL: SL, VT);
2889
2890 SDValue ResultOffset = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsScaled,
2891 N2: ScaledResultOffset, N3: Zero, Flags);
2892
2893 SDValue Log2Inv = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2894
2895 if (Subtarget->hasFastFMAF32())
2896 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: LogSrc, N2: Log2Inv, N3: ResultOffset,
2897 Flags);
2898 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LogSrc, N2: Log2Inv, Flags);
2899 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: ResultOffset);
2900 }
2901 }
2902
2903 SDValue Log2Operand = DAG.getNode(Opcode: LogOp, DL: SL, VT, Operand: Src, Flags);
2904 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2905
2906 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Log2Operand, N2: Log2BaseInvertedOperand,
2907 Flags);
2908}
2909
2910// This expansion gives a result slightly better than 1ulp.
2911SDValue AMDGPUTargetLowering::lowerFEXPF64(SDValue Op,
2912 SelectionDAG &DAG) const {
2913 SDLoc DL(Op);
2914 SDValue X = Op.getOperand(i: 0);
2915
2916 // TODO: Check if reassoc is safe. There is an output change in exp2 and
2917 // exp10, which slightly increases ulp.
2918 SDNodeFlags Flags = Op->getFlags() & ~SDNodeFlags::AllowReassociation;
2919
2920 SDValue DN, F, T;
2921
2922 if (Op.getOpcode() == ISD::FEXP2) {
2923 // dn = rint(x)
2924 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: X, Flags);
2925 // f = x - dn
2926 F = DAG.getNode(Opcode: ISD::FSUB, DL, VT: MVT::f64, N1: X, N2: DN, Flags);
2927 // t = f*C1 + f*C2
2928 SDValue C1 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2929 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2930 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C2, Flags);
2931 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C1, N3: Mul2, Flags);
2932 } else if (Op.getOpcode() == ISD::FEXP10) {
2933 // dn = rint(x * C1)
2934 SDValue C1 = DAG.getConstantFP(Val: 0x1.a934f0979a371p+1, DL, VT: MVT::f64);
2935 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2936 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2937
2938 // f = FMA(-dn, C2, FMA(-dn, C3, x))
2939 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2940 SDValue C2 = DAG.getConstantFP(Val: -0x1.9dc1da994fd21p-59, DL, VT: MVT::f64);
2941 SDValue C3 = DAG.getConstantFP(Val: 0x1.34413509f79ffp-2, DL, VT: MVT::f64);
2942 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2943 F = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2944
2945 // t = FMA(f, C4, f*C5)
2946 SDValue C4 = DAG.getConstantFP(Val: 0x1.26bb1bbb55516p+1, DL, VT: MVT::f64);
2947 SDValue C5 = DAG.getConstantFP(Val: -0x1.f48ad494ea3e9p-53, DL, VT: MVT::f64);
2948 SDValue MulF = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C5, Flags);
2949 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C4, N3: MulF, Flags);
2950 } else { // ISD::FEXP
2951 // dn = rint(x * C1)
2952 SDValue C1 = DAG.getConstantFP(Val: 0x1.71547652b82fep+0, DL, VT: MVT::f64);
2953 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2954 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2955
2956 // t = FMA(-dn, C2, FMA(-dn, C3, x))
2957 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2958 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2959 SDValue C3 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2960 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2961 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2962 }
2963
2964 // Polynomial expansion for p
2965 SDValue P = DAG.getConstantFP(Val: 0x1.ade156a5dcb37p-26, DL, VT: MVT::f64);
2966 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2967 N3: DAG.getConstantFP(Val: 0x1.28af3fca7ab0cp-22, DL, VT: MVT::f64),
2968 Flags);
2969 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2970 N3: DAG.getConstantFP(Val: 0x1.71dee623fde64p-19, DL, VT: MVT::f64),
2971 Flags);
2972 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2973 N3: DAG.getConstantFP(Val: 0x1.a01997c89e6b0p-16, DL, VT: MVT::f64),
2974 Flags);
2975 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2976 N3: DAG.getConstantFP(Val: 0x1.a01a014761f6ep-13, DL, VT: MVT::f64),
2977 Flags);
2978 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2979 N3: DAG.getConstantFP(Val: 0x1.6c16c1852b7b0p-10, DL, VT: MVT::f64),
2980 Flags);
2981 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2982 N3: DAG.getConstantFP(Val: 0x1.1111111122322p-7, DL, VT: MVT::f64), Flags);
2983 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2984 N3: DAG.getConstantFP(Val: 0x1.55555555502a1p-5, DL, VT: MVT::f64), Flags);
2985 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2986 N3: DAG.getConstantFP(Val: 0x1.5555555555511p-3, DL, VT: MVT::f64), Flags);
2987 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2988 N3: DAG.getConstantFP(Val: 0x1.000000000000bp-1, DL, VT: MVT::f64), Flags);
2989
2990 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT: MVT::f64);
2991
2992 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
2993 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
2994
2995 // z = ldexp(p, (int)dn)
2996 SDValue DNInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: MVT::i32, Operand: DN);
2997 SDValue Z = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: P, N2: DNInt, Flags);
2998
2999 // Overflow/underflow guards
3000 SDValue CondHi = DAG.getSetCC(
3001 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: 1024.0, DL, VT: MVT::f64), Cond: ISD::SETULE);
3002
3003 if (!Flags.hasNoInfs()) {
3004 SDValue PInf = DAG.getConstantFP(Val: std::numeric_limits<double>::infinity(),
3005 DL, VT: MVT::f64);
3006 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondHi, LHS: Z, RHS: PInf, Flags);
3007 }
3008
3009 SDValue CondLo = DAG.getSetCC(
3010 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: -1075.0, DL, VT: MVT::f64), Cond: ISD::SETUGE);
3011 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
3012 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondLo, LHS: Z, RHS: Zero, Flags);
3013
3014 return Z;
3015}
3016
3017SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const {
3018 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
3019 // If we have to handle denormals, scale up the input and adjust the result.
3020
3021 EVT VT = Op.getValueType();
3022 if (VT == MVT::f64)
3023 return lowerFEXPF64(Op, DAG);
3024
3025 SDLoc SL(Op);
3026 SDValue Src = Op.getOperand(i: 0);
3027 SDNodeFlags Flags = Op->getFlags();
3028
3029 if (VT == MVT::f16) {
3030 // Nothing in half is a denormal when promoted to f32.
3031 assert(!isTypeLegal(MVT::f16));
3032 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3033 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
3034 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
3035 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3036 }
3037
3038 assert(VT == MVT::f32);
3039
3040 if (!needsDenormHandlingF32(DAG, Src, Flags))
3041 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3042
3043 // bool needs_scaling = x < -0x1.f80000p+6f;
3044 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3045
3046 // -nextafter(128.0, -1)
3047 SDValue RangeCheckConst = DAG.getConstantFP(Val: -0x1.f80000p+6f, DL: SL, VT);
3048
3049 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3050
3051 SDValue NeedsScaling =
3052 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: RangeCheckConst, Cond: ISD::SETOLT);
3053
3054 SDValue SixtyFour = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3055 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3056
3057 SDValue AddOffset =
3058 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: SixtyFour, N3: Zero);
3059
3060 SDValue AddInput = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Src, N2: AddOffset, Flags);
3061 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: AddInput, Flags);
3062
3063 SDValue TwoExpNeg64 = DAG.getConstantFP(Val: 0x1.0p-64f, DL: SL, VT);
3064 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
3065 SDValue ResultScale =
3066 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: TwoExpNeg64, N3: One);
3067
3068 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScale, Flags);
3069}
3070
3071SDValue AMDGPUTargetLowering::lowerFEXPUnsafeImpl(SDValue X, const SDLoc &SL,
3072 SelectionDAG &DAG,
3073 SDNodeFlags Flags,
3074 bool IsExp10) const {
3075 // exp(x) -> exp2(M_LOG2E_F * x);
3076 // exp10(x) -> exp2(log2(10) * x);
3077 EVT VT = X.getValueType();
3078 SDValue Const =
3079 DAG.getConstantFP(Val: IsExp10 ? 0x1.a934f0p+1f : numbers::log2e, DL: SL, VT);
3080
3081 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Const, Flags);
3082 return DAG.getNode(Opcode: VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
3083 : (unsigned)ISD::FEXP2,
3084 DL: SL, VT, Operand: Mul, Flags);
3085}
3086
3087SDValue AMDGPUTargetLowering::lowerFEXPUnsafe(SDValue X, const SDLoc &SL,
3088 SelectionDAG &DAG,
3089 SDNodeFlags Flags) const {
3090 EVT VT = X.getValueType();
3091 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags))
3092 return lowerFEXPUnsafeImpl(X, SL, DAG, Flags, /*IsExp10=*/false);
3093
3094 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3095
3096 SDValue Threshold = DAG.getConstantFP(Val: -0x1.5d58a0p+6f, DL: SL, VT);
3097 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3098
3099 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3100
3101 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3102
3103 SDValue AdjustedX =
3104 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3105
3106 const SDValue Log2E = DAG.getConstantFP(Val: numbers::log2e, DL: SL, VT);
3107 SDValue ExpInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: Log2E, Flags);
3108
3109 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: ExpInput, Flags);
3110
3111 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.969d48p-93f, DL: SL, VT);
3112 SDValue AdjustedResult =
3113 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScaleFactor, Flags);
3114
3115 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: Exp2,
3116 Flags);
3117}
3118
3119/// Emit approx-funcs appropriate lowering for exp10. inf/nan should still be
3120/// handled correctly.
3121SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL,
3122 SelectionDAG &DAG,
3123 SDNodeFlags Flags) const {
3124 const EVT VT = X.getValueType();
3125
3126 const unsigned Exp2Op = VT == MVT::f32 ? static_cast<unsigned>(AMDGPUISD::EXP)
3127 : static_cast<unsigned>(ISD::FEXP2);
3128
3129 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags)) {
3130 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
3131 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3132 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3133
3134 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K0, Flags);
3135 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3136 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K1, Flags);
3137 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3138 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1);
3139 }
3140
3141 // bool s = x < -0x1.2f7030p+5f;
3142 // x += s ? 0x1.0p+5f : 0.0f;
3143 // exp10 = exp2(x * 0x1.a92000p+1f) *
3144 // exp2(x * 0x1.4f0978p-11f) *
3145 // (s ? 0x1.9f623ep-107f : 1.0f);
3146
3147 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3148
3149 SDValue Threshold = DAG.getConstantFP(Val: -0x1.2f7030p+5f, DL: SL, VT);
3150 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3151
3152 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+5f, DL: SL, VT);
3153 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3154 SDValue AdjustedX =
3155 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3156
3157 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3158 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3159
3160 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K0, Flags);
3161 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3162 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K1, Flags);
3163 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3164
3165 SDValue MulExps = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1, Flags);
3166
3167 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.9f623ep-107f, DL: SL, VT);
3168 SDValue AdjustedResult =
3169 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: MulExps, N2: ResultScaleFactor, Flags);
3170
3171 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: MulExps,
3172 Flags);
3173}
3174
3175SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
3176 EVT VT = Op.getValueType();
3177
3178 if (VT == MVT::f64)
3179 return lowerFEXPF64(Op, DAG);
3180
3181 SDLoc SL(Op);
3182 SDValue X = Op.getOperand(i: 0);
3183 SDNodeFlags Flags = Op->getFlags();
3184 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10;
3185
3186 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
3187 // library behavior. Also, is known-not-daz source sufficient?
3188 if (allowApproxFunc(DAG, Flags)) { // TODO: Does this really require fast?
3189 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags)
3190 : lowerFEXPUnsafe(X, SL, DAG, Flags);
3191 }
3192
3193 if (VT.getScalarType() == MVT::f16) {
3194 if (VT.isVector())
3195 return SDValue();
3196
3197 // Nothing in half is a denormal when promoted to f32.
3198 //
3199 // exp(f16 x) ->
3200 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
3201 //
3202 // exp10(f16 x) ->
3203 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
3204 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: X, Flags);
3205 SDValue Lowered = lowerFEXPUnsafeImpl(X: Ext, SL, DAG, Flags, IsExp10);
3206 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Lowered,
3207 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3208 }
3209
3210 assert(VT == MVT::f32);
3211
3212 // Algorithm:
3213 //
3214 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
3215 //
3216 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
3217 // n = 64*m + j, 0 <= j < 64
3218 //
3219 // e^x = 2^((64*m + j + f)/64)
3220 // = (2^m) * (2^(j/64)) * 2^(f/64)
3221 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
3222 //
3223 // f = x*(64/ln(2)) - n
3224 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
3225 //
3226 // e^x = (2^m) * (2^(j/64)) * e^r
3227 //
3228 // (2^(j/64)) is precomputed
3229 //
3230 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3231 // e^r = 1 + q
3232 //
3233 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3234 //
3235 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
3236 SDNodeFlags FlagsNoContract = Flags;
3237 FlagsNoContract.setAllowContract(false);
3238
3239 SDValue PH, PL;
3240 if (Subtarget->hasFastFMAF32()) {
3241 const float c_exp = numbers::log2ef;
3242 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
3243 const float c_exp10 = 0x1.a934f0p+1f;
3244 const float cc_exp10 = 0x1.2f346ep-24f;
3245
3246 SDValue C = DAG.getConstantFP(Val: IsExp10 ? c_exp10 : c_exp, DL: SL, VT);
3247 SDValue CC = DAG.getConstantFP(Val: IsExp10 ? cc_exp10 : cc_exp, DL: SL, VT);
3248
3249 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: C, Flags);
3250 SDValue NegPH = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: PH, Flags);
3251 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: C, N3: NegPH, Flags);
3252 PL = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: CC, N3: FMA0, Flags);
3253 } else {
3254 const float ch_exp = 0x1.714000p+0f;
3255 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
3256
3257 const float ch_exp10 = 0x1.a92000p+1f;
3258 const float cl_exp10 = 0x1.4f0978p-11f;
3259
3260 SDValue CH = DAG.getConstantFP(Val: IsExp10 ? ch_exp10 : ch_exp, DL: SL, VT);
3261 SDValue CL = DAG.getConstantFP(Val: IsExp10 ? cl_exp10 : cl_exp, DL: SL, VT);
3262
3263 SDValue XAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: X);
3264 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL: SL, VT: MVT::i32);
3265 SDValue XHAsInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: XAsInt, N2: MaskConst);
3266 SDValue XH = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: XHAsInt);
3267 SDValue XL = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: XH, Flags);
3268
3269 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XH, N2: CH, Flags);
3270
3271 SDValue XLCL = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XL, N2: CL, Flags);
3272 SDValue Mad0 = getMad(DAG, SL, VT, X: XL, Y: CH, C: XLCL, Flags);
3273 PL = getMad(DAG, SL, VT, X: XH, Y: CL, C: Mad0, Flags);
3274 }
3275
3276 SDValue E = DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SL, VT, Operand: PH, Flags);
3277
3278 // It is unsafe to contract this fsub into the PH multiply.
3279 SDValue PHSubE = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: PH, N2: E, Flags: FlagsNoContract);
3280
3281 SDValue A = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: PHSubE, N2: PL, Flags);
3282 SDValue IntE = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: SL, VT: MVT::i32, Operand: E);
3283 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: A, Flags);
3284
3285 SDValue R = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: Exp2, N2: IntE, Flags);
3286
3287 SDValue UnderflowCheckConst =
3288 DAG.getConstantFP(Val: IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, DL: SL, VT);
3289
3290 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3291 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3292 SDValue Underflow =
3293 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: UnderflowCheckConst, Cond: ISD::SETOLT);
3294
3295 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Underflow, N2: Zero, N3: R);
3296
3297 if (!Flags.hasNoInfs()) {
3298 SDValue OverflowCheckConst =
3299 DAG.getConstantFP(Val: IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, DL: SL, VT);
3300 SDValue Overflow =
3301 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: OverflowCheckConst, Cond: ISD::SETOGT);
3302 SDValue Inf =
3303 DAG.getConstantFP(Val: APFloat::getInf(Sem: APFloat::IEEEsingle()), DL: SL, VT);
3304 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Overflow, N2: Inf, N3: R);
3305 }
3306
3307 return R;
3308}
3309
3310static bool isCtlzOpc(unsigned Opc) {
3311 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
3312}
3313
3314static bool isCttzOpc(unsigned Opc) {
3315 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_POISON;
3316}
3317
3318SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op,
3319 SelectionDAG &DAG) const {
3320 auto SL = SDLoc(Op);
3321 auto Opc = Op.getOpcode();
3322 auto Arg = Op.getOperand(i: 0u);
3323 auto ResultVT = Op.getValueType();
3324
3325 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3326 return {};
3327
3328 assert(isCtlzOpc(Opc));
3329 assert(ResultVT == Arg.getValueType());
3330
3331 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3332 SDValue NumExtBits = DAG.getConstant(Val: 32u - NumBits, DL: SL, VT: MVT::i32);
3333 SDValue NewOp;
3334
3335 if (Opc == ISD::CTLZ_ZERO_POISON) {
3336 NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3337 NewOp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3338 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3339 } else {
3340 NewOp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3341 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3342 NewOp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3343 }
3344
3345 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ResultVT, Operand: NewOp);
3346}
3347
3348SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
3349 SDLoc SL(Op);
3350 SDValue Src = Op.getOperand(i: 0);
3351
3352 assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode()));
3353 bool Ctlz = isCtlzOpc(Opc: Op.getOpcode());
3354 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;
3355
3356 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_POISON ||
3357 Op.getOpcode() == ISD::CTTZ_ZERO_POISON;
3358 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3359
3360 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3361 // (ctlz hi:lo) -> (umin (ffbh src), 32)
3362 // (cttz hi:lo) -> (umin (ffbl src), 32)
3363 // (ctlz_zero_poison src) -> (ffbh src)
3364 // (cttz_zero_poison src) -> (ffbl src)
3365
3366 // 64-bit scalar version produce 32-bit result
3367 // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64)
3368 // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64)
3369 // (ctlz_zero_poison src) -> (S_FLBIT_I32_B64 src)
3370 // (cttz_zero_poison src) -> (S_FF1_I32_B64 src)
3371 SDValue NewOpr = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Src);
3372 if (!ZeroUndef) {
3373 const SDValue ConstVal = DAG.getConstant(
3374 Val: Op.getValueType().getScalarSizeInBits(), DL: SL, VT: MVT::i32);
3375 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: ConstVal);
3376 }
3377 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: Src.getValueType(), Operand: NewOpr);
3378 }
3379
3380 SDValue Lo, Hi;
3381 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3382
3383 SDValue OprLo = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Lo);
3384 SDValue OprHi = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Hi);
3385
3386 // (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64)
3387 // (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64)
3388 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
3389 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
3390
3391 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT;
3392 const SDValue Const32 = DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32);
3393 if (Ctlz)
3394 OprLo = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprLo, N2: Const32);
3395 else
3396 OprHi = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprHi, N2: Const32);
3397
3398 SDValue NewOpr;
3399 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: OprLo, N2: OprHi);
3400 if (!ZeroUndef) {
3401 const SDValue Const64 = DAG.getConstant(Val: 64, DL: SL, VT: MVT::i32);
3402 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: Const64);
3403 }
3404
3405 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: NewOpr);
3406}
3407
3408SDValue AMDGPUTargetLowering::LowerCTLS(SDValue Op, SelectionDAG &DAG) const {
3409 SDLoc SL(Op);
3410 SDValue Src = Op.getOperand(i: 0);
3411 assert(Src.getValueType() == MVT::i32 && "LowerCTLS only supports i32");
3412 SDValue Ffbh = DAG.getNode(
3413 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3414 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Src);
3415 SDValue Clamped = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: Ffbh,
3416 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3417 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: Clamped,
3418 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i32));
3419}
3420
3421SDValue AMDGPUTargetLowering::LowerINT_TO_FP16(SDValue Op, SelectionDAG &DAG,
3422 EVT FP16Ty) const {
3423 assert(FP16Ty == MVT::f16 || FP16Ty == MVT::bf16);
3424 SDLoc SL(Op);
3425 SDValue Src = Op.getOperand(i: 0);
3426 SDValue ToF32 = DAG.getNode(Opcode: Op.getOpcode(), DL: SL, VT: MVT::f32, Operand: Src);
3427 SDValue FPRoundFlag = DAG.getIntPtrConstant(Val: 0, DL: SL, /*isTarget=*/true);
3428 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: FP16Ty, N1: ToF32, N2: FPRoundFlag);
3429}
3430
3431SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
3432 bool Signed) const {
3433 // The regular method converting a 64-bit integer to float roughly consists of
3434 // 2 steps: normalization and rounding. In fact, after normalization, the
3435 // conversion from a 64-bit integer to a float is essentially the same as the
3436 // one from a 32-bit integer. The only difference is that it has more
3437 // trailing bits to be rounded. To leverage the native 32-bit conversion, a
3438 // 64-bit integer could be preprocessed and fit into a 32-bit integer then
3439 // converted into the correct float number. The basic steps for the unsigned
3440 // conversion are illustrated in the following pseudo code:
3441 //
3442 // f32 uitofp(i64 u) {
3443 // i32 hi, lo = split(u);
3444 // // Only count the leading zeros in hi as we have native support of the
3445 // // conversion from i32 to f32. If hi is all 0s, the conversion is
3446 // // reduced to a 32-bit one automatically.
3447 // i32 shamt = clz(hi); // Return 32 if hi is all 0s.
3448 // u <<= shamt;
3449 // hi, lo = split(u);
3450 // hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo.
3451 // // convert it as a 32-bit integer and scale the result back.
3452 // return uitofp(hi) * 2^(32 - shamt);
3453 // }
3454 //
3455 // The signed one follows the same principle but uses 'ffbh_i32' to count its
3456 // sign bits instead. If 'ffbh_i32' is not available, its absolute value is
3457 // converted instead followed by negation based its sign bit.
3458
3459 SDLoc SL(Op);
3460 SDValue Src = Op.getOperand(i: 0);
3461
3462 SDValue Lo, Hi;
3463 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3464 SDValue Sign;
3465 SDValue ShAmt;
3466 if (Signed && Subtarget->isGCN()) {
3467 // We also need to consider the sign bit in Lo if Hi has just sign bits,
3468 // i.e. Hi is 0 or -1. However, that only needs to take the MSB into
3469 // account. That is, the maximal shift is
3470 // - 32 if Lo and Hi have opposite signs;
3471 // - 33 if Lo and Hi have the same sign.
3472 //
3473 // Or, MaxShAmt = 33 + OppositeSign, where
3474 //
3475 // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is
3476 // - -1 if Lo and Hi have opposite signs; and
3477 // - 0 otherwise.
3478 //
3479 // All in all, ShAmt is calculated as
3480 //
3481 // umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1.
3482 //
3483 // or
3484 //
3485 // umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31).
3486 //
3487 // to reduce the critical path.
3488 SDValue OppositeSign = DAG.getNode(
3489 Opcode: ISD::SRA, DL: SL, VT: MVT::i32, N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: Lo, N2: Hi),
3490 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3491 SDValue MaxShAmt =
3492 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3493 N2: OppositeSign);
3494 // Count the leading sign bits.
3495 ShAmt = DAG.getNode(
3496 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3497 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Hi);
3498 // Different from unsigned conversion, the shift should be one bit less to
3499 // preserve the sign bit.
3500 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ShAmt,
3501 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
3502 ShAmt = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: ShAmt, N2: MaxShAmt);
3503 } else {
3504 if (Signed) {
3505 // Without 'ffbh_i32', only leading zeros could be counted. Take the
3506 // absolute value first.
3507 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: Src,
3508 N2: DAG.getConstant(Val: 63, DL: SL, VT: MVT::i64));
3509 SDValue Abs =
3510 DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64,
3511 N1: DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: Src, N2: Sign), N2: Sign);
3512 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Abs, DAG);
3513 }
3514 // Count the leading zeros.
3515 ShAmt = DAG.getNode(Opcode: ISD::CTLZ, DL: SL, VT: MVT::i32, Operand: Hi);
3516 // The shift amount for signed integers is [0, 32].
3517 }
3518 // Normalize the given 64-bit integer.
3519 SDValue Norm = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i64, N1: Src, N2: ShAmt);
3520 // Split it again.
3521 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Norm, DAG);
3522 // Calculate the adjust bit for rounding.
3523 // (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo)
3524 SDValue Adjust = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32,
3525 N1: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32), N2: Lo);
3526 // Get the 32-bit normalized integer.
3527 Norm = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Hi, N2: Adjust);
3528 // Convert the normalized 32-bit integer into f32.
3529
3530 bool UseLDEXP = isOperationLegal(Op: ISD::FLDEXP, VT: MVT::f32);
3531 unsigned Opc = Signed && UseLDEXP ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3532 SDValue FVal = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::f32, Operand: Norm);
3533
3534 // Finally, need to scale back the converted floating number as the original
3535 // 64-bit integer is converted as a 32-bit one.
3536 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3537 N2: ShAmt);
3538 // On GCN, use LDEXP directly.
3539 if (UseLDEXP)
3540 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f32, N1: FVal, N2: ShAmt);
3541
3542 // Otherwise, align 'ShAmt' to the exponent part and add it into the exponent
3543 // part directly to emulate the multiplication of 2^ShAmt. That 8-bit
3544 // exponent is enough to avoid overflowing into the sign bit.
3545 SDValue Exp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ShAmt,
3546 N2: DAG.getConstant(Val: 23, DL: SL, VT: MVT::i32));
3547 SDValue IVal =
3548 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32,
3549 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: FVal), N2: Exp);
3550 if (Signed) {
3551 // Set the sign bit.
3552 Sign = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32,
3553 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Sign),
3554 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3555 IVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: IVal, N2: Sign);
3556 }
3557 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: IVal);
3558}
3559
3560SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
3561 bool Signed) const {
3562 SDLoc SL(Op);
3563 SDValue Src = Op.getOperand(i: 0);
3564
3565 SDValue Lo, Hi;
3566 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3567
3568 SDValue CvtHi = DAG.getNode(Opcode: Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
3569 DL: SL, VT: MVT::f64, Operand: Hi);
3570
3571 SDValue CvtLo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: SL, VT: MVT::f64, Operand: Lo);
3572
3573 SDValue LdExp = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f64, N1: CvtHi,
3574 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3575 // TODO: Should this propagate fast-math-flags?
3576 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: LdExp, N2: CvtLo);
3577}
3578
3579SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
3580 SelectionDAG &DAG) const {
3581 // TODO: Factor out code common with LowerSINT_TO_FP.
3582 EVT DestVT = Op.getValueType();
3583 SDValue Src = Op.getOperand(i: 0);
3584 EVT SrcVT = Src.getValueType();
3585
3586 if (SrcVT == MVT::i16) {
3587 if (DestVT == MVT::f16)
3588 return Op;
3589 SDLoc DL(Op);
3590
3591 // Promote src to i32
3592 SDValue Ext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: Src);
3593 return DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: DestVT, Operand: Ext);
3594 }
3595
3596 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3597 return LowerINT_TO_FP16(Op, DAG, FP16Ty: DestVT);
3598
3599 if (SrcVT != MVT::i64)
3600 return Op;
3601
3602 if (DestVT == MVT::f32)
3603 return LowerINT_TO_FP32(Op, DAG, Signed: false);
3604
3605 assert(DestVT == MVT::f64);
3606 return LowerINT_TO_FP64(Op, DAG, Signed: false);
3607}
3608
3609SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
3610 SelectionDAG &DAG) const {
3611 EVT DestVT = Op.getValueType();
3612
3613 SDValue Src = Op.getOperand(i: 0);
3614 EVT SrcVT = Src.getValueType();
3615
3616 if (SrcVT == MVT::i16) {
3617 if (DestVT == MVT::f16)
3618 return Op;
3619
3620 SDLoc DL(Op);
3621 // Promote src to i32
3622 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i32, Operand: Src);
3623 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DestVT, Operand: Ext);
3624 }
3625
3626 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3627 return LowerINT_TO_FP16(Op, DAG, FP16Ty: DestVT);
3628
3629 if (SrcVT != MVT::i64)
3630 return Op;
3631
3632 // TODO: Factor out code common with LowerUINT_TO_FP.
3633
3634 if (DestVT == MVT::f32)
3635 return LowerINT_TO_FP32(Op, DAG, Signed: true);
3636
3637 assert(DestVT == MVT::f64);
3638 return LowerINT_TO_FP64(Op, DAG, Signed: true);
3639}
3640
3641SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
3642 bool Signed) const {
3643 SDLoc SL(Op);
3644
3645 SDValue Src = Op.getOperand(i: 0);
3646 EVT SrcVT = Src.getValueType();
3647
3648 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3649
3650 // The basic idea of converting a floating point number into a pair of 32-bit
3651 // integers is illustrated as follows:
3652 //
3653 // tf := trunc(val);
3654 // hif := floor(tf * 2^-32);
3655 // lof := tf - hif * 2^32; // lof is always positive due to floor.
3656 // hi := fptoi(hif);
3657 // lo := fptoi(lof);
3658 //
3659 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: SrcVT, Operand: Src);
3660 SDValue Sign;
3661 if (Signed && SrcVT == MVT::f32) {
3662 // However, a 32-bit floating point number has only 23 bits mantissa and
3663 // it's not enough to hold all the significant bits of `lof` if val is
3664 // negative. To avoid the loss of precision, We need to take the absolute
3665 // value after truncating and flip the result back based on the original
3666 // signedness.
3667 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i32,
3668 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Trunc),
3669 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3670 Trunc = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: SrcVT, Operand: Trunc);
3671 }
3672
3673 SDValue K0, K1;
3674 if (SrcVT == MVT::f64) {
3675 K0 = DAG.getConstantFP(
3676 Val: llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), DL: SL,
3677 VT: SrcVT);
3678 K1 = DAG.getConstantFP(
3679 Val: llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), DL: SL,
3680 VT: SrcVT);
3681 } else {
3682 K0 = DAG.getConstantFP(
3683 Val: llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), DL: SL, VT: SrcVT);
3684 K1 = DAG.getConstantFP(
3685 Val: llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), DL: SL, VT: SrcVT);
3686 }
3687 // TODO: Should this propagate fast-math-flags?
3688 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: SrcVT, N1: Trunc, N2: K0);
3689
3690 SDValue FloorMul = DAG.getNode(Opcode: ISD::FFLOOR, DL: SL, VT: SrcVT, Operand: Mul);
3691
3692 SDValue Fma = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: SrcVT, N1: FloorMul, N2: K1, N3: Trunc);
3693
3694 SDValue Hi = DAG.getNode(Opcode: (Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
3695 : ISD::FP_TO_UINT,
3696 DL: SL, VT: MVT::i32, Operand: FloorMul);
3697 SDValue Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SL, VT: MVT::i32, Operand: Fma);
3698
3699 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3700 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Lo, Hi}));
3701
3702 if (Signed && SrcVT == MVT::f32) {
3703 assert(Sign);
3704 // Flip the result based on the signedness, which is either all 0s or 1s.
3705 Sign = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3706 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Sign, Sign}));
3707 // r := xor(r, sign) - sign;
3708 Result =
3709 DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i64,
3710 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64, N1: Result, N2: Sign), N2: Sign);
3711 }
3712
3713 return Result;
3714}
3715
3716SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
3717 SDLoc DL(Op);
3718 SDValue N0 = Op.getOperand(i: 0);
3719
3720 // Convert to target node to get known bits
3721 if (N0.getValueType() == MVT::f32)
3722 return DAG.getNode(Opcode: AMDGPUISD::FP_TO_FP16, DL, VT: Op.getValueType(), Operand: N0);
3723
3724 if (Op->getFlags().hasApproximateFuncs()) {
3725 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
3726 return SDValue();
3727 }
3728
3729 return LowerF64ToF16Safe(Src: N0, DL, DAG);
3730}
3731
3732// return node in i32
3733SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
3734 SelectionDAG &DAG) const {
3735 assert(Src.getSimpleValueType() == MVT::f64);
3736
3737 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
3738 // TODO: We can generate better code for True16.
3739 const unsigned ExpMask = 0x7ff;
3740 const unsigned ExpBiasf64 = 1023;
3741 const unsigned ExpBiasf16 = 15;
3742 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3743 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
3744 SDValue U = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Src);
3745 SDValue UH = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: U,
3746 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3747 UH = DAG.getZExtOrTrunc(Op: UH, DL, VT: MVT::i32);
3748 U = DAG.getZExtOrTrunc(Op: U, DL, VT: MVT::i32);
3749 SDValue E = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3750 N2: DAG.getConstant(Val: 20, DL, VT: MVT::i64));
3751 E = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: E,
3752 N2: DAG.getConstant(Val: ExpMask, DL, VT: MVT::i32));
3753 // Subtract the fp64 exponent bias (1023) to get the real exponent and
3754 // add the f16 bias (15) to get the biased exponent for the f16 format.
3755 E = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: E,
3756 N2: DAG.getConstant(Val: -ExpBiasf64 + ExpBiasf16, DL, VT: MVT::i32));
3757
3758 SDValue M = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3759 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32));
3760 M = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M,
3761 N2: DAG.getConstant(Val: 0xffe, DL, VT: MVT::i32));
3762
3763 SDValue MaskedSig = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: UH,
3764 N2: DAG.getConstant(Val: 0x1ff, DL, VT: MVT::i32));
3765 MaskedSig = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: MaskedSig, N2: U);
3766
3767 SDValue Lo40Set = DAG.getSelectCC(DL, LHS: MaskedSig, RHS: Zero, True: Zero, False: One, Cond: ISD::SETEQ);
3768 M = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M, N2: Lo40Set);
3769
3770 // (M != 0 ? 0x0200 : 0) | 0x7c00;
3771 SDValue I = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
3772 N1: DAG.getSelectCC(DL, LHS: M, RHS: Zero, True: DAG.getConstant(Val: 0x0200, DL, VT: MVT::i32),
3773 False: Zero, Cond: ISD::SETNE), N2: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32));
3774
3775 // N = M | (E << 12);
3776 SDValue N = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3777 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: E,
3778 N2: DAG.getConstant(Val: 12, DL, VT: MVT::i32)));
3779
3780 // B = clamp(1-E, 0, 13);
3781 SDValue OneSubExp = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
3782 N1: One, N2: E);
3783 SDValue B = DAG.getNode(Opcode: ISD::SMAX, DL, VT: MVT::i32, N1: OneSubExp, N2: Zero);
3784 B = DAG.getNode(Opcode: ISD::SMIN, DL, VT: MVT::i32, N1: B,
3785 N2: DAG.getConstant(Val: 13, DL, VT: MVT::i32));
3786
3787 SDValue SigSetHigh = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3788 N2: DAG.getConstant(Val: 0x1000, DL, VT: MVT::i32));
3789
3790 SDValue D = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SigSetHigh, N2: B);
3791 SDValue D0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: D, N2: B);
3792 SDValue D1 = DAG.getSelectCC(DL, LHS: D0, RHS: SigSetHigh, True: One, False: Zero, Cond: ISD::SETNE);
3793 D = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: D, N2: D1);
3794
3795 SDValue V = DAG.getSelectCC(DL, LHS: E, RHS: One, True: D, False: N, Cond: ISD::SETLT);
3796 SDValue VLow3 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: V,
3797 N2: DAG.getConstant(Val: 0x7, DL, VT: MVT::i32));
3798 V = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: V,
3799 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3800 SDValue V0 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 3, DL, VT: MVT::i32),
3801 True: One, False: Zero, Cond: ISD::SETEQ);
3802 SDValue V1 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 5, DL, VT: MVT::i32),
3803 True: One, False: Zero, Cond: ISD::SETGT);
3804 V1 = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: V0, N2: V1);
3805 V = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: V, N2: V1);
3806
3807 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 30, DL, VT: MVT::i32),
3808 True: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32), False: V, Cond: ISD::SETGT);
3809 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 1039, DL, VT: MVT::i32),
3810 True: I, False: V, Cond: ISD::SETEQ);
3811
3812 // Extract the sign bit.
3813 SDValue Sign = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3814 N2: DAG.getConstant(Val: 16, DL, VT: MVT::i32));
3815 Sign = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Sign,
3816 N2: DAG.getConstant(Val: 0x8000, DL, VT: MVT::i32));
3817
3818 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Sign, N2: V);
3819}
3820
3821SDValue AMDGPUTargetLowering::LowerFP_TO_INT(const SDValue Op,
3822 SelectionDAG &DAG) const {
3823 SDValue Src = Op.getOperand(i: 0);
3824 unsigned OpOpcode = Op.getOpcode();
3825 EVT SrcVT = Src.getValueType();
3826 EVT DestVT = Op.getValueType();
3827
3828 // Will be selected natively
3829 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3830 return Op;
3831
3832 if (SrcVT == MVT::bf16 || (SrcVT == MVT::f16 && DestVT == MVT::i32)) {
3833 SDLoc DL(Op);
3834 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3835 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DestVT, Operand: PromotedSrc);
3836 }
3837
3838 // Promote i16 to i32
3839 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3840 SDLoc DL(Op);
3841
3842 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3843 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToInt32);
3844 }
3845
3846 if (DestVT != MVT::i64)
3847 return Op;
3848
3849 if (SrcVT == MVT::f16 ||
3850 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
3851 SDLoc DL(Op);
3852
3853 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3854 unsigned Ext =
3855 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3856 return DAG.getNode(Opcode: Ext, DL, VT: MVT::i64, Operand: FpToInt32);
3857 }
3858
3859 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3860 return LowerFP_TO_INT64(Op, DAG, Signed: OpOpcode == ISD::FP_TO_SINT);
3861
3862 return SDValue();
3863}
3864
3865SDValue AMDGPUTargetLowering::LowerFP_TO_INT_SAT(const SDValue Op,
3866 SelectionDAG &DAG) const {
3867 SDValue Src = Op.getOperand(i: 0);
3868 unsigned OpOpcode = Op.getOpcode();
3869 EVT SrcVT = Src.getValueType();
3870 EVT DstVT = Op.getValueType();
3871 SDValue SatVTOp = Op.getNode()->getOperand(Num: 1);
3872 EVT SatVT = cast<VTSDNode>(Val&: SatVTOp)->getVT();
3873 SDLoc DL(Op);
3874
3875 uint64_t DstWidth = DstVT.getScalarSizeInBits();
3876 uint64_t SatWidth = SatVT.getScalarSizeInBits();
3877 assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
3878
3879 // Scalar cases will be selected natively to v_cvt_/s_cvt_ instructions.
3880 // v2f32 -> v2i16 will be selected natively to v_cvt_pk_[iu]16_f32.
3881 if (SatWidth == DstWidth) {
3882 if ((DstVT == MVT::i32 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
3883 (DstVT == MVT::i16 && (SrcVT == MVT::f16 || SrcVT == MVT::f32)) ||
3884 (DstVT == MVT::v2i16 && SrcVT == MVT::v2f32))
3885 return Op;
3886 }
3887
3888 // Vectors can only be selected natively.
3889 if (DstVT.isVector())
3890 return SDValue();
3891
3892 // Perform all saturation at selected width (i16 or i32) and truncate
3893 if (SatWidth < DstWidth && SatWidth <= 32) {
3894 // For f16 conversion with sub-i16 saturation perform saturation
3895 // at i16, if available in the target. This removes the need for extra f16
3896 // to f32 conversion. For all the others use i32.
3897 MVT ResultVT =
3898 Subtarget->has16BitInsts() && SrcVT == MVT::f16 && SatWidth < 16
3899 ? MVT::i16
3900 : MVT::i32;
3901
3902 const SDValue ResultVTOp = DAG.getValueType(ResultVT);
3903 const uint64_t ResultWidth = ResultVT.getScalarSizeInBits();
3904
3905 // First, convert input float into selected integer (i16 or i32)
3906 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: ResultVT, N1: Src, N2: ResultVTOp);
3907 SDValue IntSatVal;
3908
3909 // Then, clamp at the saturation width using either i16 or i32 instructions
3910 if (OpOpcode == ISD::FP_TO_SINT_SAT) {
3911 SDValue MinConst = DAG.getConstant(
3912 Val: APInt::getSignedMaxValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3913 SDValue MaxConst = DAG.getConstant(
3914 Val: APInt::getSignedMinValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3915 SDValue MinVal = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3916 IntSatVal = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ResultVT, N1: MinVal, N2: MaxConst);
3917 } else {
3918 SDValue MinConst = DAG.getConstant(
3919 Val: APInt::getMaxValue(numBits: SatWidth).zext(width: ResultWidth), DL, VT: ResultVT);
3920 IntSatVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3921 }
3922
3923 // Finally, after saturating at i16 or i32 fit into the destination type
3924 return DAG.getExtOrTrunc(IsSigned: OpOpcode == ISD::FP_TO_SINT_SAT, Op: IntSatVal, DL,
3925 VT: DstVT);
3926 }
3927
3928 // SatWidth == DstWidth or SatWidth > 32
3929
3930 // Saturate at i32 for i64 dst and f16/bf16 src (will invoke f16 promotion
3931 // below)
3932 if (DstVT == MVT::i64 &&
3933 (SrcVT == MVT::f16 || SrcVT == MVT::bf16 ||
3934 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP))) {
3935 const SDValue Int32VTOp = DAG.getValueType(MVT::i32);
3936 return DAG.getNode(Opcode: OpOpcode, DL, VT: DstVT, N1: Src, N2: Int32VTOp);
3937 }
3938
3939 // Promote f16/bf16 src to f32 for i32 conversion
3940 if (DstVT == MVT::i32 && (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
3941 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3942 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DstVT, N1: PromotedSrc, N2: SatVTOp);
3943 }
3944
3945 // For DstWidth < 16, promote i1 and i8 dst to i16 (if legal) with sub-i16
3946 // saturation. For DstWidth == 16, promote i16 dst to i32 with sub-i32
3947 // saturation; this covers i16.f32 and i16.f64
3948 if (DstWidth < 32) {
3949 // Note: this triggers SatWidth < DstWidth above to generate saturated
3950 // truncate by requesting MVT::i16/i32 destination with SatWidth < 16/32.
3951 MVT PromoteVT =
3952 (DstWidth < 16 && Subtarget->has16BitInsts()) ? MVT::i16 : MVT::i32;
3953 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: PromoteVT, N1: Src, N2: SatVTOp);
3954 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: FpToInt);
3955 }
3956
3957 // TODO: can we implement i64 dst for f32/f64?
3958
3959 return SDValue();
3960}
3961
3962SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
3963 SelectionDAG &DAG) const {
3964 EVT ExtraVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3965 MVT VT = Op.getSimpleValueType();
3966 MVT ScalarVT = VT.getScalarType();
3967
3968 assert(VT.isVector());
3969
3970 SDValue Src = Op.getOperand(i: 0);
3971 SDLoc DL(Op);
3972
3973 // TODO: Don't scalarize on Evergreen?
3974 unsigned NElts = VT.getVectorNumElements();
3975 SmallVector<SDValue, 8> Args;
3976 DAG.ExtractVectorElements(Op: Src, Args, Start: 0, Count: NElts);
3977
3978 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
3979 for (unsigned I = 0; I < NElts; ++I)
3980 Args[I] = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: ScalarVT, N1: Args[I], N2: VTOp);
3981
3982 return DAG.getBuildVector(VT, DL, Ops: Args);
3983}
3984
3985//===----------------------------------------------------------------------===//
3986// Custom DAG optimizations
3987//===----------------------------------------------------------------------===//
3988
3989static bool isU24(SDValue Op, SelectionDAG &DAG) {
3990 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
3991}
3992
3993static bool isI24(SDValue Op, SelectionDAG &DAG) {
3994 EVT VT = Op.getValueType();
3995 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
3996 // as unsigned 24-bit values.
3997 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24;
3998}
3999
4000static SDValue simplifyMul24(SDNode *Node24,
4001 TargetLowering::DAGCombinerInfo &DCI) {
4002 SelectionDAG &DAG = DCI.DAG;
4003 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4004 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
4005
4006 SDValue LHS = IsIntrin ? Node24->getOperand(Num: 1) : Node24->getOperand(Num: 0);
4007 SDValue RHS = IsIntrin ? Node24->getOperand(Num: 2) : Node24->getOperand(Num: 1);
4008 unsigned NewOpcode = Node24->getOpcode();
4009 if (IsIntrin) {
4010 unsigned IID = Node24->getConstantOperandVal(Num: 0);
4011 switch (IID) {
4012 case Intrinsic::amdgcn_mul_i24:
4013 NewOpcode = AMDGPUISD::MUL_I24;
4014 break;
4015 case Intrinsic::amdgcn_mul_u24:
4016 NewOpcode = AMDGPUISD::MUL_U24;
4017 break;
4018 case Intrinsic::amdgcn_mulhi_i24:
4019 NewOpcode = AMDGPUISD::MULHI_I24;
4020 break;
4021 case Intrinsic::amdgcn_mulhi_u24:
4022 NewOpcode = AMDGPUISD::MULHI_U24;
4023 break;
4024 default:
4025 llvm_unreachable("Expected 24-bit mul intrinsic");
4026 }
4027 }
4028
4029 APInt Demanded = APInt::getLowBitsSet(numBits: LHS.getValueSizeInBits(), loBitsSet: 24);
4030
4031 // First try to simplify using SimplifyMultipleUseDemandedBits which allows
4032 // the operands to have other uses, but will only perform simplifications that
4033 // involve bypassing some nodes for this user.
4034 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(Op: LHS, DemandedBits: Demanded, DAG);
4035 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(Op: RHS, DemandedBits: Demanded, DAG);
4036 if (DemandedLHS || DemandedRHS)
4037 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(Node24), VTList: Node24->getVTList(),
4038 N1: DemandedLHS ? DemandedLHS : LHS,
4039 N2: DemandedRHS ? DemandedRHS : RHS);
4040
4041 // Now try SimplifyDemandedBits which can simplify the nodes used by our
4042 // operands if this node is the only user.
4043 if (TLI.SimplifyDemandedBits(Op: LHS, DemandedBits: Demanded, DCI))
4044 return SDValue(Node24, 0);
4045 if (TLI.SimplifyDemandedBits(Op: RHS, DemandedBits: Demanded, DCI))
4046 return SDValue(Node24, 0);
4047
4048 return SDValue();
4049}
4050
4051template <typename IntTy>
4052static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
4053 uint32_t Width, const SDLoc &DL) {
4054 if (Width + Offset < 32) {
4055 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
4056 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
4057 if constexpr (std::is_signed_v<IntTy>) {
4058 return DAG.getSignedConstant(Val: Result, DL, VT: MVT::i32);
4059 } else {
4060 return DAG.getConstant(Result, DL, MVT::i32);
4061 }
4062 }
4063
4064 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
4065}
4066
4067static bool hasVolatileUser(SDNode *Val) {
4068 for (SDNode *U : Val->users()) {
4069 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: U)) {
4070 if (M->isVolatile())
4071 return true;
4072 }
4073 }
4074
4075 return false;
4076}
4077
4078bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
4079 // i32 vectors are the canonical memory type.
4080 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
4081 return false;
4082
4083 if (!VT.isByteSized())
4084 return false;
4085
4086 unsigned Size = VT.getStoreSize();
4087
4088 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
4089 return false;
4090
4091 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
4092 return false;
4093
4094 return true;
4095}
4096
4097// Replace load of an illegal type with a bitcast from a load of a friendlier
4098// type.
4099SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
4100 DAGCombinerInfo &DCI) const {
4101 if (!DCI.isBeforeLegalize())
4102 return SDValue();
4103
4104 LoadSDNode *LN = cast<LoadSDNode>(Val: N);
4105 if (!LN->isSimple() || !ISD::isNormalLoad(N: LN) || hasVolatileUser(Val: LN))
4106 return SDValue();
4107
4108 SDLoc SL(N);
4109 SelectionDAG &DAG = DCI.DAG;
4110 EVT VT = LN->getMemoryVT();
4111
4112 unsigned Size = VT.getStoreSize();
4113 Align Alignment = LN->getAlign();
4114 if (Alignment < Size && isTypeLegal(VT)) {
4115 unsigned IsFast;
4116 unsigned AS = LN->getAddressSpace();
4117
4118 // Expand unaligned loads earlier than legalization. Due to visitation order
4119 // problems during legalization, the emitted instructions to pack and unpack
4120 // the bytes again are not eliminated in the case of an unaligned copy.
4121 if (!allowsMisalignedMemoryAccesses(
4122 VT, AddrSpace: AS, Alignment, Flags: LN->getMemOperand()->getFlags(), &IsFast)) {
4123 if (VT.isVector())
4124 return SplitVectorLoad(Op: SDValue(LN, 0), DAG);
4125
4126 SDValue Ops[2];
4127 std::tie(args&: Ops[0], args&: Ops[1]) = expandUnalignedLoad(LD: LN, DAG);
4128
4129 return DAG.getMergeValues(Ops, dl: SDLoc(N));
4130 }
4131
4132 if (!IsFast)
4133 return SDValue();
4134 }
4135
4136 if (!shouldCombineMemoryType(VT))
4137 return SDValue();
4138
4139 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4140
4141 SDValue NewLoad
4142 = DAG.getLoad(VT: NewVT, dl: SL, Chain: LN->getChain(),
4143 Ptr: LN->getBasePtr(), MMO: LN->getMemOperand());
4144
4145 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewLoad);
4146 DCI.CombineTo(N, Res0: BC, Res1: NewLoad.getValue(R: 1));
4147 return SDValue(N, 0);
4148}
4149
4150// Replace store of an illegal type with a store of a bitcast to a friendlier
4151// type.
4152SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
4153 DAGCombinerInfo &DCI) const {
4154 if (!DCI.isBeforeLegalize())
4155 return SDValue();
4156
4157 StoreSDNode *SN = cast<StoreSDNode>(Val: N);
4158 if (!SN->isSimple() || !ISD::isNormalStore(N: SN))
4159 return SDValue();
4160
4161 EVT VT = SN->getMemoryVT();
4162 unsigned Size = VT.getStoreSize();
4163
4164 SDLoc SL(N);
4165 SelectionDAG &DAG = DCI.DAG;
4166 Align Alignment = SN->getAlign();
4167 if (Alignment < Size && isTypeLegal(VT)) {
4168 unsigned IsFast;
4169 unsigned AS = SN->getAddressSpace();
4170
4171 // Expand unaligned stores earlier than legalization. Due to visitation
4172 // order problems during legalization, the emitted instructions to pack and
4173 // unpack the bytes again are not eliminated in the case of an unaligned
4174 // copy.
4175 if (!allowsMisalignedMemoryAccesses(
4176 VT, AddrSpace: AS, Alignment, Flags: SN->getMemOperand()->getFlags(), &IsFast)) {
4177 if (VT.isVector())
4178 return SplitVectorStore(Op: SDValue(SN, 0), DAG);
4179
4180 return expandUnalignedStore(ST: SN, DAG);
4181 }
4182
4183 if (!IsFast)
4184 return SDValue();
4185 }
4186
4187 if (!shouldCombineMemoryType(VT))
4188 return SDValue();
4189
4190 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4191 SDValue Val = SN->getValue();
4192
4193 // DCI.AddToWorklist(Val.getNode());
4194
4195 bool OtherUses = !Val.hasOneUse();
4196 SDValue CastVal = DAG.getBitcast(VT: NewVT, V: Val);
4197 if (OtherUses) {
4198 SDValue CastBack = DAG.getBitcast(VT, V: CastVal);
4199 DAG.ReplaceAllUsesOfValueWith(From: Val, To: CastBack);
4200 }
4201
4202 return DAG.getStore(Chain: SN->getChain(), dl: SL, Val: CastVal,
4203 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
4204}
4205
4206// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
4207// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
4208// issues.
4209SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
4210 DAGCombinerInfo &DCI) const {
4211 SelectionDAG &DAG = DCI.DAG;
4212 SDValue N0 = N->getOperand(Num: 0);
4213
4214 // (vt2 (assertzext (truncate vt0:x), vt1)) ->
4215 // (vt2 (truncate (assertzext vt0:x, vt1)))
4216 if (N0.getOpcode() == ISD::TRUNCATE) {
4217 SDValue N1 = N->getOperand(Num: 1);
4218 EVT ExtVT = cast<VTSDNode>(Val&: N1)->getVT();
4219 SDLoc SL(N);
4220
4221 SDValue Src = N0.getOperand(i: 0);
4222 EVT SrcVT = Src.getValueType();
4223 if (SrcVT.bitsGE(VT: ExtVT)) {
4224 SDValue NewInReg = DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: SrcVT, N1: Src, N2: N1);
4225 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: N->getValueType(ResNo: 0), Operand: NewInReg);
4226 }
4227 }
4228
4229 return SDValue();
4230}
4231
4232SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
4233 SDNode *N, DAGCombinerInfo &DCI) const {
4234 unsigned IID = N->getConstantOperandVal(Num: 0);
4235 switch (IID) {
4236 case Intrinsic::amdgcn_mul_i24:
4237 case Intrinsic::amdgcn_mul_u24:
4238 case Intrinsic::amdgcn_mulhi_i24:
4239 case Intrinsic::amdgcn_mulhi_u24:
4240 return simplifyMul24(Node24: N, DCI);
4241 case Intrinsic::amdgcn_fract:
4242 case Intrinsic::amdgcn_rsq:
4243 case Intrinsic::amdgcn_rcp_legacy:
4244 case Intrinsic::amdgcn_rsq_legacy:
4245 case Intrinsic::amdgcn_rsq_clamp:
4246 case Intrinsic::amdgcn_tanh:
4247 case Intrinsic::amdgcn_prng_b32: {
4248 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
4249 SDValue Src = N->getOperand(Num: 1);
4250 return Src.isUndef() ? Src : SDValue();
4251 }
4252 case Intrinsic::amdgcn_frexp_exp: {
4253 // frexp_exp (fneg x) -> frexp_exp x
4254 // frexp_exp (fabs x) -> frexp_exp x
4255 // frexp_exp (fneg (fabs x)) -> frexp_exp x
4256 SDValue Src = N->getOperand(Num: 1);
4257 SDValue PeekSign = peekFPSignOps(Val: Src);
4258 if (PeekSign == Src)
4259 return SDValue();
4260 return SDValue(DCI.DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: PeekSign),
4261 0);
4262 }
4263 default:
4264 return SDValue();
4265 }
4266}
4267
4268/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
4269/// binary operation \p Opc to it with the corresponding constant operands.
4270SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
4271 DAGCombinerInfo &DCI, const SDLoc &SL,
4272 unsigned Opc, SDValue LHS,
4273 uint32_t ValLo, uint32_t ValHi) const {
4274 SelectionDAG &DAG = DCI.DAG;
4275 SDValue Lo, Hi;
4276 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: LHS, DAG);
4277
4278 SDValue LoRHS = DAG.getConstant(Val: ValLo, DL: SL, VT: MVT::i32);
4279 SDValue HiRHS = DAG.getConstant(Val: ValHi, DL: SL, VT: MVT::i32);
4280
4281 SDValue LoAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Lo, N2: LoRHS);
4282 SDValue HiAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Hi, N2: HiRHS);
4283
4284 // Re-visit the ands. It's possible we eliminated one of them and it could
4285 // simplify the vector.
4286 DCI.AddToWorklist(N: Lo.getNode());
4287 DCI.AddToWorklist(N: Hi.getNode());
4288
4289 SDValue Vec = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoAnd, HiAnd});
4290 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
4291}
4292
4293SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
4294 DAGCombinerInfo &DCI) const {
4295 EVT VT = N->getValueType(ResNo: 0);
4296 SDValue LHS = N->getOperand(Num: 0);
4297 SDValue RHS = N->getOperand(Num: 1);
4298 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4299 SDLoc SL(N);
4300 SelectionDAG &DAG = DCI.DAG;
4301
4302 unsigned RHSVal;
4303 if (CRHS) {
4304 RHSVal = CRHS->getZExtValue();
4305 if (!RHSVal)
4306 return LHS;
4307
4308 switch (LHS->getOpcode()) {
4309 default:
4310 break;
4311 case ISD::ZERO_EXTEND:
4312 case ISD::SIGN_EXTEND:
4313 case ISD::ANY_EXTEND: {
4314 SDValue X = LHS->getOperand(Num: 0);
4315
4316 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
4317 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16)) {
4318 // Prefer build_vector as the canonical form if packed types are legal.
4319 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
4320 SDValue Vec = DAG.getBuildVector(
4321 VT: MVT::v2i16, DL: SL,
4322 Ops: {DAG.getConstant(Val: 0, DL: SL, VT: MVT::i16), LHS->getOperand(Num: 0)});
4323 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Vec);
4324 }
4325
4326 // shl (ext x) => zext (shl x), if shift does not overflow int
4327 if (VT != MVT::i64)
4328 break;
4329 KnownBits Known = DAG.computeKnownBits(Op: X);
4330 unsigned LZ = Known.countMinLeadingZeros();
4331 if (LZ < RHSVal)
4332 break;
4333 EVT XVT = X.getValueType();
4334 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: XVT, N1: X, N2: SDValue(CRHS, 0));
4335 return DAG.getZExtOrTrunc(Op: Shl, DL: SL, VT);
4336 }
4337 }
4338 }
4339
4340 if (VT.getScalarType() != MVT::i64)
4341 return SDValue();
4342
4343 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4344 // common case, splitting this into a move and a 32-bit shift is faster and
4345 // the same code size.
4346 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4347
4348 EVT ElementType = VT.getScalarType();
4349 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4350 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4351
4352 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4353 return SDValue();
4354 SDValue ShiftAmt;
4355
4356 if (CRHS) {
4357 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4358 VT: TargetType);
4359 } else {
4360 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4361 const SDValue ShiftMask =
4362 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4363 // This AND instruction will clamp out of bounds shift values.
4364 // It will also be removed during later instruction selection.
4365 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4366 }
4367
4368 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: LHS);
4369 SDValue NewShift =
4370 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: TargetType, N1: Lo, N2: ShiftAmt, Flags: N->getFlags());
4371
4372 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4373 SDValue Vec;
4374
4375 if (VT.isVector()) {
4376 EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4377 unsigned NElts = TargetType.getVectorNumElements();
4378 SmallVector<SDValue, 8> HiOps;
4379 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4380
4381 DAG.ExtractVectorElements(Op: NewShift, Args&: HiOps, Start: 0, Count: NElts);
4382 for (unsigned I = 0; I != NElts; ++I)
4383 HiAndLoOps[2 * I + 1] = HiOps[I];
4384 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4385 } else {
4386 EVT ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4387 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {Zero, NewShift});
4388 }
4389 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4390}
4391
4392SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
4393 DAGCombinerInfo &DCI) const {
4394 SDValue RHS = N->getOperand(Num: 1);
4395 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4396 EVT VT = N->getValueType(ResNo: 0);
4397 SDValue LHS = N->getOperand(Num: 0);
4398 SelectionDAG &DAG = DCI.DAG;
4399 SDLoc SL(N);
4400
4401 if (VT.getScalarType() != MVT::i64)
4402 return SDValue();
4403
4404 // For C >= 32
4405 // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
4406
4407 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4408 // common case, splitting this into a move and a 32-bit shift is faster and
4409 // the same code size.
4410 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4411
4412 EVT ElementType = VT.getScalarType();
4413 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4414 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4415
4416 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4417 return SDValue();
4418
4419 SDValue ShiftFullAmt =
4420 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4421 SDValue ShiftAmt;
4422 if (CRHS) {
4423 unsigned RHSVal = CRHS->getZExtValue();
4424 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4425 VT: TargetType);
4426 } else if (Known.getMinValue().getZExtValue() ==
4427 (ElementType.getSizeInBits() - 1)) {
4428 ShiftAmt = ShiftFullAmt;
4429 } else {
4430 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4431 const SDValue ShiftMask =
4432 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4433 // This AND instruction will clamp out of bounds shift values.
4434 // It will also be removed during later instruction selection.
4435 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4436 }
4437
4438 EVT ConcatType;
4439 SDValue Hi;
4440 SDLoc LHSSL(LHS);
4441 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4442 if (VT.isVector()) {
4443 unsigned NElts = TargetType.getVectorNumElements();
4444 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4445 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4446 SmallVector<SDValue, 8> HiOps(NElts);
4447 SmallVector<SDValue, 16> HiAndLoOps;
4448
4449 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, Start: 0, Count: NElts * 2);
4450 for (unsigned I = 0; I != NElts; ++I) {
4451 HiOps[I] = HiAndLoOps[2 * I + 1];
4452 }
4453 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4454 } else {
4455 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4456 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4457 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4458 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4459 }
4460
4461 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
4462 SDValue NewShift, HiShift;
4463 if (KnownLHS.isNegative()) {
4464 HiShift = DAG.getAllOnesConstant(DL: SL, VT: TargetType);
4465 NewShift =
4466 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4467 } else if (CRHS &&
4468 CRHS->getZExtValue() == (ElementType.getSizeInBits() - 1)) {
4469 NewShift = HiShift =
4470 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4471 } else {
4472 Hi = DAG.getFreeze(V: Hi);
4473 HiShift = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftFullAmt);
4474 NewShift =
4475 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4476 }
4477
4478 SDValue Vec;
4479 if (VT.isVector()) {
4480 unsigned NElts = TargetType.getVectorNumElements();
4481 SmallVector<SDValue, 8> HiOps;
4482 SmallVector<SDValue, 8> LoOps;
4483 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
4484
4485 DAG.ExtractVectorElements(Op: HiShift, Args&: HiOps, Start: 0, Count: NElts);
4486 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4487 for (unsigned I = 0; I != NElts; ++I) {
4488 HiAndLoOps[2 * I + 1] = HiOps[I];
4489 HiAndLoOps[2 * I] = LoOps[I];
4490 }
4491 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4492 } else {
4493 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, HiShift});
4494 }
4495 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4496}
4497
4498SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
4499 DAGCombinerInfo &DCI) const {
4500 SDValue RHS = N->getOperand(Num: 1);
4501 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4502 EVT VT = N->getValueType(ResNo: 0);
4503 SDValue LHS = N->getOperand(Num: 0);
4504 SelectionDAG &DAG = DCI.DAG;
4505 SDLoc SL(N);
4506 unsigned RHSVal;
4507
4508 if (CRHS) {
4509 RHSVal = CRHS->getZExtValue();
4510
4511 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
4512 // this improves the ability to match BFE patterns in isel.
4513 if (LHS.getOpcode() == ISD::AND) {
4514 if (auto *Mask = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
4515 unsigned MaskIdx, MaskLen;
4516 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4517 MaskIdx == RHSVal) {
4518 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT,
4519 N1: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 0),
4520 N2: N->getOperand(Num: 1)),
4521 N2: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 1),
4522 N2: N->getOperand(Num: 1)));
4523 }
4524 }
4525 }
4526 }
4527
4528 if (VT.getScalarType() != MVT::i64)
4529 return SDValue();
4530
4531 // for C >= 32
4532 // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
4533
4534 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4535 // common case, splitting this into a move and a 32-bit shift is faster and
4536 // the same code size.
4537 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4538
4539 EVT ElementType = VT.getScalarType();
4540 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4541 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4542
4543 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4544 return SDValue();
4545
4546 SDValue ShiftAmt;
4547 if (CRHS) {
4548 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4549 VT: TargetType);
4550 } else {
4551 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4552 const SDValue ShiftMask =
4553 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4554 // This AND instruction will clamp out of bounds shift values.
4555 // It will also be removed during later instruction selection.
4556 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4557 }
4558
4559 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4560 EVT ConcatType;
4561 SDValue Hi;
4562 SDLoc LHSSL(LHS);
4563 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4564 if (VT.isVector()) {
4565 unsigned NElts = TargetType.getVectorNumElements();
4566 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4567 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4568 SmallVector<SDValue, 8> HiOps(NElts);
4569 SmallVector<SDValue, 16> HiAndLoOps;
4570
4571 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, /*Start=*/0, Count: NElts * 2);
4572 for (unsigned I = 0; I != NElts; ++I)
4573 HiOps[I] = HiAndLoOps[2 * I + 1];
4574 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4575 } else {
4576 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4577 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4578 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4579 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4580 }
4581
4582 SDValue NewShift =
4583 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4584
4585 SDValue Vec;
4586 if (VT.isVector()) {
4587 unsigned NElts = TargetType.getVectorNumElements();
4588 SmallVector<SDValue, 8> LoOps;
4589 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4590
4591 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4592 for (unsigned I = 0; I != NElts; ++I)
4593 HiAndLoOps[2 * I] = LoOps[I];
4594 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4595 } else {
4596 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, Zero});
4597 }
4598 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4599}
4600
4601SDValue AMDGPUTargetLowering::performTruncateCombine(
4602 SDNode *N, DAGCombinerInfo &DCI) const {
4603 SDLoc SL(N);
4604 SelectionDAG &DAG = DCI.DAG;
4605 EVT VT = N->getValueType(ResNo: 0);
4606 SDValue Src = N->getOperand(Num: 0);
4607
4608 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
4609 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
4610 SDValue Vec = Src.getOperand(i: 0);
4611 if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
4612 SDValue Elt0 = Vec.getOperand(i: 0);
4613 EVT EltVT = Elt0.getValueType();
4614 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
4615 if (EltVT.isFloatingPoint()) {
4616 Elt0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL,
4617 VT: EltVT.changeTypeToInteger(), Operand: Elt0);
4618 }
4619
4620 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Elt0);
4621 }
4622 }
4623 }
4624
4625 // Equivalent of above for accessing the high element of a vector as an
4626 // integer operation.
4627 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
4628 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
4629 if (auto *K = isConstOrConstSplat(N: Src.getOperand(i: 1))) {
4630 SDValue BV = stripBitcast(Val: Src.getOperand(i: 0));
4631 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
4632 EVT SrcEltVT = BV.getOperand(i: 0).getValueType();
4633 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
4634 unsigned BitIndex = K->getZExtValue();
4635 unsigned PartIndex = BitIndex / SrcEltSize;
4636
4637 if (PartIndex * SrcEltSize == BitIndex &&
4638 PartIndex < BV.getNumOperands()) {
4639 if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
4640 SDValue SrcElt =
4641 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcEltVT.changeTypeToInteger(),
4642 Operand: BV.getOperand(i: PartIndex));
4643 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: SrcElt);
4644 }
4645 }
4646 }
4647 }
4648 }
4649
4650 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
4651 //
4652 // i16 (trunc (srl i64:x, K)), K <= 16 ->
4653 // i16 (trunc (srl (i32 (trunc x), K)))
4654 if (VT.getScalarSizeInBits() < 32) {
4655 EVT SrcVT = Src.getValueType();
4656 if (SrcVT.getScalarSizeInBits() > 32 &&
4657 (Src.getOpcode() == ISD::SRL ||
4658 Src.getOpcode() == ISD::SRA ||
4659 Src.getOpcode() == ISD::SHL)) {
4660 SDValue Amt = Src.getOperand(i: 1);
4661 KnownBits Known = DAG.computeKnownBits(Op: Amt);
4662
4663 // - For left shifts, do the transform as long as the shift
4664 // amount is still legal for i32, so when ShiftAmt < 32 (<= 31)
4665 // - For right shift, do it if ShiftAmt <= (32 - Size) to avoid
4666 // losing information stored in the high bits when truncating.
4667 const unsigned MaxCstSize =
4668 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
4669 if (Known.getMaxValue().ule(RHS: MaxCstSize)) {
4670 EVT MidVT = VT.isVector() ?
4671 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
4672 NumElements: VT.getVectorNumElements()) : MVT::i32;
4673
4674 EVT NewShiftVT = getShiftAmountTy(LHSTy: MidVT, DL: DAG.getDataLayout());
4675 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MidVT,
4676 Operand: Src.getOperand(i: 0));
4677 DCI.AddToWorklist(N: Trunc.getNode());
4678
4679 if (Amt.getValueType() != NewShiftVT) {
4680 Amt = DAG.getZExtOrTrunc(Op: Amt, DL: SL, VT: NewShiftVT);
4681 DCI.AddToWorklist(N: Amt.getNode());
4682 }
4683
4684 SDValue ShrunkShift = DAG.getNode(Opcode: Src.getOpcode(), DL: SL, VT: MidVT,
4685 N1: Trunc, N2: Amt);
4686 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: ShrunkShift);
4687 }
4688 }
4689 }
4690
4691 return SDValue();
4692}
4693
4694// We need to specifically handle i64 mul here to avoid unnecessary conversion
4695// instructions. If we only match on the legalized i64 mul expansion,
4696// SimplifyDemandedBits will be unable to remove them because there will be
4697// multiple uses due to the separate mul + mulh[su].
4698static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
4699 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
4700 if (Size <= 32) {
4701 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4702 return DAG.getNode(Opcode: MulOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4703 }
4704
4705 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4706 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
4707
4708 SDValue MulLo = DAG.getNode(Opcode: MulLoOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4709 SDValue MulHi = DAG.getNode(Opcode: MulHiOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4710
4711 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: MulLo, N2: MulHi);
4712}
4713
4714/// If \p V is an add of a constant 1, returns the other operand. Otherwise
4715/// return SDValue().
4716static SDValue getAddOneOp(const SDNode *V) {
4717 if (V->getOpcode() != ISD::ADD)
4718 return SDValue();
4719
4720 return isOneConstant(V: V->getOperand(Num: 1)) ? V->getOperand(Num: 0) : SDValue();
4721}
4722
4723SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
4724 DAGCombinerInfo &DCI) const {
4725 assert(N->getOpcode() == ISD::MUL);
4726 EVT VT = N->getValueType(ResNo: 0);
4727
4728 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4729 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4730 // unnecessarily). isDivergent() is used as an approximation of whether the
4731 // value is in an SGPR.
4732 if (!N->isDivergent())
4733 return SDValue();
4734
4735 unsigned Size = VT.getSizeInBits();
4736 if (VT.isVector() || Size > 64)
4737 return SDValue();
4738
4739 SelectionDAG &DAG = DCI.DAG;
4740 SDLoc DL(N);
4741
4742 SDValue N0 = N->getOperand(Num: 0);
4743 SDValue N1 = N->getOperand(Num: 1);
4744
4745 // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
4746 // matching.
4747
4748 // mul x, (add y, 1) -> add (mul x, y), x
4749 auto IsFoldableAdd = [](SDValue V) -> SDValue {
4750 SDValue AddOp = getAddOneOp(V: V.getNode());
4751 if (!AddOp)
4752 return SDValue();
4753
4754 if (V.hasOneUse() || all_of(Range: V->users(), P: [](const SDNode *U) -> bool {
4755 return U->getOpcode() == ISD::MUL;
4756 }))
4757 return AddOp;
4758
4759 return SDValue();
4760 };
4761
4762 // FIXME: The selection pattern is not properly checking for commuted
4763 // operands, so we have to place the mul in the LHS
4764 if (SDValue MulOper = IsFoldableAdd(N0)) {
4765 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1, N2: MulOper);
4766 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N1);
4767 }
4768
4769 if (SDValue MulOper = IsFoldableAdd(N1)) {
4770 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N0, N2: MulOper);
4771 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N0);
4772 }
4773
4774 // There are i16 integer mul/mad.
4775 if (isTypeLegal(VT: MVT::i16) && VT.getScalarType().bitsLE(VT: MVT::i16))
4776 return SDValue();
4777
4778 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4779 // in the source into any_extends if the result of the mul is truncated. Since
4780 // we can assume the high bits are whatever we want, use the underlying value
4781 // to avoid the unknown high bits from interfering.
4782 if (N0.getOpcode() == ISD::ANY_EXTEND)
4783 N0 = N0.getOperand(i: 0);
4784
4785 if (N1.getOpcode() == ISD::ANY_EXTEND)
4786 N1 = N1.getOperand(i: 0);
4787
4788 SDValue Mul;
4789
4790 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4791 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4792 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4793 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: false);
4794 } else if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4795 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4796 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4797 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: true);
4798 } else {
4799 return SDValue();
4800 }
4801
4802 // We need to use sext even for MUL_U24, because MUL_U24 is used
4803 // for signed multiply of 8 and 16-bit types.
4804 return DAG.getSExtOrTrunc(Op: Mul, DL, VT);
4805}
4806
4807SDValue
4808AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N,
4809 DAGCombinerInfo &DCI) const {
4810 if (N->getValueType(ResNo: 0) != MVT::i32)
4811 return SDValue();
4812
4813 SelectionDAG &DAG = DCI.DAG;
4814 SDLoc DL(N);
4815
4816 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
4817 SDValue N0 = N->getOperand(Num: 0);
4818 SDValue N1 = N->getOperand(Num: 1);
4819
4820 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4821 // in the source into any_extends if the result of the mul is truncated. Since
4822 // we can assume the high bits are whatever we want, use the underlying value
4823 // to avoid the unknown high bits from interfering.
4824 if (N0.getOpcode() == ISD::ANY_EXTEND)
4825 N0 = N0.getOperand(i: 0);
4826 if (N1.getOpcode() == ISD::ANY_EXTEND)
4827 N1 = N1.getOperand(i: 0);
4828
4829 // Try to use two fast 24-bit multiplies (one for each half of the result)
4830 // instead of one slow extending multiply.
4831 unsigned LoOpcode = 0;
4832 unsigned HiOpcode = 0;
4833 if (Signed) {
4834 if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4835 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4836 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4837 LoOpcode = AMDGPUISD::MUL_I24;
4838 HiOpcode = AMDGPUISD::MULHI_I24;
4839 }
4840 } else {
4841 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4842 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4843 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4844 LoOpcode = AMDGPUISD::MUL_U24;
4845 HiOpcode = AMDGPUISD::MULHI_U24;
4846 }
4847 }
4848 if (!LoOpcode)
4849 return SDValue();
4850
4851 SDValue Lo = DAG.getNode(Opcode: LoOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4852 SDValue Hi = DAG.getNode(Opcode: HiOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4853 DCI.CombineTo(N, Res0: Lo, Res1: Hi);
4854 return SDValue(N, 0);
4855}
4856
4857SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
4858 DAGCombinerInfo &DCI) const {
4859 EVT VT = N->getValueType(ResNo: 0);
4860
4861 if (!Subtarget->hasMulI24() || VT.isVector())
4862 return SDValue();
4863
4864 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4865 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4866 // unnecessarily). isDivergent() is used as an approximation of whether the
4867 // value is in an SGPR.
4868 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4869 // valu op anyway)
4870 if (Subtarget->hasSMulHi() && !N->isDivergent())
4871 return SDValue();
4872
4873 SelectionDAG &DAG = DCI.DAG;
4874 SDLoc DL(N);
4875
4876 SDValue N0 = N->getOperand(Num: 0);
4877 SDValue N1 = N->getOperand(Num: 1);
4878
4879 if (!isI24(Op: N0, DAG) || !isI24(Op: N1, DAG))
4880 return SDValue();
4881
4882 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4883 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4884
4885 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_I24, DL, VT: MVT::i32, N1: N0, N2: N1);
4886 DCI.AddToWorklist(N: Mulhi.getNode());
4887 return DAG.getSExtOrTrunc(Op: Mulhi, DL, VT);
4888}
4889
4890SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
4891 DAGCombinerInfo &DCI) const {
4892 EVT VT = N->getValueType(ResNo: 0);
4893
4894 if (VT.isVector() || VT.getSizeInBits() > 32 || !Subtarget->hasMulU24())
4895 return SDValue();
4896
4897 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4898 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4899 // unnecessarily). isDivergent() is used as an approximation of whether the
4900 // value is in an SGPR.
4901 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4902 // valu op anyway)
4903 if (!N->isDivergent() && Subtarget->hasSMulHi())
4904 return SDValue();
4905
4906 SelectionDAG &DAG = DCI.DAG;
4907 SDLoc DL(N);
4908
4909 SDValue N0 = N->getOperand(Num: 0);
4910 SDValue N1 = N->getOperand(Num: 1);
4911
4912 if (!isU24(Op: N0, DAG) || !isU24(Op: N1, DAG))
4913 return SDValue();
4914
4915 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4916 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4917
4918 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_U24, DL, VT: MVT::i32, N1: N0, N2: N1);
4919 DCI.AddToWorklist(N: Mulhi.getNode());
4920 return DAG.getZExtOrTrunc(Op: Mulhi, DL, VT);
4921}
4922
4923SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
4924 SDValue Op,
4925 const SDLoc &DL,
4926 unsigned Opc) const {
4927 EVT VT = Op.getValueType();
4928 if (VT.bitsGT(VT: MVT::i32))
4929 return SDValue();
4930
4931 if (VT != MVT::i32)
4932 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: Op);
4933
4934 SDValue FFBX = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, Operand: Op);
4935 if (VT != MVT::i32)
4936 FFBX = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FFBX);
4937
4938 return FFBX;
4939}
4940
4941// The native instructions return -1 on 0 input. Optimize out a select that
4942// produces -1 on 0.
4943//
4944// TODO: If zero is not undef, we could also do this if the output is compared
4945// against the bitwidth.
4946//
4947// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
4948SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
4949 SDValue LHS, SDValue RHS,
4950 DAGCombinerInfo &DCI) const {
4951 if (!isNullConstant(V: Cond.getOperand(i: 1)))
4952 return SDValue();
4953
4954 SelectionDAG &DAG = DCI.DAG;
4955 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
4956 SDValue CmpLHS = Cond.getOperand(i: 0);
4957
4958 // select (setcc x, 0, eq), -1, (ctlz_zero_poison x) -> ffbh_u32 x
4959 // select (setcc x, 0, eq), -1, (cttz_zero_poison x) -> ffbl_u32 x
4960 if (CCOpcode == ISD::SETEQ &&
4961 (isCtlzOpc(Opc: RHS.getOpcode()) || isCttzOpc(Opc: RHS.getOpcode())) &&
4962 RHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: LHS)) {
4963 unsigned Opc =
4964 isCttzOpc(Opc: RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4965 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4966 }
4967
4968 // select (setcc x, 0, ne), (ctlz_zero_poison x), -1 -> ffbh_u32 x
4969 // select (setcc x, 0, ne), (cttz_zero_poison x), -1 -> ffbl_u32 x
4970 if (CCOpcode == ISD::SETNE &&
4971 (isCtlzOpc(Opc: LHS.getOpcode()) || isCttzOpc(Opc: LHS.getOpcode())) &&
4972 LHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: RHS)) {
4973 unsigned Opc =
4974 isCttzOpc(Opc: LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4975
4976 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4977 }
4978
4979 return SDValue();
4980}
4981
4982static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
4983 unsigned Op,
4984 const SDLoc &SL,
4985 SDValue Cond,
4986 SDValue N1,
4987 SDValue N2) {
4988 SelectionDAG &DAG = DCI.DAG;
4989 EVT VT = N1.getValueType();
4990
4991 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cond,
4992 N2: N1.getOperand(i: 0), N3: N2.getOperand(i: 0));
4993 DCI.AddToWorklist(N: NewSelect.getNode());
4994 return DAG.getNode(Opcode: Op, DL: SL, VT, Operand: NewSelect);
4995}
4996
4997// Pull a free FP operation out of a select so it may fold into uses.
4998//
4999// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
5000// select c, (fneg x), k -> fneg (select c, x, (fneg k))
5001//
5002// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
5003// select c, (fabs x), +k -> fabs (select c, x, k)
5004SDValue
5005AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
5006 SDValue N) const {
5007 SelectionDAG &DAG = DCI.DAG;
5008 SDValue Cond = N.getOperand(i: 0);
5009 SDValue LHS = N.getOperand(i: 1);
5010 SDValue RHS = N.getOperand(i: 2);
5011
5012 EVT VT = N.getValueType();
5013 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
5014 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
5015 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5016 return SDValue();
5017
5018 return distributeOpThroughSelect(DCI, Op: LHS.getOpcode(),
5019 SL: SDLoc(N), Cond, N1: LHS, N2: RHS);
5020 }
5021
5022 bool Inv = false;
5023 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
5024 std::swap(a&: LHS, b&: RHS);
5025 Inv = true;
5026 }
5027
5028 // TODO: Support vector constants.
5029 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
5030 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS &&
5031 !selectSupportsSourceMods(N: N.getNode())) {
5032 SDLoc SL(N);
5033 // If one side is an fneg/fabs and the other is a constant, we can push the
5034 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
5035 SDValue NewLHS = LHS.getOperand(i: 0);
5036 SDValue NewRHS = RHS;
5037
5038 // Careful: if the neg can be folded up, don't try to pull it back down.
5039 bool ShouldFoldNeg = true;
5040
5041 if (NewLHS.hasOneUse()) {
5042 unsigned Opc = NewLHS.getOpcode();
5043 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(N: NewLHS.getNode()))
5044 ShouldFoldNeg = false;
5045 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
5046 ShouldFoldNeg = false;
5047 }
5048
5049 if (ShouldFoldNeg) {
5050 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative())
5051 return SDValue();
5052
5053 // We're going to be forced to use a source modifier anyway, there's no
5054 // point to pulling the negate out unless we can get a size reduction by
5055 // negating the constant.
5056 //
5057 // TODO: Generalize to use getCheaperNegatedExpression which doesn't know
5058 // about cheaper constants.
5059 if (NewLHS.getOpcode() == ISD::FABS &&
5060 getConstantNegateCost(C: CRHS) != NegatibleCost::Cheaper)
5061 return SDValue();
5062
5063 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5064 return SDValue();
5065
5066 if (LHS.getOpcode() == ISD::FNEG)
5067 NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5068
5069 if (Inv)
5070 std::swap(a&: NewLHS, b&: NewRHS);
5071
5072 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT,
5073 N1: Cond, N2: NewLHS, N3: NewRHS);
5074 DCI.AddToWorklist(N: NewSelect.getNode());
5075 return DAG.getNode(Opcode: LHS.getOpcode(), DL: SL, VT, Operand: NewSelect);
5076 }
5077 }
5078
5079 return SDValue();
5080}
5081
5082SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
5083 DAGCombinerInfo &DCI) const {
5084 if (SDValue Folded = foldFreeOpFromSelect(DCI, N: SDValue(N, 0)))
5085 return Folded;
5086
5087 SDValue Cond = N->getOperand(Num: 0);
5088 if (Cond.getOpcode() != ISD::SETCC)
5089 return SDValue();
5090
5091 EVT VT = N->getValueType(ResNo: 0);
5092 SDValue LHS = Cond.getOperand(i: 0);
5093 SDValue RHS = Cond.getOperand(i: 1);
5094 SDValue CC = Cond.getOperand(i: 2);
5095
5096 SDValue True = N->getOperand(Num: 1);
5097 SDValue False = N->getOperand(Num: 2);
5098
5099 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
5100 SelectionDAG &DAG = DCI.DAG;
5101 if (DAG.isConstantValueOfAnyType(N: True) &&
5102 !DAG.isConstantValueOfAnyType(N: False)) {
5103 // Swap cmp + select pair to move constant to false input.
5104 // This will allow using VOPC cndmasks more often.
5105 // select (setcc x, y), k, x -> select (setccinv x, y), x, k
5106
5107 SDLoc SL(N);
5108 ISD::CondCode NewCC =
5109 getSetCCInverse(Operation: cast<CondCodeSDNode>(Val&: CC)->get(), Type: LHS.getValueType());
5110
5111 SDValue NewCond = DAG.getSetCC(DL: SL, VT: Cond.getValueType(), LHS, RHS, Cond: NewCC);
5112 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NewCond, N2: False, N3: True);
5113 }
5114
5115 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
5116 SDValue MinMax
5117 = combineFMinMaxLegacy(DL: SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
5118 // Revisit this node so we can catch min3/max3/med3 patterns.
5119 //DCI.AddToWorklist(MinMax.getNode());
5120 return MinMax;
5121 }
5122 }
5123
5124 // There's no reason to not do this if the condition has other uses.
5125 return performCtlz_CttzCombine(SL: SDLoc(N), Cond, LHS: True, RHS: False, DCI);
5126}
5127
5128static bool isInv2Pi(const APFloat &APF) {
5129 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
5130 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
5131 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
5132
5133 return APF.bitwiseIsEqual(RHS: KF16) ||
5134 APF.bitwiseIsEqual(RHS: KF32) ||
5135 APF.bitwiseIsEqual(RHS: KF64);
5136}
5137
5138// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
5139// additional cost to negate them.
5140TargetLowering::NegatibleCost
5141AMDGPUTargetLowering::getConstantNegateCost(const ConstantFPSDNode *C) const {
5142 if (C->isZero())
5143 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5144
5145 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(APF: C->getValueAPF()))
5146 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5147
5148 return NegatibleCost::Neutral;
5149}
5150
5151bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
5152 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5153 return getConstantNegateCost(C) == NegatibleCost::Expensive;
5154 return false;
5155}
5156
5157bool AMDGPUTargetLowering::isConstantCheaperToNegate(SDValue N) const {
5158 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5159 return getConstantNegateCost(C) == NegatibleCost::Cheaper;
5160 return false;
5161}
5162
5163static unsigned inverseMinMax(unsigned Opc) {
5164 switch (Opc) {
5165 case ISD::FMAXNUM:
5166 return ISD::FMINNUM;
5167 case ISD::FMINNUM:
5168 return ISD::FMAXNUM;
5169 case ISD::FMAXNUM_IEEE:
5170 return ISD::FMINNUM_IEEE;
5171 case ISD::FMINNUM_IEEE:
5172 return ISD::FMAXNUM_IEEE;
5173 case ISD::FMAXIMUM:
5174 return ISD::FMINIMUM;
5175 case ISD::FMINIMUM:
5176 return ISD::FMAXIMUM;
5177 case ISD::FMAXIMUMNUM:
5178 return ISD::FMINIMUMNUM;
5179 case ISD::FMINIMUMNUM:
5180 return ISD::FMAXIMUMNUM;
5181 case AMDGPUISD::FMAX_LEGACY:
5182 return AMDGPUISD::FMIN_LEGACY;
5183 case AMDGPUISD::FMIN_LEGACY:
5184 return AMDGPUISD::FMAX_LEGACY;
5185 default:
5186 llvm_unreachable("invalid min/max opcode");
5187 }
5188}
5189
5190/// \return true if it's profitable to try to push an fneg into its source
5191/// instruction.
5192bool AMDGPUTargetLowering::shouldFoldFNegIntoSrc(SDNode *N, SDValue N0) {
5193 // If the input has multiple uses and we can either fold the negate down, or
5194 // the other uses cannot, give up. This both prevents unprofitable
5195 // transformations and infinite loops: we won't repeatedly try to fold around
5196 // a negate that has no 'good' form.
5197 if (N0.hasOneUse()) {
5198 // This may be able to fold into the source, but at a code size cost. Don't
5199 // fold if the fold into the user is free.
5200 if (allUsesHaveSourceMods(N, CostThreshold: 0))
5201 return false;
5202 } else {
5203 if (fnegFoldsIntoOp(N: N0.getNode()) &&
5204 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N: N0.getNode())))
5205 return false;
5206 }
5207
5208 return true;
5209}
5210
5211SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
5212 DAGCombinerInfo &DCI) const {
5213 SelectionDAG &DAG = DCI.DAG;
5214 SDValue N0 = N->getOperand(Num: 0);
5215 EVT VT = N->getValueType(ResNo: 0);
5216
5217 unsigned Opc = N0.getOpcode();
5218
5219 if (!shouldFoldFNegIntoSrc(N, N0))
5220 return SDValue();
5221
5222 SDLoc SL(N);
5223 switch (Opc) {
5224 case ISD::FADD: {
5225 if (!N0->getFlags().hasNoSignedZeros() && !N->getFlags().hasNoSignedZeros())
5226 return SDValue();
5227
5228 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
5229 SDValue LHS = N0.getOperand(i: 0);
5230 SDValue RHS = N0.getOperand(i: 1);
5231
5232 if (LHS.getOpcode() != ISD::FNEG)
5233 LHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5234 else
5235 LHS = LHS.getOperand(i: 0);
5236
5237 if (RHS.getOpcode() != ISD::FNEG)
5238 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5239 else
5240 RHS = RHS.getOperand(i: 0);
5241
5242 SDValue Res = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5243 if (Res.getOpcode() != ISD::FADD)
5244 return SDValue(); // Op got folded away.
5245 if (!N0.hasOneUse())
5246 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5247 return Res;
5248 }
5249 case ISD::FMUL:
5250 case AMDGPUISD::FMUL_LEGACY: {
5251 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
5252 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
5253 SDValue LHS = N0.getOperand(i: 0);
5254 SDValue RHS = N0.getOperand(i: 1);
5255
5256 if (LHS.getOpcode() == ISD::FNEG)
5257 LHS = LHS.getOperand(i: 0);
5258 else if (RHS.getOpcode() == ISD::FNEG)
5259 RHS = RHS.getOperand(i: 0);
5260 else
5261 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5262
5263 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5264 if (Res.getOpcode() != Opc)
5265 return SDValue(); // Op got folded away.
5266 if (!N0.hasOneUse())
5267 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5268 return Res;
5269 }
5270 case ISD::FMA:
5271 case ISD::FMAD: {
5272 // TODO: handle llvm.amdgcn.fma.legacy
5273 if (!N0->getFlags().hasNoSignedZeros() && !N->getFlags().hasNoSignedZeros())
5274 return SDValue();
5275
5276 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
5277 SDValue LHS = N0.getOperand(i: 0);
5278 SDValue MHS = N0.getOperand(i: 1);
5279 SDValue RHS = N0.getOperand(i: 2);
5280
5281 if (LHS.getOpcode() == ISD::FNEG)
5282 LHS = LHS.getOperand(i: 0);
5283 else if (MHS.getOpcode() == ISD::FNEG)
5284 MHS = MHS.getOperand(i: 0);
5285 else
5286 MHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: MHS);
5287
5288 if (RHS.getOpcode() != ISD::FNEG)
5289 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5290 else
5291 RHS = RHS.getOperand(i: 0);
5292
5293 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: MHS, N3: RHS);
5294 if (Res.getOpcode() != Opc)
5295 return SDValue(); // Op got folded away.
5296 if (!N0.hasOneUse())
5297 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5298 return Res;
5299 }
5300 case ISD::FMAXNUM:
5301 case ISD::FMINNUM:
5302 case ISD::FMAXNUM_IEEE:
5303 case ISD::FMINNUM_IEEE:
5304 case ISD::FMINIMUM:
5305 case ISD::FMAXIMUM:
5306 case ISD::FMINIMUMNUM:
5307 case ISD::FMAXIMUMNUM:
5308 case AMDGPUISD::FMAX_LEGACY:
5309 case AMDGPUISD::FMIN_LEGACY: {
5310 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
5311 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
5312 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
5313 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
5314
5315 SDValue LHS = N0.getOperand(i: 0);
5316 SDValue RHS = N0.getOperand(i: 1);
5317
5318 // 0 doesn't have a negated inline immediate.
5319 // TODO: This constant check should be generalized to other operations.
5320 if (isConstantCostlierToNegate(N: RHS))
5321 return SDValue();
5322
5323 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5324 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5325 unsigned Opposite = inverseMinMax(Opc);
5326
5327 SDValue Res = DAG.getNode(Opcode: Opposite, DL: SL, VT, N1: NegLHS, N2: NegRHS, Flags: N0->getFlags());
5328 if (Res.getOpcode() != Opposite)
5329 return SDValue(); // Op got folded away.
5330 if (!N0.hasOneUse())
5331 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5332 return Res;
5333 }
5334 case AMDGPUISD::FMED3: {
5335 // med3 sorts a NaN input as smaller than everything regardless of its sign,
5336 // so negating all operands does not sign-flip the median when an input may
5337 // be NaN.
5338 if (!N0->getFlags().hasNoNaNs())
5339 return SDValue();
5340
5341 SDValue Ops[3];
5342 for (unsigned I = 0; I < 3; ++I)
5343 Ops[I] = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N0->getOperand(Num: I), Flags: N0->getFlags());
5344
5345 SDValue Res = DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT, Ops, Flags: N0->getFlags());
5346 if (Res.getOpcode() != AMDGPUISD::FMED3)
5347 return SDValue(); // Op got folded away.
5348
5349 if (!N0.hasOneUse()) {
5350 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res);
5351 DAG.ReplaceAllUsesWith(From: N0, To: Neg);
5352
5353 for (SDNode *U : Neg->users())
5354 DCI.AddToWorklist(N: U);
5355 }
5356
5357 return Res;
5358 }
5359 case ISD::FP_EXTEND:
5360 case ISD::FTRUNC:
5361 case ISD::FRINT:
5362 case ISD::FNEARBYINT: // XXX - Should fround be handled?
5363 case ISD::FROUNDEVEN:
5364 case ISD::FSIN:
5365 case ISD::FCANONICALIZE:
5366 case AMDGPUISD::RCP:
5367 case AMDGPUISD::RCP_LEGACY:
5368 case AMDGPUISD::RCP_IFLAG:
5369 case AMDGPUISD::SIN_HW: {
5370 SDValue CvtSrc = N0.getOperand(i: 0);
5371 if (CvtSrc.getOpcode() == ISD::FNEG) {
5372 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
5373 // (fneg (rcp (fneg x))) -> (rcp x)
5374 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: CvtSrc.getOperand(i: 0));
5375 }
5376
5377 if (!N0.hasOneUse())
5378 return SDValue();
5379
5380 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
5381 // (fneg (rcp x)) -> (rcp (fneg x))
5382 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5383 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: Neg, Flags: N0->getFlags());
5384 }
5385 case ISD::FP_ROUND: {
5386 SDValue CvtSrc = N0.getOperand(i: 0);
5387
5388 if (CvtSrc.getOpcode() == ISD::FNEG) {
5389 // (fneg (fp_round (fneg x))) -> (fp_round x)
5390 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT,
5391 N1: CvtSrc.getOperand(i: 0), N2: N0.getOperand(i: 1));
5392 }
5393
5394 if (!N0.hasOneUse())
5395 return SDValue();
5396
5397 // (fneg (fp_round x)) -> (fp_round (fneg x))
5398 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5399 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Neg, N2: N0.getOperand(i: 1));
5400 }
5401 case ISD::FP16_TO_FP: {
5402 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
5403 // f16, but legalization of f16 fneg ends up pulling it out of the source.
5404 // Put the fneg back as a legal source operation that can be matched later.
5405 SDLoc SL(N);
5406
5407 SDValue Src = N0.getOperand(i: 0);
5408 EVT SrcVT = Src.getValueType();
5409
5410 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
5411 SDValue IntFNeg = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: SrcVT, N1: Src,
5412 N2: DAG.getConstant(Val: 0x8000, DL: SL, VT: SrcVT));
5413 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFNeg);
5414 }
5415 case ISD::SELECT: {
5416 // fneg (select c, a, b) -> select c, (fneg a), (fneg b)
5417 // TODO: Invert conditions of foldFreeOpFromSelect
5418 return SDValue();
5419 }
5420 case ISD::BITCAST: {
5421 SDLoc SL(N);
5422 SDValue BCSrc = N0.getOperand(i: 0);
5423 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
5424 SDValue HighBits = BCSrc.getOperand(i: BCSrc.getNumOperands() - 1);
5425 if (VT != MVT::f64 || HighBits.getValueType().getSizeInBits() != 32 ||
5426 !fnegFoldsIntoOp(N: HighBits.getNode()))
5427 return SDValue();
5428
5429 // f64 fneg only really needs to operate on the high half of of the
5430 // register, so try to force it to an f32 operation to help make use of
5431 // source modifiers.
5432 //
5433 //
5434 // fneg (f64 (bitcast (build_vector x, y))) ->
5435 // f64 (bitcast (build_vector (bitcast i32:x to f32),
5436 // (fneg (bitcast i32:y to f32)))
5437
5438 SDValue CastHi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: HighBits);
5439 SDValue NegHi = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: CastHi);
5440 SDValue CastBack =
5441 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: HighBits.getValueType(), Operand: NegHi);
5442
5443 SmallVector<SDValue, 8> Ops(BCSrc->ops());
5444 Ops.back() = CastBack;
5445 DCI.AddToWorklist(N: NegHi.getNode());
5446 SDValue Build =
5447 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: BCSrc.getValueType(), Ops);
5448 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Build);
5449
5450 if (!N0.hasOneUse())
5451 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Result));
5452 return Result;
5453 }
5454
5455 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 &&
5456 BCSrc.hasOneUse()) {
5457 // fneg (bitcast (f32 (select cond, i32:lhs, i32:rhs))) ->
5458 // select cond, (bitcast i32:lhs to f32), (bitcast i32:rhs to f32)
5459
5460 // TODO: Cast back result for multiple uses is beneficial in some cases.
5461
5462 SDValue LHS =
5463 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 1));
5464 SDValue RHS =
5465 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 2));
5466
5467 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: LHS);
5468 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHS);
5469
5470 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: BCSrc.getOperand(i: 0), N2: NegLHS,
5471 N3: NegRHS);
5472 }
5473
5474 return SDValue();
5475 }
5476 default:
5477 return SDValue();
5478 }
5479}
5480
5481SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
5482 DAGCombinerInfo &DCI) const {
5483 SelectionDAG &DAG = DCI.DAG;
5484 SDValue N0 = N->getOperand(Num: 0);
5485
5486 if (!N0.hasOneUse())
5487 return SDValue();
5488
5489 switch (N0.getOpcode()) {
5490 case ISD::FP16_TO_FP: {
5491 assert(!isTypeLegal(MVT::f16) && "should only see if f16 is illegal");
5492 SDLoc SL(N);
5493 SDValue Src = N0.getOperand(i: 0);
5494 EVT SrcVT = Src.getValueType();
5495
5496 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
5497 SDValue IntFAbs = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SrcVT, N1: Src,
5498 N2: DAG.getConstant(Val: 0x7fff, DL: SL, VT: SrcVT));
5499 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFAbs);
5500 }
5501 case ISD::FP_ROUND: {
5502 SDLoc SL(N);
5503 SDValue CvtSrc = N0.getOperand(i: 0);
5504
5505 // fabs (fp_round x) -> fp_round (fabs x)
5506 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc,
5507 Flags: N->getFlags());
5508 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: N->getValueType(ResNo: 0), N1: Abs,
5509 N2: N0.getOperand(i: 1), Flags: N0->getFlags());
5510 }
5511 default:
5512 return SDValue();
5513 }
5514}
5515
5516SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
5517 DAGCombinerInfo &DCI) const {
5518 const auto *CFP = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
5519 if (!CFP)
5520 return SDValue();
5521
5522 std::optional<APFloat> Result = AMDGPU::evaluateRcp(Val: CFP->getValueAPF());
5523 if (!Result)
5524 return SDValue();
5525
5526 return DCI.DAG.getConstantFP(Val: *Result, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
5527}
5528
5529bool AMDGPUTargetLowering::isInt64ImmLegal(SDNode *N, SelectionDAG &DAG) const {
5530 if (!Subtarget->isGCN())
5531 return false;
5532
5533 ConstantSDNode *SDConstant = dyn_cast<ConstantSDNode>(Val: N);
5534 ConstantFPSDNode *SDFPConstant = dyn_cast<ConstantFPSDNode>(Val: N);
5535 auto &ST = DAG.getSubtarget<GCNSubtarget>();
5536 const auto *TII = ST.getInstrInfo();
5537
5538 if (!ST.hasVMovB64Inst() || (!SDConstant && !SDFPConstant))
5539 return false;
5540
5541 if (ST.has64BitLiterals())
5542 return true;
5543
5544 if (SDConstant) {
5545 const APInt &APVal = SDConstant->getAPIntValue();
5546 return isUInt<32>(x: APVal.getZExtValue()) || TII->isInlineConstant(Imm: APVal);
5547 }
5548
5549 APInt Val = SDFPConstant->getValueAPF().bitcastToAPInt();
5550 return isUInt<32>(x: Val.getZExtValue()) || TII->isInlineConstant(Imm: Val);
5551}
5552
5553SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
5554 DAGCombinerInfo &DCI) const {
5555 SelectionDAG &DAG = DCI.DAG;
5556 SDLoc DL(N);
5557
5558 switch(N->getOpcode()) {
5559 default:
5560 break;
5561 case ISD::BITCAST: {
5562 EVT DestVT = N->getValueType(ResNo: 0);
5563
5564 // Push casts through vector builds. This helps avoid emitting a large
5565 // number of copies when materializing floating point vector constants.
5566 //
5567 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
5568 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
5569 if (DestVT.isVector()) {
5570 SDValue Src = N->getOperand(Num: 0);
5571 if (Src.getOpcode() == ISD::BUILD_VECTOR &&
5572 (DCI.getDAGCombineLevel() < AfterLegalizeDAG ||
5573 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: DestVT))) {
5574 EVT SrcVT = Src.getValueType();
5575 unsigned NElts = DestVT.getVectorNumElements();
5576
5577 if (SrcVT.getVectorNumElements() == NElts) {
5578 EVT DestEltVT = DestVT.getVectorElementType();
5579
5580 SmallVector<SDValue, 8> CastedElts;
5581 SDLoc SL(N);
5582 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
5583 SDValue Elt = Src.getOperand(i: I);
5584 CastedElts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: DestEltVT, Operand: Elt));
5585 }
5586
5587 return DAG.getBuildVector(VT: DestVT, DL: SL, Ops: CastedElts);
5588 }
5589 }
5590 }
5591
5592 if (DestVT.getSizeInBits() != 64 || !DestVT.isVector())
5593 break;
5594
5595 // Fold bitcasts of constants.
5596 //
5597 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
5598 // TODO: Generalize and move to DAGCombiner
5599 SDValue Src = N->getOperand(Num: 0);
5600 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Src)) {
5601 SDLoc SL(N);
5602 if (isInt64ImmLegal(N: C, DAG))
5603 break;
5604 uint64_t CVal = C->getZExtValue();
5605 SDValue BV = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5606 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5607 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5608 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: BV);
5609 }
5610
5611 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
5612 const APInt &Val = C->getValueAPF().bitcastToAPInt();
5613 SDLoc SL(N);
5614 if (isInt64ImmLegal(N: C, DAG))
5615 break;
5616 uint64_t CVal = Val.getZExtValue();
5617 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5618 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5619 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5620
5621 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: Vec);
5622 }
5623
5624 break;
5625 }
5626 case ISD::SHL:
5627 case ISD::SRA:
5628 case ISD::SRL: {
5629 // Range metadata can be invalidated when loads are converted to legal types
5630 // (e.g. v2i64 -> v4i32).
5631 // Try to convert vector shl/sra/srl before type legalization so that range
5632 // metadata can be utilized.
5633 if (!(N->getValueType(ResNo: 0).isVector() &&
5634 DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
5635 DCI.getDAGCombineLevel() < AfterLegalizeDAG)
5636 break;
5637 if (N->getOpcode() == ISD::SHL)
5638 return performShlCombine(N, DCI);
5639 if (N->getOpcode() == ISD::SRA)
5640 return performSraCombine(N, DCI);
5641 return performSrlCombine(N, DCI);
5642 }
5643 case ISD::TRUNCATE:
5644 return performTruncateCombine(N, DCI);
5645 case ISD::MUL:
5646 return performMulCombine(N, DCI);
5647 case AMDGPUISD::MUL_U24:
5648 case AMDGPUISD::MUL_I24: {
5649 if (SDValue Simplified = simplifyMul24(Node24: N, DCI))
5650 return Simplified;
5651 break;
5652 }
5653 case AMDGPUISD::MULHI_I24:
5654 case AMDGPUISD::MULHI_U24:
5655 return simplifyMul24(Node24: N, DCI);
5656 case ISD::SMUL_LOHI:
5657 case ISD::UMUL_LOHI:
5658 return performMulLoHiCombine(N, DCI);
5659 case ISD::MULHS:
5660 return performMulhsCombine(N, DCI);
5661 case ISD::MULHU:
5662 return performMulhuCombine(N, DCI);
5663 case ISD::SELECT:
5664 return performSelectCombine(N, DCI);
5665 case ISD::FNEG:
5666 return performFNegCombine(N, DCI);
5667 case ISD::FABS:
5668 return performFAbsCombine(N, DCI);
5669 case AMDGPUISD::BFE_I32:
5670 case AMDGPUISD::BFE_U32: {
5671 assert(N->getValueType(0) == MVT::i32 &&
5672 "BFE_I32/BFE_U32 is a 32-bit operation");
5673 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
5674 if (!Width)
5675 break;
5676
5677 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
5678 if (WidthVal == 0)
5679 return DAG.getConstant(Val: 0, DL, VT: MVT::i32);
5680
5681 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
5682 if (!Offset)
5683 break;
5684
5685 SDValue BitsFrom = N->getOperand(Num: 0);
5686 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
5687
5688 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
5689
5690 if (OffsetVal == 0) {
5691 // This is already sign / zero extended, so try to fold away extra BFEs.
5692 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WidthVal);
5693 if (Signed) {
5694 if (DAG.ComputeNumSignBits(Op: BitsFrom) >= 32 - WidthVal + 1)
5695 return BitsFrom;
5696
5697 // This is a sign_extend_inreg. Replace it to take advantage of existing
5698 // DAG Combines. If not eliminated, we will match back to BFE during
5699 // selection.
5700
5701 // TODO: The sext_inreg of extended types ends, although we can could
5702 // handle them in a single BFE.
5703 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32, N1: BitsFrom,
5704 N2: DAG.getValueType(SmallVT));
5705 }
5706
5707 if (DAG.MaskedValueIsZero(Op: BitsFrom,
5708 Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - WidthVal)))
5709 return BitsFrom;
5710
5711 return DAG.getZeroExtendInReg(Op: BitsFrom, DL, VT: SmallVT);
5712 }
5713
5714 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(Val&: BitsFrom)) {
5715 if (Signed) {
5716 return constantFoldBFE<int32_t>(DAG,
5717 Src0: CVal->getSExtValue(),
5718 Offset: OffsetVal,
5719 Width: WidthVal,
5720 DL);
5721 }
5722
5723 return constantFoldBFE<uint32_t>(DAG,
5724 Src0: CVal->getZExtValue(),
5725 Offset: OffsetVal,
5726 Width: WidthVal,
5727 DL);
5728 }
5729
5730 if ((OffsetVal + WidthVal) >= 32 &&
5731 !(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
5732 SDValue ShiftVal = DAG.getConstant(Val: OffsetVal, DL, VT: MVT::i32);
5733 return DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32,
5734 N1: BitsFrom, N2: ShiftVal);
5735 }
5736
5737 if (BitsFrom.hasOneUse()) {
5738 APInt Demanded = APInt::getBitsSet(numBits: 32,
5739 loBit: OffsetVal,
5740 hiBit: OffsetVal + WidthVal);
5741
5742 KnownBits Known;
5743 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
5744 !DCI.isBeforeLegalizeOps());
5745 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5746 if (TLI.ShrinkDemandedConstant(Op: BitsFrom, DemandedBits: Demanded, TLO) ||
5747 TLI.SimplifyDemandedBits(Op: BitsFrom, DemandedBits: Demanded, Known, TLO)) {
5748 DCI.CommitTargetLoweringOpt(TLO);
5749 }
5750 }
5751
5752 break;
5753 }
5754 case ISD::LOAD:
5755 return performLoadCombine(N, DCI);
5756 case ISD::STORE:
5757 return performStoreCombine(N, DCI);
5758 case AMDGPUISD::RCP:
5759 case AMDGPUISD::RCP_IFLAG:
5760 return performRcpCombine(N, DCI);
5761 case ISD::AssertZext:
5762 case ISD::AssertSext:
5763 return performAssertSZExtCombine(N, DCI);
5764 case ISD::INTRINSIC_WO_CHAIN:
5765 return performIntrinsicWOChainCombine(N, DCI);
5766 case AMDGPUISD::FMAD_FTZ: {
5767 SDValue N0 = N->getOperand(Num: 0);
5768 SDValue N1 = N->getOperand(Num: 1);
5769 SDValue N2 = N->getOperand(Num: 2);
5770 EVT VT = N->getValueType(ResNo: 0);
5771
5772 // FMAD_FTZ is a FMAD + flush denormals to zero.
5773 // We flush the inputs, the intermediate step, and the output.
5774 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Val&: N0);
5775 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
5776 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
5777 if (N0CFP && N1CFP && N2CFP) {
5778 const auto FTZ = [](const APFloat &V) {
5779 if (V.isDenormal()) {
5780 APFloat Zero(V.getSemantics(), 0);
5781 return V.isNegative() ? -Zero : Zero;
5782 }
5783 return V;
5784 };
5785
5786 APFloat V0 = FTZ(N0CFP->getValueAPF());
5787 APFloat V1 = FTZ(N1CFP->getValueAPF());
5788 APFloat V2 = FTZ(N2CFP->getValueAPF());
5789 V0.multiply(RHS: V1, RM: APFloat::rmNearestTiesToEven);
5790 V0 = FTZ(V0);
5791 V0.add(RHS: V2, RM: APFloat::rmNearestTiesToEven);
5792 return DAG.getConstantFP(Val: FTZ(V0), DL, VT);
5793 }
5794 break;
5795 }
5796 }
5797 return SDValue();
5798}
5799
5800bool AMDGPUTargetLowering::SimplifyDemandedBitsForTargetNode(
5801 SDValue Op, const APInt &OriginalDemandedBits,
5802 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
5803 unsigned Depth) const {
5804 switch (Op.getOpcode()) {
5805 case ISD::INTRINSIC_WO_CHAIN: {
5806 switch (Op.getConstantOperandVal(i: 0)) {
5807 case Intrinsic::amdgcn_readfirstlane:
5808 case Intrinsic::amdgcn_readlane:
5809 case Intrinsic::amdgcn_wwm: {
5810 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5811 DemandedElts: OriginalDemandedElts, Known, TLO, Depth: Depth + 1))
5812 return true;
5813 break;
5814 }
5815 case Intrinsic::amdgcn_set_inactive:
5816 case Intrinsic::amdgcn_set_inactive_chain_arg: {
5817 // The result is operand 1 in active lanes and operand 2 in inactive
5818 // lanes, so the known bits are the intersection of both operands.
5819 KnownBits KnownValue, KnownInactive;
5820 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5821 DemandedElts: OriginalDemandedElts, Known&: KnownValue, TLO,
5822 Depth: Depth + 1))
5823 return true;
5824 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), DemandedBits: OriginalDemandedBits,
5825 DemandedElts: OriginalDemandedElts, Known&: KnownInactive, TLO,
5826 Depth: Depth + 1))
5827 return true;
5828 Known = KnownValue.intersectWith(RHS: KnownInactive);
5829 break;
5830 }
5831 default:
5832 break;
5833 }
5834 break;
5835 }
5836 default:
5837 break;
5838 }
5839
5840 return false;
5841}
5842
5843//===----------------------------------------------------------------------===//
5844// Helper functions
5845//===----------------------------------------------------------------------===//
5846
5847SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
5848 const TargetRegisterClass *RC,
5849 Register Reg, EVT VT,
5850 const SDLoc &SL,
5851 bool RawReg) const {
5852 MachineFunction &MF = DAG.getMachineFunction();
5853 MachineRegisterInfo &MRI = MF.getRegInfo();
5854 Register VReg;
5855
5856 if (!MRI.isLiveIn(Reg)) {
5857 VReg = MRI.createVirtualRegister(RegClass: RC);
5858 MRI.addLiveIn(Reg, vreg: VReg);
5859 } else {
5860 VReg = MRI.getLiveInVirtReg(PReg: Reg);
5861 }
5862
5863 if (RawReg)
5864 return DAG.getRegister(Reg: VReg, VT);
5865
5866 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: VReg, VT);
5867}
5868
5869// This may be called multiple times, and nothing prevents creating multiple
5870// objects at the same offset. See if we already defined this object.
5871static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
5872 int64_t Offset) {
5873 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
5874 if (MFI.getObjectOffset(ObjectIdx: I) == Offset) {
5875 assert(MFI.getObjectSize(I) == Size);
5876 return I;
5877 }
5878 }
5879
5880 return MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
5881}
5882
5883SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
5884 EVT VT,
5885 const SDLoc &SL,
5886 int64_t Offset) const {
5887 MachineFunction &MF = DAG.getMachineFunction();
5888 MachineFrameInfo &MFI = MF.getFrameInfo();
5889 int FI = getOrCreateFixedStackObject(MFI, Size: VT.getStoreSize(), Offset);
5890
5891 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
5892 SDValue Ptr = DAG.getFrameIndex(FI, VT: MVT::i32);
5893
5894 return DAG.getLoad(VT, dl: SL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: SrcPtrInfo, Alignment: Align(4),
5895 MMOFlags: MachineMemOperand::MODereferenceable |
5896 MachineMemOperand::MOInvariant);
5897}
5898
5899SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
5900 const SDLoc &SL,
5901 SDValue Chain,
5902 SDValue ArgVal,
5903 int64_t Offset) const {
5904 MachineFunction &MF = DAG.getMachineFunction();
5905 MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
5906 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5907
5908 SDValue Ptr = DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32);
5909 // Stores to the argument stack area are relative to the stack pointer.
5910 SDValue SP =
5911 DAG.getCopyFromReg(Chain, dl: SL, Reg: Info->getStackPtrOffsetReg(), VT: MVT::i32);
5912 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: SP, N2: Ptr);
5913 SDValue Store = DAG.getStore(Chain, dl: SL, Val: ArgVal, Ptr, PtrInfo: DstInfo, Alignment: Align(4),
5914 MMOFlags: MachineMemOperand::MODereferenceable);
5915 return Store;
5916}
5917
5918SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
5919 const TargetRegisterClass *RC,
5920 EVT VT, const SDLoc &SL,
5921 const ArgDescriptor &Arg) const {
5922 assert(Arg && "Attempting to load missing argument");
5923
5924 SDValue V = Arg.isRegister() ?
5925 CreateLiveInRegister(DAG, RC, Reg: Arg.getRegister(), VT, SL) :
5926 loadStackInputValue(DAG, VT, SL, Offset: Arg.getStackOffset());
5927
5928 if (!Arg.isMasked())
5929 return V;
5930
5931 unsigned Mask = Arg.getMask();
5932 unsigned Shift = llvm::countr_zero<unsigned>(Val: Mask);
5933 V = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: V,
5934 N2: DAG.getShiftAmountConstant(Val: Shift, VT, DL: SL));
5935 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT, N1: V,
5936 N2: DAG.getConstant(Val: Mask >> Shift, DL: SL, VT));
5937}
5938
5939uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5940 uint64_t ExplicitKernArgSize, const ImplicitParameter Param) const {
5941 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
5942 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr();
5943 uint64_t ArgOffset =
5944 alignTo(Size: ExplicitKernArgSize, A: Alignment) + ExplicitArgOffset;
5945 switch (Param) {
5946 case FIRST_IMPLICIT:
5947 return ArgOffset;
5948 case PRIVATE_BASE:
5949 return ArgOffset + AMDGPU::ImplicitArg::PRIVATE_BASE_OFFSET;
5950 case SHARED_BASE:
5951 return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET;
5952 case QUEUE_PTR:
5953 return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET;
5954 }
5955 llvm_unreachable("unexpected implicit parameter type");
5956}
5957
5958uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5959 const MachineFunction &MF, const ImplicitParameter Param) const {
5960 const AMDGPUMachineFunctionInfo *MFI =
5961 MF.getInfo<AMDGPUMachineFunctionInfo>();
5962 return getImplicitParameterOffset(ExplicitKernArgSize: MFI->getExplicitKernArgSize(), Param);
5963}
5964
5965SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
5966 SelectionDAG &DAG, int Enabled,
5967 int &RefinementSteps,
5968 bool &UseOneConstNR,
5969 bool Reciprocal) const {
5970 EVT VT = Operand.getValueType();
5971
5972 if (VT == MVT::f32) {
5973 RefinementSteps = 0;
5974 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(Operand), VT, Operand);
5975 }
5976
5977 // TODO: There is also f64 rsq instruction, but the documentation is less
5978 // clear on its precision.
5979
5980 return SDValue();
5981}
5982
5983SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
5984 SelectionDAG &DAG, int Enabled,
5985 int &RefinementSteps) const {
5986 EVT VT = Operand.getValueType();
5987
5988 if (VT == MVT::f32) {
5989 // Reciprocal, < 1 ulp error.
5990 //
5991 // This reciprocal approximation converges to < 0.5 ulp error with one
5992 // newton rhapson performed with two fused multiple adds (FMAs).
5993
5994 RefinementSteps = 0;
5995 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SDLoc(Operand), VT, Operand);
5996 }
5997
5998 // TODO: There is also f64 rcp instruction, but the documentation is less
5999 // clear on its precision.
6000
6001 return SDValue();
6002}
6003
6004static unsigned workitemIntrinsicDim(unsigned ID) {
6005 switch (ID) {
6006 case Intrinsic::amdgcn_workitem_id_x:
6007 return 0;
6008 case Intrinsic::amdgcn_workitem_id_y:
6009 return 1;
6010 case Intrinsic::amdgcn_workitem_id_z:
6011 return 2;
6012 default:
6013 llvm_unreachable("not a workitem intrinsic");
6014 }
6015}
6016
6017void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
6018 const SDValue Op, KnownBits &Known,
6019 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
6020
6021 Known.resetAll(); // Don't know anything.
6022
6023 unsigned Opc = Op.getOpcode();
6024
6025 switch (Opc) {
6026 default:
6027 break;
6028 case AMDGPUISD::CARRY:
6029 case AMDGPUISD::BORROW: {
6030 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31);
6031 break;
6032 }
6033
6034 case AMDGPUISD::BFE_I32:
6035 case AMDGPUISD::BFE_U32: {
6036 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6037 if (!CWidth)
6038 return;
6039
6040 uint32_t Width = CWidth->getZExtValue() & 0x1f;
6041
6042 if (Opc == AMDGPUISD::BFE_U32)
6043 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - Width);
6044
6045 break;
6046 }
6047 case AMDGPUISD::FP_TO_FP16: {
6048 unsigned BitWidth = Known.getBitWidth();
6049
6050 // High bits are zero.
6051 Known.Zero = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 16);
6052 break;
6053 }
6054 case AMDGPUISD::MUL_U24:
6055 case AMDGPUISD::MUL_I24: {
6056 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6057 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6058 unsigned BitWidth = Op.getScalarValueSizeInBits();
6059
6060 // Sign/Zero extend from 24 bits.
6061 if (Opc == AMDGPUISD::MUL_I24) {
6062 LHSKnown = LHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6063 RHSKnown = RHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6064 } else {
6065 LHSKnown = LHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6066 RHSKnown = RHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6067 }
6068
6069 // TODO: SelfMultiply can be poison, but not undef.
6070 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
6071 if (SelfMultiply)
6072 SelfMultiply &= DAG.isGuaranteedNotToBeUndefOrPoison(
6073 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
6074 Depth: Depth + 1);
6075
6076 Known = KnownBits::mul(LHS: LHSKnown, RHS: RHSKnown, NoUndefSelfMultiply: SelfMultiply);
6077 break;
6078 }
6079 case AMDGPUISD::PERM: {
6080 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6081 if (!CMask)
6082 return;
6083
6084 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6085 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6086 unsigned Sel = CMask->getZExtValue();
6087
6088 for (unsigned I = 0; I < 32; I += 8) {
6089 unsigned SelBits = Sel & 0xff;
6090 if (SelBits < 4) {
6091 SelBits *= 8;
6092 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6093 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6094 } else if (SelBits < 7) {
6095 SelBits = (SelBits & 3) * 8;
6096 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6097 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6098 } else if (SelBits == 0x0c) {
6099 Known.Zero |= 0xFFull << I;
6100 } else if (SelBits > 0x0c) {
6101 Known.One |= 0xFFull << I;
6102 }
6103 Sel >>= 8;
6104 }
6105 break;
6106 }
6107 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
6108 Known.Zero.setHighBits(24);
6109 break;
6110 }
6111 case AMDGPUISD::BUFFER_LOAD_USHORT: {
6112 Known.Zero.setHighBits(16);
6113 break;
6114 }
6115 case AMDGPUISD::LDS: {
6116 auto *GA = cast<GlobalAddressSDNode>(Val: Op.getOperand(i: 0).getNode());
6117 Align Alignment = GA->getGlobal()->getPointerAlignment(DL: DAG.getDataLayout());
6118
6119 Known.Zero.setHighBits(16);
6120 Known.Zero.setLowBits(Log2(A: Alignment));
6121 break;
6122 }
6123 case AMDGPUISD::SMIN3:
6124 case AMDGPUISD::SMAX3:
6125 case AMDGPUISD::SMED3:
6126 case AMDGPUISD::UMIN3:
6127 case AMDGPUISD::UMAX3:
6128 case AMDGPUISD::UMED3: {
6129 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6130 if (Known2.isUnknown())
6131 break;
6132
6133 KnownBits Known1 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6134 if (Known1.isUnknown())
6135 break;
6136
6137 KnownBits Known0 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6138 if (Known0.isUnknown())
6139 break;
6140
6141 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
6142 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
6143 Known.One = Known0.One & Known1.One & Known2.One;
6144 break;
6145 }
6146 case ISD::INTRINSIC_WO_CHAIN: {
6147 unsigned IID = Op.getConstantOperandVal(i: 0);
6148 switch (IID) {
6149 case Intrinsic::amdgcn_workitem_id_x:
6150 case Intrinsic::amdgcn_workitem_id_y:
6151 case Intrinsic::amdgcn_workitem_id_z: {
6152 unsigned MaxValue = Subtarget->getMaxWorkitemID(
6153 Kernel: DAG.getMachineFunction().getFunction(), Dimension: workitemIntrinsicDim(ID: IID));
6154 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
6155 break;
6156 }
6157 default:
6158 break;
6159 }
6160 }
6161 }
6162}
6163
6164unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
6165 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6166 unsigned Depth) const {
6167 switch (Op.getOpcode()) {
6168 case AMDGPUISD::BFE_I32: {
6169 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6170 if (!Width)
6171 return 1;
6172
6173 unsigned SignBits = 32 - (Width->getZExtValue() & 0x1f) + 1;
6174 if (!isNullConstant(V: Op.getOperand(i: 1)))
6175 return SignBits;
6176
6177 // TODO: Could probably figure something out with non-0 offsets.
6178 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6179 return std::max(a: SignBits, b: Op0SignBits);
6180 }
6181
6182 case AMDGPUISD::BFE_U32: {
6183 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6184 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
6185 }
6186
6187 case AMDGPUISD::CARRY:
6188 case AMDGPUISD::BORROW:
6189 return 31;
6190 case AMDGPUISD::BUFFER_LOAD_BYTE:
6191 return 25;
6192 case AMDGPUISD::BUFFER_LOAD_SHORT:
6193 return 17;
6194 case AMDGPUISD::BUFFER_LOAD_UBYTE:
6195 return 24;
6196 case AMDGPUISD::BUFFER_LOAD_USHORT:
6197 return 16;
6198 case AMDGPUISD::FP_TO_FP16:
6199 return 16;
6200 case AMDGPUISD::SMIN3:
6201 case AMDGPUISD::SMAX3:
6202 case AMDGPUISD::SMED3:
6203 case AMDGPUISD::UMIN3:
6204 case AMDGPUISD::UMAX3:
6205 case AMDGPUISD::UMED3: {
6206 unsigned Tmp2 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6207 if (Tmp2 == 1)
6208 return 1; // Early out.
6209
6210 unsigned Tmp1 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6211 if (Tmp1 == 1)
6212 return 1; // Early out.
6213
6214 unsigned Tmp0 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6215 if (Tmp0 == 1)
6216 return 1; // Early out.
6217
6218 return std::min(l: {Tmp0, Tmp1, Tmp2});
6219 }
6220 default:
6221 return 1;
6222 }
6223}
6224
6225unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
6226 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
6227 const MachineRegisterInfo &MRI, unsigned Depth) const {
6228 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
6229 if (!MI)
6230 return 1;
6231
6232 // TODO: Check range metadata on MMO.
6233 switch (MI->getOpcode()) {
6234 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
6235 return 25;
6236 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
6237 return 17;
6238 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
6239 return 24;
6240 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
6241 return 16;
6242 case AMDGPU::G_AMDGPU_SMED3:
6243 case AMDGPU::G_AMDGPU_UMED3: {
6244 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
6245 unsigned Tmp2 = Analysis.computeNumSignBits(R: Src2, DemandedElts, Depth: Depth + 1);
6246 if (Tmp2 == 1)
6247 return 1;
6248 unsigned Tmp1 = Analysis.computeNumSignBits(R: Src1, DemandedElts, Depth: Depth + 1);
6249 if (Tmp1 == 1)
6250 return 1;
6251 unsigned Tmp0 = Analysis.computeNumSignBits(R: Src0, DemandedElts, Depth: Depth + 1);
6252 if (Tmp0 == 1)
6253 return 1;
6254 return std::min(l: {Tmp0, Tmp1, Tmp2});
6255 }
6256 default:
6257 return 1;
6258 }
6259}
6260
6261bool AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode(
6262 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6263 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
6264 unsigned Opcode = Op.getOpcode();
6265 switch (Opcode) {
6266 case AMDGPUISD::BFE_I32:
6267 case AMDGPUISD::BFE_U32:
6268 return false;
6269 }
6270 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
6271 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
6272}
6273
6274bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
6275 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
6276 unsigned Depth) const {
6277 unsigned Opcode = Op.getOpcode();
6278 switch (Opcode) {
6279 case AMDGPUISD::FMIN_LEGACY:
6280 case AMDGPUISD::FMAX_LEGACY:
6281 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6282 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6283 case AMDGPUISD::FMUL_LEGACY:
6284 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
6285 if (SNaN)
6286 return true;
6287 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6288 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6289 }
6290 case AMDGPUISD::FMED3:
6291 case AMDGPUISD::FMIN3:
6292 case AMDGPUISD::FMAX3:
6293 case AMDGPUISD::FMINIMUM3:
6294 case AMDGPUISD::FMAXIMUM3:
6295 case AMDGPUISD::FMAD_FTZ: {
6296 if (SNaN)
6297 return true;
6298 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6299 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6300 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6301 }
6302 case AMDGPUISD::CVT_F32_UBYTE0:
6303 case AMDGPUISD::CVT_F32_UBYTE1:
6304 case AMDGPUISD::CVT_F32_UBYTE2:
6305 case AMDGPUISD::CVT_F32_UBYTE3:
6306 return true;
6307
6308 case AMDGPUISD::RCP:
6309 case AMDGPUISD::RSQ:
6310 case AMDGPUISD::RCP_LEGACY:
6311 case AMDGPUISD::RSQ_CLAMP: {
6312 if (SNaN)
6313 return true;
6314
6315 // TODO: Need is known positive check.
6316 return false;
6317 }
6318 case ISD::FLDEXP:
6319 case AMDGPUISD::FRACT: {
6320 if (SNaN)
6321 return true;
6322 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6323 }
6324 case AMDGPUISD::DIV_SCALE:
6325 case AMDGPUISD::DIV_FMAS:
6326 case AMDGPUISD::DIV_FIXUP:
6327 // TODO: Refine on operands.
6328 return SNaN;
6329 case AMDGPUISD::SIN_HW:
6330 case AMDGPUISD::COS_HW: {
6331 // TODO: Need check for infinity
6332 return SNaN;
6333 }
6334 case ISD::INTRINSIC_WO_CHAIN: {
6335 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
6336 // TODO: Handle more intrinsics
6337 switch (IntrinsicID) {
6338 case Intrinsic::amdgcn_cubeid:
6339 case Intrinsic::amdgcn_cvt_off_f32_i4:
6340 return true;
6341
6342 case Intrinsic::amdgcn_frexp_mant: {
6343 if (SNaN)
6344 return true;
6345 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6346 }
6347 case Intrinsic::amdgcn_cvt_pkrtz: {
6348 if (SNaN)
6349 return true;
6350 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6351 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6352 }
6353 case Intrinsic::amdgcn_rcp:
6354 case Intrinsic::amdgcn_rsq:
6355 case Intrinsic::amdgcn_rcp_legacy:
6356 case Intrinsic::amdgcn_rsq_legacy:
6357 case Intrinsic::amdgcn_rsq_clamp:
6358 case Intrinsic::amdgcn_tanh: {
6359 if (SNaN)
6360 return true;
6361
6362 // TODO: Need is known positive check.
6363 return false;
6364 }
6365 case Intrinsic::amdgcn_trig_preop:
6366 case Intrinsic::amdgcn_fdot2:
6367 // TODO: Refine on operand
6368 return SNaN;
6369 case Intrinsic::amdgcn_fma_legacy:
6370 if (SNaN)
6371 return true;
6372 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6373 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1) &&
6374 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 3), SNaN, Depth: Depth + 1);
6375 default:
6376 return false;
6377 }
6378 }
6379 default:
6380 return false;
6381 }
6382}
6383
6384bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
6385 Register N0, Register N1) const {
6386 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
6387}
6388