1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This is the parent TargetLowering class for hardware code gen
11/// targets.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUISelLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUMachineFunctionInfo.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUSelectionDAGInfo.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/Analysis.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Support/KnownBits.h"
29#include "llvm/Target/TargetMachine.h"
30
31using namespace llvm;
32
33#define GET_CALLING_CONV_IMPL
34#include "AMDGPUGenCallingConv.inc"
35
36static cl::opt<bool> AMDGPUBypassSlowDiv(
37 "amdgpu-bypass-slow-div",
38 cl::desc("Skip 64-bit divide for dynamic 32-bit values"),
39 cl::init(Val: true));
40
41// Find a larger type to do a load / store of a vector with.
42EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
43 unsigned StoreSize = VT.getStoreSizeInBits();
44 if (StoreSize <= 32)
45 return EVT::getIntegerVT(Context&: Ctx, BitWidth: StoreSize);
46
47 if (StoreSize % 32 == 0)
48 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i32, NumElements: StoreSize / 32);
49
50 return VT;
51}
52
53unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
54 return DAG.computeKnownBits(Op).countMaxActiveBits();
55}
56
57unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
58 // In order for this to be a signed 24-bit value, bit 23, must
59 // be a sign bit.
60 return DAG.ComputeMaxSignificantBits(Op);
61}
62
63AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
64 const TargetSubtargetInfo &STI,
65 const AMDGPUSubtarget &AMDGPUSTI)
66 : TargetLowering(TM, STI), Subtarget(&AMDGPUSTI) {
67 // Always lower memset, memcpy, and memmove intrinsics to load/store
68 // instructions, rather then generating calls to memset, mempcy or memmove.
69 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U;
70 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U;
71 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U;
72
73 // Enable ganging up loads and stores in the memcpy DAG lowering.
74 MaxGluedStoresPerMemcpy = 16;
75
76 // Lower floating point store/load to integer store/load to reduce the number
77 // of patterns in tablegen.
78 setOperationAction(Op: ISD::LOAD, VT: MVT::f32, Action: Promote);
79 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
80
81 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Promote);
82 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
83
84 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f32, Action: Promote);
85 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
86
87 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f32, Action: Promote);
88 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
89
90 setOperationAction(Op: ISD::LOAD, VT: MVT::v5f32, Action: Promote);
91 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
92
93 setOperationAction(Op: ISD::LOAD, VT: MVT::v6f32, Action: Promote);
94 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
95
96 setOperationAction(Op: ISD::LOAD, VT: MVT::v7f32, Action: Promote);
97 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
98
99 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f32, Action: Promote);
100 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
101
102 setOperationAction(Op: ISD::LOAD, VT: MVT::v9f32, Action: Promote);
103 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
104
105 setOperationAction(Op: ISD::LOAD, VT: MVT::v10f32, Action: Promote);
106 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
107
108 setOperationAction(Op: ISD::LOAD, VT: MVT::v11f32, Action: Promote);
109 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
110
111 setOperationAction(Op: ISD::LOAD, VT: MVT::v12f32, Action: Promote);
112 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
113
114 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f32, Action: Promote);
115 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
116
117 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f32, Action: Promote);
118 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
119
120 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Promote);
121 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i64, DestVT: MVT::v2i32);
122
123 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
124 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
125
126 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Promote);
127 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f64, DestVT: MVT::v2i32);
128
129 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Promote);
130 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
131
132 setOperationAction(Op: ISD::LOAD, VT: MVT::v3i64, Action: Promote);
133 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
134
135 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i64, Action: Promote);
136 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
137
138 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f64, Action: Promote);
139 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
140
141 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f64, Action: Promote);
142 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
143
144 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i64, Action: Promote);
145 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
146
147 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f64, Action: Promote);
148 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
149
150 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i64, Action: Promote);
151 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
152
153 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f64, Action: Promote);
154 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
155
156 setOperationAction(Op: ISD::LOAD, VT: MVT::i128, Action: Promote);
157 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i128, DestVT: MVT::v4i32);
158
159 // TODO: Would be better to consume as directly legal
160 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f32, Action: Promote);
161 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
162
163 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f64, Action: Promote);
164 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f64, DestVT: MVT::i64);
165
166 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f16, Action: Promote);
167 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
168
169 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::bf16, Action: Promote);
170 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
171
172 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2f32, Action: Promote);
173 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2f32, DestVT: MVT::i64);
174
175 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f32, Action: Promote);
176 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
177
178 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f64, Action: Promote);
179 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f64, DestVT: MVT::i64);
180
181 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f16, Action: Promote);
182 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
183
184 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::bf16, Action: Promote);
185 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
186
187 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2f32, Action: Promote);
188 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2f32, DestVT: MVT::i64);
189
190 // There are no 64-bit extloads. These should be done as a 32-bit extload and
191 // an extension to 64-bit.
192 for (MVT VT : MVT::integer_valuetypes())
193 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64, MemVT: VT,
194 Action: Expand);
195
196 for (MVT VT : MVT::integer_valuetypes()) {
197 if (VT == MVT::i64)
198 continue;
199
200 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) {
201 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i1, Action: Promote);
202 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i8, Action: Legal);
203 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i16, Action: Legal);
204 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i32, Action: Expand);
205 }
206 }
207
208 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
209 for (auto MemVT :
210 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
211 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT,
212 Action: Expand);
213
214 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
215 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
216 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
218 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
220 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
221 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
222 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
223 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
224 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
225 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
226 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
227 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
228
229 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
230 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
231 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
232 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
233 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
234 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
235
236 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
237 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
238 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
239 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
240 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
241 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
242 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
243 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
244 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
245 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
246 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
247 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
248
249 setOperationAction(Op: ISD::STORE, VT: MVT::f32, Action: Promote);
250 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
251
252 setOperationAction(Op: ISD::STORE, VT: MVT::v2f32, Action: Promote);
253 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
254
255 setOperationAction(Op: ISD::STORE, VT: MVT::v3f32, Action: Promote);
256 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
257
258 setOperationAction(Op: ISD::STORE, VT: MVT::v4f32, Action: Promote);
259 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
260
261 setOperationAction(Op: ISD::STORE, VT: MVT::v5f32, Action: Promote);
262 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
263
264 setOperationAction(Op: ISD::STORE, VT: MVT::v6f32, Action: Promote);
265 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
266
267 setOperationAction(Op: ISD::STORE, VT: MVT::v7f32, Action: Promote);
268 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
269
270 setOperationAction(Op: ISD::STORE, VT: MVT::v8f32, Action: Promote);
271 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
272
273 setOperationAction(Op: ISD::STORE, VT: MVT::v9f32, Action: Promote);
274 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
275
276 setOperationAction(Op: ISD::STORE, VT: MVT::v10f32, Action: Promote);
277 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
278
279 setOperationAction(Op: ISD::STORE, VT: MVT::v11f32, Action: Promote);
280 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
281
282 setOperationAction(Op: ISD::STORE, VT: MVT::v12f32, Action: Promote);
283 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
284
285 setOperationAction(Op: ISD::STORE, VT: MVT::v16f32, Action: Promote);
286 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
287
288 setOperationAction(Op: ISD::STORE, VT: MVT::v32f32, Action: Promote);
289 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
290
291 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Promote);
292 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i64, DestVT: MVT::v2i32);
293
294 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
295 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
296
297 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Promote);
298 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f64, DestVT: MVT::v2i32);
299
300 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Promote);
301 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
302
303 setOperationAction(Op: ISD::STORE, VT: MVT::v3i64, Action: Promote);
304 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
305
306 setOperationAction(Op: ISD::STORE, VT: MVT::v3f64, Action: Promote);
307 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
308
309 setOperationAction(Op: ISD::STORE, VT: MVT::v4i64, Action: Promote);
310 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
311
312 setOperationAction(Op: ISD::STORE, VT: MVT::v4f64, Action: Promote);
313 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
314
315 setOperationAction(Op: ISD::STORE, VT: MVT::v8i64, Action: Promote);
316 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
317
318 setOperationAction(Op: ISD::STORE, VT: MVT::v8f64, Action: Promote);
319 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
320
321 setOperationAction(Op: ISD::STORE, VT: MVT::v16i64, Action: Promote);
322 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
323
324 setOperationAction(Op: ISD::STORE, VT: MVT::v16f64, Action: Promote);
325 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
326
327 setOperationAction(Op: ISD::STORE, VT: MVT::i128, Action: Promote);
328 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i128, DestVT: MVT::v4i32);
329
330 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i1, Action: Expand);
331 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i8, Action: Expand);
332 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
333 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i32, Action: Expand);
334
335 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i1, Action: Expand);
336 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i8, Action: Expand);
337 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i16, Action: Expand);
338 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i32, Action: Expand);
339
340 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
341 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
342 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
343 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
344 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
345 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
346 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
347 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
348 setTruncStoreAction(ValVT: MVT::v6f32, MemVT: MVT::v6f16, Action: Expand);
349 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
350 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
351 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
352 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
353 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
354 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
355
356 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
357 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
358 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
359
360 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
361 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
362 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
363
364 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i8, Action: Expand);
365
366 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
367 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
368 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i8, Action: Expand);
369 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i1, Action: Expand);
370 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
371 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
372 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
373
374 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i32, Action: Expand);
375 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i16, Action: Expand);
376 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
377 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
378 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
379
380 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i1, Action: Expand);
381 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i8, Action: Expand);
382 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i16, Action: Expand);
383
384 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i1, Action: Expand);
385 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i8, Action: Expand);
386 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i16, Action: Expand);
387
388 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i1, Action: Expand);
389 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i8, Action: Expand);
390 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i16, Action: Expand);
391
392 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
393 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
394 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
395
396 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
397 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
398 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
399 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i16, Action: Expand);
400 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
401 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
402 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i1, Action: Expand);
403
404 setOperationAction(Ops: ISD::Constant, VTs: {MVT::i32, MVT::i64}, Action: Legal);
405 setOperationAction(Ops: ISD::ConstantFP, VTs: {MVT::f32, MVT::f64}, Action: Legal);
406
407 setOperationAction(Ops: {ISD::BR_JT, ISD::BRIND}, VT: MVT::Other, Action: Expand);
408
409 // For R600, this is totally unsupported, just custom lower to produce an
410 // error.
411 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
412
413 // Library functions. These default to Expand, but we have instructions
414 // for them.
415 setOperationAction(Ops: {ISD::FCEIL, ISD::FPOW, ISD::FABS, ISD::FFLOOR,
416 ISD::FROUNDEVEN, ISD::FTRUNC},
417 VTs: {MVT::f16, MVT::f32}, Action: Legal);
418 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM}, VT: MVT::f32, Action: Legal);
419
420 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Custom);
421 setOperationAction(Ops: ISD::FROUND, VTs: {MVT::f32, MVT::f64}, Action: Custom);
422 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND},
423 VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
424
425 setOperationAction(
426 Ops: {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, VT: MVT::f32,
427 Action: Custom);
428 setOperationAction(Ops: {ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, VT: MVT::f64, Action: Custom);
429
430 setOperationAction(Ops: ISD::FNEARBYINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
431
432 setOperationAction(Ops: ISD::FRINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
433
434 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VTs: {MVT::f16, MVT::f32, MVT::f64},
435 Action: Expand);
436
437 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
438 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
439 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Custom);
440
441 setOperationAction(Ops: {ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, VT: MVT::f16,
442 Action: Custom);
443
444 setOperationAction(Ops: ISD::FCANONICALIZE, VTs: {MVT::f32, MVT::f64}, Action: Legal);
445
446 // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
447 // scalarization code. Can be removed when IS_FPCLASS expand isn't called by
448 // default unless marked custom/legal.
449 setOperationAction(Ops: ISD::IS_FPCLASS,
450 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
451 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
452 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64,
453 MVT::v16f64},
454 Action: Custom);
455
456 // Expand to fneg + fadd.
457 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
458
459 setOperationAction(Ops: ISD::CONCAT_VECTORS,
460 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
461 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
462 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
463 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
464 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
465 Action: Custom);
466
467 setOperationAction(
468 Ops: ISD::EXTRACT_SUBVECTOR,
469 VTs: {MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32,
470 MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32,
471 MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32,
472 MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32,
473 MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
474 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
475 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64},
476 Action: Custom);
477
478 setOperationAction(Ops: {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP}, VT: MVT::f64,
479 Action: Expand);
480 setOperationAction(Ops: ISD::FP_TO_FP16, VTs: {MVT::f64, MVT::f32}, Action: Custom);
481
482 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
483 for (MVT VT : ScalarIntVTs) {
484 // These should use [SU]DIVREM, so set them to expand
485 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT,
486 Action: Expand);
487
488 // GPU does not have divrem function for signed or unsigned.
489 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT, Action: Custom);
490
491 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
492 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
493
494 setOperationAction(Ops: {ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Action: Expand);
495
496 setOperationAction(Ops: {ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT,
497 Action: Expand);
498 }
499
500 // The hardware supports 32-bit FSHR, but not FSHL.
501 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
502
503 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VTs: {MVT::i32, MVT::i64}, Action: Expand);
504
505 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT: MVT::i16, Action: Expand);
506
507 setOperationAction(Ops: {ISD::MUL, ISD::MULHU, ISD::MULHS}, VT: MVT::i64, Action: Expand);
508 setOperationAction(Ops: {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT,
509 ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
510 ISD::FP_TO_UINT_SAT},
511 VT: MVT::i64, Action: Custom);
512 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: Expand);
513
514 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i32,
515 Action: Legal);
516
517 setOperationAction(
518 Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON},
519 VT: MVT::i64, Action: Custom);
520
521 for (auto VT : {MVT::i8, MVT::i16})
522 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT, Action: Custom);
523
524 static const MVT::SimpleValueType VectorIntTypes[] = {
525 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
526 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
527
528 for (MVT VT : VectorIntTypes) {
529 // Expand the following operations for the current type by default.
530 // clang-format off
531 setOperationAction(Ops: {ISD::ADD, ISD::AND,
532 ISD::FP_TO_SINT, ISD::FP_TO_UINT,
533 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
534 ISD::MUL, ISD::MULHU,
535 ISD::MULHS, ISD::OR,
536 ISD::SHL, ISD::SRA,
537 ISD::SRL, ISD::ROTL,
538 ISD::ROTR, ISD::SUB,
539 ISD::SINT_TO_FP, ISD::UINT_TO_FP,
540 ISD::SDIV, ISD::UDIV,
541 ISD::SREM, ISD::UREM,
542 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
543 ISD::SDIVREM, ISD::UDIVREM,
544 ISD::SELECT, ISD::VSELECT,
545 ISD::SELECT_CC, ISD::XOR,
546 ISD::BSWAP, ISD::CTPOP,
547 ISD::CTTZ, ISD::CTLZ,
548 ISD::VECTOR_SHUFFLE, ISD::SETCC,
549 ISD::ADDRSPACECAST},
550 VT, Action: Expand);
551 // clang-format on
552 }
553
554 static const MVT::SimpleValueType FloatVectorTypes[] = {
555 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
556 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
557
558 for (MVT VT : FloatVectorTypes) {
559 setOperationAction(
560 Ops: {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM,
561 ISD::FADD, ISD::FCEIL, ISD::FCOS,
562 ISD::FDIV, ISD::FEXP2, ISD::FEXP,
563 ISD::FEXP10, ISD::FLOG2, ISD::FREM,
564 ISD::FLOG, ISD::FLOG10, ISD::FPOW,
565 ISD::FFLOOR, ISD::FTRUNC, ISD::FMUL,
566 ISD::FMA, ISD::FRINT, ISD::FNEARBYINT,
567 ISD::FSQRT, ISD::FSIN, ISD::FSUB,
568 ISD::FNEG, ISD::VSELECT, ISD::SELECT_CC,
569 ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, ISD::SETCC,
570 ISD::FCANONICALIZE, ISD::FROUNDEVEN},
571 VT, Action: Expand);
572 }
573
574 // This causes using an unrolled select operation rather than expansion with
575 // bit operations. This is in general better, but the alternative using BFI
576 // instructions may be better if the select sources are SGPRs.
577 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f32, Action: Promote);
578 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
579
580 setOperationAction(Op: ISD::SELECT, VT: MVT::v3f32, Action: Promote);
581 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
582
583 setOperationAction(Op: ISD::SELECT, VT: MVT::v4f32, Action: Promote);
584 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
585
586 setOperationAction(Op: ISD::SELECT, VT: MVT::v5f32, Action: Promote);
587 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
588
589 setOperationAction(Op: ISD::SELECT, VT: MVT::v6f32, Action: Promote);
590 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
591
592 setOperationAction(Op: ISD::SELECT, VT: MVT::v7f32, Action: Promote);
593 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
594
595 setOperationAction(Op: ISD::SELECT, VT: MVT::v9f32, Action: Promote);
596 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
597
598 setOperationAction(Op: ISD::SELECT, VT: MVT::v10f32, Action: Promote);
599 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
600
601 setOperationAction(Op: ISD::SELECT, VT: MVT::v11f32, Action: Promote);
602 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
603
604 setOperationAction(Op: ISD::SELECT, VT: MVT::v12f32, Action: Promote);
605 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
606
607 setSchedulingPreference(Sched::RegPressure);
608 setJumpIsExpensive(true);
609
610 setMinCmpXchgSizeInBits(32);
611 setSupportsUnalignedAtomics(false);
612
613 PredictableSelectIsExpensive = false;
614
615 // We want to find all load dependencies for long chains of stores to enable
616 // merging into very wide vectors. The problem is with vectors with > 4
617 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
618 // vectors are a legal type, even though we have to split the loads
619 // usually. When we can more precisely specify load legality per address
620 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
621 // smarter so that they can figure out what to do in 2 iterations without all
622 // N > 4 stores on the same chain.
623 GatherAllAliasesMaxDepth = 16;
624
625 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
626 // about these during lowering.
627 MaxStoresPerMemcpy = 0xffffffff;
628 MaxStoresPerMemmove = 0xffffffff;
629 MaxStoresPerMemset = 0xffffffff;
630
631 // The expansion for 64-bit division is enormous.
632 if (AMDGPUBypassSlowDiv)
633 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
634
635 setTargetDAGCombine({ISD::BITCAST, ISD::SHL,
636 ISD::SRA, ISD::SRL,
637 ISD::TRUNCATE, ISD::MUL,
638 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
639 ISD::MULHU, ISD::MULHS,
640 ISD::SELECT, ISD::SELECT_CC,
641 ISD::STORE, ISD::FADD,
642 ISD::FSUB, ISD::FNEG,
643 ISD::FABS, ISD::AssertZext,
644 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN});
645
646 setMaxAtomicSizeInBitsSupported(64);
647 setMaxDivRemBitWidthSupported(64);
648 setMaxLargeFPConvertBitWidthSupported(64);
649}
650
651bool AMDGPUTargetLowering::mayIgnoreSignedZero(SDValue Op) const {
652 const auto Flags = Op.getNode()->getFlags();
653 if (Flags.hasNoSignedZeros())
654 return true;
655
656 return false;
657}
658
659//===----------------------------------------------------------------------===//
660// Target Information
661//===----------------------------------------------------------------------===//
662
663LLVM_READNONE
664static bool fnegFoldsIntoOpcode(unsigned Opc) {
665 switch (Opc) {
666 case ISD::FADD:
667 case ISD::FSUB:
668 case ISD::FMUL:
669 case ISD::FMA:
670 case ISD::FMAD:
671 case ISD::FMINNUM:
672 case ISD::FMAXNUM:
673 case ISD::FMINNUM_IEEE:
674 case ISD::FMAXNUM_IEEE:
675 case ISD::FMINIMUM:
676 case ISD::FMAXIMUM:
677 case ISD::FMINIMUMNUM:
678 case ISD::FMAXIMUMNUM:
679 case ISD::SELECT:
680 case ISD::FSIN:
681 case ISD::FTRUNC:
682 case ISD::FRINT:
683 case ISD::FNEARBYINT:
684 case ISD::FROUNDEVEN:
685 case ISD::FCANONICALIZE:
686 case AMDGPUISD::RCP:
687 case AMDGPUISD::RCP_LEGACY:
688 case AMDGPUISD::RCP_IFLAG:
689 case AMDGPUISD::SIN_HW:
690 case AMDGPUISD::FMUL_LEGACY:
691 case AMDGPUISD::FMIN_LEGACY:
692 case AMDGPUISD::FMAX_LEGACY:
693 case AMDGPUISD::FMED3:
694 // TODO: handle llvm.amdgcn.fma.legacy
695 return true;
696 case ISD::BITCAST:
697 llvm_unreachable("bitcast is special cased");
698 default:
699 return false;
700 }
701}
702
703static bool fnegFoldsIntoOp(const SDNode *N) {
704 unsigned Opc = N->getOpcode();
705 if (Opc == ISD::BITCAST) {
706 // TODO: Is there a benefit to checking the conditions performFNegCombine
707 // does? We don't for the other cases.
708 SDValue BCSrc = N->getOperand(Num: 0);
709 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
710 return BCSrc.getNumOperands() == 2 &&
711 BCSrc.getOperand(i: 1).getValueSizeInBits() == 32;
712 }
713
714 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32;
715 }
716
717 return fnegFoldsIntoOpcode(Opc);
718}
719
720/// \p returns true if the operation will definitely need to use a 64-bit
721/// encoding, and thus will use a VOP3 encoding regardless of the source
722/// modifiers.
723LLVM_READONLY
724static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
725 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) ||
726 VT == MVT::f64;
727}
728
729/// Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the
730/// type for ISD::SELECT.
731LLVM_READONLY
732static bool selectSupportsSourceMods(const SDNode *N) {
733 // TODO: Only applies if select will be vector
734 return N->getValueType(ResNo: 0) == MVT::f32;
735}
736
737// Most FP instructions support source modifiers, but this could be refined
738// slightly.
739LLVM_READONLY
740static bool hasSourceMods(const SDNode *N) {
741 if (isa<MemSDNode>(Val: N))
742 return false;
743
744 switch (N->getOpcode()) {
745 case ISD::CopyToReg:
746 case ISD::FDIV:
747 case ISD::FREM:
748 case ISD::INLINEASM:
749 case ISD::INLINEASM_BR:
750 case AMDGPUISD::DIV_SCALE:
751 case ISD::INTRINSIC_W_CHAIN:
752
753 // TODO: Should really be looking at the users of the bitcast. These are
754 // problematic because bitcasts are used to legalize all stores to integer
755 // types.
756 case ISD::BITCAST:
757 return false;
758 case ISD::INTRINSIC_WO_CHAIN: {
759 switch (N->getConstantOperandVal(Num: 0)) {
760 case Intrinsic::amdgcn_interp_p1:
761 case Intrinsic::amdgcn_interp_p2:
762 case Intrinsic::amdgcn_interp_mov:
763 case Intrinsic::amdgcn_interp_p1_f16:
764 case Intrinsic::amdgcn_interp_p2_f16:
765 return false;
766 default:
767 return true;
768 }
769 }
770 case ISD::SELECT:
771 return selectSupportsSourceMods(N);
772 default:
773 return true;
774 }
775}
776
777bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N,
778 unsigned CostThreshold) {
779 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
780 // it is truly free to use a source modifier in all cases. If there are
781 // multiple users but for each one will necessitate using VOP3, there will be
782 // a code size increase. Try to avoid increasing code size unless we know it
783 // will save on the instruction count.
784 unsigned NumMayIncreaseSize = 0;
785 MVT VT = N->getValueType(ResNo: 0).getScalarType().getSimpleVT();
786
787 assert(!N->use_empty());
788
789 // XXX - Should this limit number of uses to check?
790 for (const SDNode *U : N->users()) {
791 if (!hasSourceMods(N: U))
792 return false;
793
794 if (!opMustUseVOP3Encoding(N: U, VT)) {
795 if (++NumMayIncreaseSize > CostThreshold)
796 return false;
797 }
798 }
799
800 return true;
801}
802
803EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
804 ISD::NodeType ExtendKind) const {
805 assert(!VT.isVector() && "only scalar expected");
806
807 // Round to the next multiple of 32-bits.
808 unsigned Size = VT.getSizeInBits();
809 if (Size <= 32)
810 return MVT::i32;
811 return EVT::getIntegerVT(Context, BitWidth: 32 * ((Size + 31) / 32));
812}
813
814unsigned AMDGPUTargetLowering::getVectorIdxWidth(const DataLayout &) const {
815 return 32;
816}
817
818bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
819 return true;
820}
821
822// The backend supports 32 and 64 bit floating point immediates.
823// FIXME: Why are we reporting vectors of FP immediates as legal?
824bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
825 bool ForCodeSize) const {
826 return isTypeLegal(VT: VT.getScalarType());
827}
828
829// We don't want to shrink f64 / f32 constants.
830bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
831 EVT ScalarVT = VT.getScalarType();
832 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
833}
834
835bool AMDGPUTargetLowering::shouldReduceLoadWidth(
836 SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
837 std::optional<unsigned> ByteOffset) const {
838 // TODO: This may be worth removing. Check regression tests for diffs.
839 if (!TargetLoweringBase::shouldReduceLoadWidth(Load: N, ExtTy, NewVT, ByteOffset))
840 return false;
841
842 unsigned NewSize = NewVT.getStoreSizeInBits();
843
844 // If we are reducing to a 32-bit load or a smaller multi-dword load,
845 // this is always better.
846 if (NewSize >= 32)
847 return true;
848
849 EVT OldVT = N->getValueType(ResNo: 0);
850 unsigned OldSize = OldVT.getStoreSizeInBits();
851
852 MemSDNode *MN = cast<MemSDNode>(Val: N);
853 unsigned AS = MN->getAddressSpace();
854 // Do not shrink an aligned scalar load to sub-dword.
855 // Scalar engine cannot do sub-dword loads.
856 // TODO: Update this for GFX12 which does have scalar sub-dword loads.
857 if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) &&
858 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
859 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
860 (isa<LoadSDNode>(Val: N) && AS == AMDGPUAS::GLOBAL_ADDRESS &&
861 MN->isInvariant())) &&
862 AMDGPU::isUniformMMO(MMO: MN->getMemOperand()))
863 return false;
864
865 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
866 // extloads, so doing one requires using a buffer_load. In cases where we
867 // still couldn't use a scalar load, using the wider load shouldn't really
868 // hurt anything.
869
870 // If the old size already had to be an extload, there's no harm in continuing
871 // to reduce the width.
872 return (OldSize < 32);
873}
874
875bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
876 const SelectionDAG &DAG,
877 const MachineMemOperand &MMO) const {
878
879 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
880
881 if (LoadTy.getScalarType() == MVT::i32)
882 return false;
883
884 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
885 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
886
887 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
888 return false;
889
890 unsigned Fast = 0;
891 return allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
892 VT: CastTy, MMO, Fast: &Fast) &&
893 Fast;
894}
895
896// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
897// profitable with the expansion for 64-bit since it's generally good to
898// speculate things.
899bool AMDGPUTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
900 return true;
901}
902
903bool AMDGPUTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
904 return true;
905}
906
907bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const {
908 switch (N->getOpcode()) {
909 case ISD::EntryToken:
910 case ISD::TokenFactor:
911 return true;
912 case ISD::INTRINSIC_WO_CHAIN: {
913 unsigned IntrID = N->getConstantOperandVal(Num: 0);
914 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
915 }
916 case ISD::INTRINSIC_W_CHAIN: {
917 unsigned IntrID = N->getConstantOperandVal(Num: 1);
918 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
919 }
920 case ISD::LOAD:
921 if (cast<LoadSDNode>(Val: N)->getMemOperand()->getAddrSpace() ==
922 AMDGPUAS::CONSTANT_ADDRESS_32BIT)
923 return true;
924 return false;
925 case AMDGPUISD::SETCC: // ballot-style instruction
926 return true;
927 }
928 return false;
929}
930
931SDValue AMDGPUTargetLowering::getNegatedExpression(
932 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
933 NegatibleCost &Cost, unsigned Depth) const {
934
935 switch (Op.getOpcode()) {
936 case ISD::FMA:
937 case ISD::FMAD: {
938 // Negating a fma is not free if it has users without source mods.
939 if (!allUsesHaveSourceMods(N: Op.getNode()))
940 return SDValue();
941 break;
942 }
943 case AMDGPUISD::RCP: {
944 SDValue Src = Op.getOperand(i: 0);
945 EVT VT = Op.getValueType();
946 SDLoc SL(Op);
947
948 SDValue NegSrc = getNegatedExpression(Op: Src, DAG, LegalOperations,
949 ForCodeSize, Cost, Depth: Depth + 1);
950 if (NegSrc)
951 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: NegSrc, Flags: Op->getFlags());
952 return SDValue();
953 }
954 default:
955 break;
956 }
957
958 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps: LegalOperations,
959 OptForSize: ForCodeSize, Cost, Depth);
960}
961
962//===---------------------------------------------------------------------===//
963// Target Properties
964//===---------------------------------------------------------------------===//
965
966bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
967 assert(VT.isFloatingPoint());
968
969 // Packed operations do not have a fabs modifier.
970 // Report this based on the end legalized type.
971 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
972}
973
974bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
975 assert(VT.isFloatingPoint());
976 // Report this based on the end legalized type.
977 VT = VT.getScalarType();
978 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
979}
980
981bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
982 unsigned NumElem,
983 unsigned AS) const {
984 return true;
985}
986
987bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
988 // There are few operations which truly have vector input operands. Any vector
989 // operation is going to involve operations on each component, and a
990 // build_vector will be a copy per element, so it always makes sense to use a
991 // build_vector input in place of the extracted element to avoid a copy into a
992 // super register.
993 //
994 // We should probably only do this if all users are extracts only, but this
995 // should be the common case.
996 return true;
997}
998
999bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
1000 // Truncate is just accessing a subregister.
1001
1002 unsigned SrcSize = Source.getSizeInBits();
1003 unsigned DestSize = Dest.getSizeInBits();
1004
1005 return DestSize < SrcSize && DestSize % 32 == 0 ;
1006}
1007
1008bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
1009 // Truncate is just accessing a subregister.
1010
1011 unsigned SrcSize = Source->getScalarSizeInBits();
1012 unsigned DestSize = Dest->getScalarSizeInBits();
1013
1014 if (DestSize== 16 && Subtarget->has16BitInsts())
1015 return SrcSize >= 32;
1016
1017 return DestSize < SrcSize && DestSize % 32 == 0;
1018}
1019
1020bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
1021 unsigned SrcSize = Src->getScalarSizeInBits();
1022 unsigned DestSize = Dest->getScalarSizeInBits();
1023
1024 if (SrcSize == 16 && Subtarget->has16BitInsts())
1025 return DestSize >= 32;
1026
1027 return SrcSize == 32 && DestSize == 64;
1028}
1029
1030bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
1031 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
1032 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
1033 // this will enable reducing 64-bit operations the 32-bit, which is always
1034 // good.
1035
1036 if (Src == MVT::i16)
1037 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1038
1039 return Src == MVT::i32 && Dest == MVT::i64;
1040}
1041
1042bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
1043 EVT DestVT) const {
1044 switch (N->getOpcode()) {
1045 case ISD::ABS:
1046 case ISD::ADD:
1047 case ISD::SUB:
1048 case ISD::SHL:
1049 case ISD::SRL:
1050 case ISD::SRA:
1051 case ISD::AND:
1052 case ISD::OR:
1053 case ISD::XOR:
1054 case ISD::MUL:
1055 case ISD::SETCC:
1056 case ISD::SELECT:
1057 case ISD::SMIN:
1058 case ISD::SMAX:
1059 case ISD::UMIN:
1060 case ISD::UMAX:
1061 case ISD::USUBSAT:
1062 if (isTypeLegal(VT: MVT::i16) &&
1063 (!DestVT.isVector() ||
1064 !isOperationLegal(Op: ISD::ADD, VT: MVT::v2i16))) { // Check if VOP3P
1065 // Don't narrow back down to i16 if promoted to i32 already.
1066 if (!N->isDivergent() && DestVT.isInteger() &&
1067 DestVT.getScalarSizeInBits() > 1 &&
1068 DestVT.getScalarSizeInBits() <= 16 &&
1069 SrcVT.getScalarSizeInBits() > 16) {
1070 return false;
1071 }
1072 }
1073 return true;
1074 default:
1075 break;
1076 }
1077
1078 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
1079 // limited number of native 64-bit operations. Shrinking an operation to fit
1080 // in a single 32-bit register should always be helpful. As currently used,
1081 // this is much less general than the name suggests, and is only used in
1082 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
1083 // not profitable, and may actually be harmful.
1084 if (isa<LoadSDNode>(Val: N))
1085 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
1086
1087 return true;
1088}
1089
1090bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
1091 const SDNode* N, CombineLevel Level) const {
1092 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
1093 N->getOpcode() == ISD::SRL) &&
1094 "Expected shift op");
1095
1096 SDValue ShiftLHS = N->getOperand(Num: 0);
1097 if (!ShiftLHS->hasOneUse())
1098 return false;
1099
1100 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
1101 !ShiftLHS.getOperand(i: 0)->hasOneUse())
1102 return false;
1103
1104 // Always commute pre-type legalization and right shifts.
1105 // We're looking for shl(or(x,y),z) patterns.
1106 if (Level < CombineLevel::AfterLegalizeTypes ||
1107 N->getOpcode() != ISD::SHL || N->getOperand(Num: 0).getOpcode() != ISD::OR)
1108 return true;
1109
1110 // If only user is a i32 right-shift, then don't destroy a BFE pattern.
1111 if (N->getValueType(ResNo: 0) == MVT::i32 && N->hasOneUse() &&
1112 (N->user_begin()->getOpcode() == ISD::SRA ||
1113 N->user_begin()->getOpcode() == ISD::SRL))
1114 return false;
1115
1116 // Don't destroy or(shl(load_zext(),c), load_zext()) patterns.
1117 auto IsShiftAndLoad = [](SDValue LHS, SDValue RHS) {
1118 if (LHS.getOpcode() != ISD::SHL)
1119 return false;
1120 auto *RHSLd = dyn_cast<LoadSDNode>(Val&: RHS);
1121 auto *LHS0 = dyn_cast<LoadSDNode>(Val: LHS.getOperand(i: 0));
1122 auto *LHS1 = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
1123 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD &&
1124 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1125 RHSLd->getExtensionType() == ISD::ZEXTLOAD;
1126 };
1127 SDValue LHS = N->getOperand(Num: 0).getOperand(i: 0);
1128 SDValue RHS = N->getOperand(Num: 0).getOperand(i: 1);
1129 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS));
1130}
1131
1132//===---------------------------------------------------------------------===//
1133// TargetLowering Callbacks
1134//===---------------------------------------------------------------------===//
1135
1136CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
1137 bool IsVarArg) {
1138 switch (CC) {
1139 case CallingConv::AMDGPU_VS:
1140 case CallingConv::AMDGPU_GS:
1141 case CallingConv::AMDGPU_PS:
1142 case CallingConv::AMDGPU_CS:
1143 case CallingConv::AMDGPU_HS:
1144 case CallingConv::AMDGPU_ES:
1145 case CallingConv::AMDGPU_LS:
1146 return CC_AMDGPU;
1147 case CallingConv::AMDGPU_CS_Chain:
1148 case CallingConv::AMDGPU_CS_ChainPreserve:
1149 return CC_AMDGPU_CS_CHAIN;
1150 case CallingConv::C:
1151 case CallingConv::Fast:
1152 case CallingConv::Cold:
1153 return CC_AMDGPU_Func;
1154 case CallingConv::AMDGPU_Gfx:
1155 case CallingConv::AMDGPU_Gfx_WholeWave:
1156 return CC_SI_Gfx;
1157 case CallingConv::AMDGPU_KERNEL:
1158 case CallingConv::SPIR_KERNEL:
1159 default:
1160 reportFatalUsageError(reason: "unsupported calling convention for call");
1161 }
1162}
1163
1164CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
1165 bool IsVarArg) {
1166 switch (CC) {
1167 case CallingConv::AMDGPU_KERNEL:
1168 case CallingConv::SPIR_KERNEL:
1169 llvm_unreachable("kernels should not be handled here");
1170 case CallingConv::AMDGPU_VS:
1171 case CallingConv::AMDGPU_GS:
1172 case CallingConv::AMDGPU_PS:
1173 case CallingConv::AMDGPU_CS:
1174 case CallingConv::AMDGPU_CS_Chain:
1175 case CallingConv::AMDGPU_CS_ChainPreserve:
1176 case CallingConv::AMDGPU_HS:
1177 case CallingConv::AMDGPU_ES:
1178 case CallingConv::AMDGPU_LS:
1179 return RetCC_SI_Shader;
1180 case CallingConv::AMDGPU_Gfx:
1181 case CallingConv::AMDGPU_Gfx_WholeWave:
1182 return RetCC_SI_Gfx;
1183 case CallingConv::C:
1184 case CallingConv::Fast:
1185 case CallingConv::Cold:
1186 return RetCC_AMDGPU_Func;
1187 default:
1188 reportFatalUsageError(reason: "unsupported calling convention");
1189 }
1190}
1191
1192/// The SelectionDAGBuilder will automatically promote function arguments
1193/// with illegal types. However, this does not work for the AMDGPU targets
1194/// since the function arguments are stored in memory as these illegal types.
1195/// In order to handle this properly we need to get the original types sizes
1196/// from the LLVM IR Function and fixup the ISD:InputArg values before
1197/// passing them to AnalyzeFormalArguments()
1198
1199/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
1200/// input values across multiple registers. Each item in the Ins array
1201/// represents a single value that will be stored in registers. Ins[x].VT is
1202/// the value type of the value that will be stored in the register, so
1203/// whatever SDNode we lower the argument to needs to be this type.
1204///
1205/// In order to correctly lower the arguments we need to know the size of each
1206/// argument. Since Ins[x].VT gives us the size of the register that will
1207/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
1208/// for the original function argument so that we can deduce the correct memory
1209/// type to use for Ins[x]. In most cases the correct memory type will be
1210/// Ins[x].ArgVT. However, this will not always be the case. If, for example,
1211/// we have a kernel argument of type v8i8, this argument will be split into
1212/// 8 parts and each part will be represented by its own item in the Ins array.
1213/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
1214/// the argument before it was split. From this, we deduce that the memory type
1215/// for each individual part is i8. We pass the memory type as LocVT to the
1216/// calling convention analysis function and the register type (Ins[x].VT) as
1217/// the ValVT.
1218void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
1219 CCState &State,
1220 const SmallVectorImpl<ISD::InputArg> &Ins) const {
1221 const MachineFunction &MF = State.getMachineFunction();
1222 const Function &Fn = MF.getFunction();
1223 LLVMContext &Ctx = Fn.getContext();
1224 const unsigned ExplicitOffset = Subtarget->getExplicitKernelArgOffset();
1225 CallingConv::ID CC = Fn.getCallingConv();
1226
1227 Align MaxAlign = Align(1);
1228 uint64_t ExplicitArgOffset = 0;
1229 const DataLayout &DL = Fn.getDataLayout();
1230
1231 unsigned InIndex = 0;
1232
1233 for (const Argument &Arg : Fn.args()) {
1234 const bool IsByRef = Arg.hasByRefAttr();
1235 Type *BaseArgTy = Arg.getType();
1236 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1237 Align Alignment = DL.getValueOrABITypeAlignment(
1238 Alignment: IsByRef ? Arg.getParamAlign() : std::nullopt, Ty: MemArgTy);
1239 MaxAlign = std::max(a: Alignment, b: MaxAlign);
1240 uint64_t AllocSize = DL.getTypeAllocSize(Ty: MemArgTy);
1241
1242 uint64_t ArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + ExplicitOffset;
1243 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + AllocSize;
1244
1245 // We're basically throwing away everything passed into us and starting over
1246 // to get accurate in-memory offsets. The "PartOffset" is completely useless
1247 // to us as computed in Ins.
1248 //
1249 // We also need to figure out what type legalization is trying to do to get
1250 // the correct memory offsets.
1251
1252 SmallVector<EVT, 16> ValueVTs;
1253 SmallVector<uint64_t, 16> Offsets;
1254 ComputeValueVTs(TLI: *this, DL, Ty: BaseArgTy, ValueVTs, /*MemVTs=*/nullptr,
1255 FixedOffsets: &Offsets, StartingOffset: ArgOffset);
1256
1257 for (unsigned Value = 0, NumValues = ValueVTs.size();
1258 Value != NumValues; ++Value) {
1259 uint64_t BasePartOffset = Offsets[Value];
1260
1261 EVT ArgVT = ValueVTs[Value];
1262 EVT MemVT = ArgVT;
1263 MVT RegisterVT = getRegisterTypeForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1264 unsigned NumRegs = getNumRegistersForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1265
1266 if (NumRegs == 1) {
1267 // This argument is not split, so the IR type is the memory type.
1268 if (ArgVT.isExtended()) {
1269 // We have an extended type, like i24, so we should just use the
1270 // register type.
1271 MemVT = RegisterVT;
1272 } else {
1273 MemVT = ArgVT;
1274 }
1275 } else if (ArgVT.isVector() && RegisterVT.isVector() &&
1276 ArgVT.getScalarType() == RegisterVT.getScalarType()) {
1277 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
1278 // We have a vector value which has been split into a vector with
1279 // the same scalar type, but fewer elements. This should handle
1280 // all the floating-point vector types.
1281 MemVT = RegisterVT;
1282 } else if (ArgVT.isVector() &&
1283 ArgVT.getVectorNumElements() == NumRegs) {
1284 // This arg has been split so that each element is stored in a separate
1285 // register.
1286 MemVT = ArgVT.getScalarType();
1287 } else if (ArgVT.isExtended()) {
1288 // We have an extended type, like i65.
1289 MemVT = RegisterVT;
1290 } else {
1291 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
1292 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0);
1293 if (RegisterVT.isInteger()) {
1294 MemVT = EVT::getIntegerVT(Context&: State.getContext(), BitWidth: MemoryBits);
1295 } else if (RegisterVT.isVector()) {
1296 assert(!RegisterVT.getScalarType().isFloatingPoint());
1297 unsigned NumElements = RegisterVT.getVectorNumElements();
1298 assert(MemoryBits % NumElements == 0);
1299 // This vector type has been split into another vector type with
1300 // a different elements size.
1301 EVT ScalarVT = EVT::getIntegerVT(Context&: State.getContext(),
1302 BitWidth: MemoryBits / NumElements);
1303 MemVT = EVT::getVectorVT(Context&: State.getContext(), VT: ScalarVT, NumElements);
1304 } else {
1305 llvm_unreachable("cannot deduce memory type.");
1306 }
1307 }
1308
1309 // Convert one element vectors to scalar.
1310 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
1311 MemVT = MemVT.getScalarType();
1312
1313 // Round up vec3/vec5 argument.
1314 if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
1315 MemVT = MemVT.getPow2VectorType(Context&: State.getContext());
1316 } else if (!MemVT.isSimple() && !MemVT.isVector()) {
1317 MemVT = MemVT.getRoundIntegerType(Context&: State.getContext());
1318 }
1319
1320 unsigned PartOffset = 0;
1321 for (unsigned i = 0; i != NumRegs; ++i) {
1322 State.addLoc(V: CCValAssign::getCustomMem(ValNo: InIndex++, ValVT: RegisterVT,
1323 Offset: BasePartOffset + PartOffset,
1324 LocVT: MemVT.getSimpleVT(),
1325 HTP: CCValAssign::Full));
1326 PartOffset += MemVT.getStoreSize();
1327 }
1328 }
1329 }
1330}
1331
1332SDValue AMDGPUTargetLowering::LowerReturn(
1333 SDValue Chain, CallingConv::ID CallConv,
1334 bool isVarArg,
1335 const SmallVectorImpl<ISD::OutputArg> &Outs,
1336 const SmallVectorImpl<SDValue> &OutVals,
1337 const SDLoc &DL, SelectionDAG &DAG) const {
1338 // FIXME: Fails for r600 tests
1339 //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
1340 // "wave terminate should not have return values");
1341 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM, DL, VT: MVT::Other, Operand: Chain);
1342}
1343
1344//===---------------------------------------------------------------------===//
1345// Target specific lowering
1346//===---------------------------------------------------------------------===//
1347
1348/// Selects the correct CCAssignFn for a given CallingConvention value.
1349CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1350 bool IsVarArg) {
1351 return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1352}
1353
1354CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1355 bool IsVarArg) {
1356 return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1357}
1358
1359SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1360 SelectionDAG &DAG,
1361 MachineFrameInfo &MFI,
1362 int ClobberedFI) const {
1363 SmallVector<SDValue, 8> ArgChains;
1364 int64_t FirstByte = MFI.getObjectOffset(ObjectIdx: ClobberedFI);
1365 int64_t LastByte = FirstByte + MFI.getObjectSize(ObjectIdx: ClobberedFI) - 1;
1366
1367 // Include the original chain at the beginning of the list. When this is
1368 // used by target LowerCall hooks, this helps legalize find the
1369 // CALLSEQ_BEGIN node.
1370 ArgChains.push_back(Elt: Chain);
1371
1372 // Add a chain value for each stack argument corresponding
1373 for (SDNode *U : DAG.getEntryNode().getNode()->users()) {
1374 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U)) {
1375 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr())) {
1376 if (FI->getIndex() < 0) {
1377 int64_t InFirstByte = MFI.getObjectOffset(ObjectIdx: FI->getIndex());
1378 int64_t InLastByte = InFirstByte;
1379 InLastByte += MFI.getObjectSize(ObjectIdx: FI->getIndex()) - 1;
1380
1381 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1382 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1383 ArgChains.push_back(Elt: SDValue(L, 1));
1384 }
1385 }
1386 }
1387 }
1388
1389 // Build a tokenfactor for all the chains.
1390 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
1391}
1392
1393SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1394 SmallVectorImpl<SDValue> &InVals,
1395 StringRef Reason) const {
1396 SDValue Callee = CLI.Callee;
1397 SelectionDAG &DAG = CLI.DAG;
1398
1399 const Function &Fn = DAG.getMachineFunction().getFunction();
1400
1401 StringRef FuncName("<unknown>");
1402
1403 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Val&: Callee))
1404 FuncName = G->getSymbol();
1405 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
1406 FuncName = G->getGlobal()->getName();
1407
1408 DAG.getContext()->diagnose(
1409 DI: DiagnosticInfoUnsupported(Fn, Reason + FuncName, CLI.DL.getDebugLoc()));
1410
1411 if (!CLI.IsTailCall) {
1412 for (ISD::InputArg &Arg : CLI.Ins)
1413 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
1414 }
1415
1416 // FIXME: Hack because R600 doesn't handle callseq pseudos yet.
1417 if (getTargetMachine().getTargetTriple().getArch() == Triple::r600)
1418 return CLI.Chain;
1419
1420 SDValue Chain = DAG.getCALLSEQ_START(Chain: CLI.Chain, InSize: 0, OutSize: 0, DL: CLI.DL);
1421 return DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, /*InGlue=*/Glue: SDValue(), DL: CLI.DL);
1422}
1423
1424SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1425 SmallVectorImpl<SDValue> &InVals) const {
1426 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported call to function ");
1427}
1428
1429SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1430 SelectionDAG &DAG) const {
1431 const Function &Fn = DAG.getMachineFunction().getFunction();
1432
1433 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1434 Fn, "unsupported dynamic alloca", SDLoc(Op).getDebugLoc()));
1435 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()), Op.getOperand(i: 0)};
1436 return DAG.getMergeValues(Ops, dl: SDLoc());
1437}
1438
1439SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1440 SelectionDAG &DAG) const {
1441 switch (Op.getOpcode()) {
1442 default:
1443 Op->print(OS&: errs(), G: &DAG);
1444 llvm_unreachable("Custom lowering code for this "
1445 "instruction is not implemented yet!");
1446 break;
1447 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1448 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1449 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1450 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1451 case ISD::SDIVREM:
1452 return LowerSDIVREM(Op, DAG);
1453 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1454 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1455 case ISD::FRINT: return LowerFRINT(Op, DAG);
1456 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1457 case ISD::FROUNDEVEN:
1458 return LowerFROUNDEVEN(Op, DAG);
1459 case ISD::FROUND: return LowerFROUND(Op, DAG);
1460 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1461 case ISD::FLOG2:
1462 return LowerFLOG2(Op, DAG);
1463 case ISD::FLOG:
1464 case ISD::FLOG10:
1465 return LowerFLOGCommon(Op, DAG);
1466 case ISD::FEXP:
1467 case ISD::FEXP10:
1468 return lowerFEXP(Op, DAG);
1469 case ISD::FEXP2:
1470 return lowerFEXP2(Op, DAG);
1471 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1472 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1473 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1474 case ISD::FP_TO_SINT:
1475 case ISD::FP_TO_UINT:
1476 return LowerFP_TO_INT(Op, DAG);
1477 case ISD::FP_TO_SINT_SAT:
1478 case ISD::FP_TO_UINT_SAT:
1479 return LowerFP_TO_INT_SAT(Op, DAG);
1480 case ISD::CTTZ:
1481 case ISD::CTTZ_ZERO_POISON:
1482 case ISD::CTLZ:
1483 case ISD::CTLZ_ZERO_POISON:
1484 return LowerCTLZ_CTTZ(Op, DAG);
1485 case ISD::CTLS:
1486 return LowerCTLS(Op, DAG);
1487 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1488 }
1489 return Op;
1490}
1491
1492void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
1493 SmallVectorImpl<SDValue> &Results,
1494 SelectionDAG &DAG) const {
1495 switch (N->getOpcode()) {
1496 case ISD::SIGN_EXTEND_INREG:
1497 // Different parts of legalization seem to interpret which type of
1498 // sign_extend_inreg is the one to check for custom lowering. The extended
1499 // from type is what really matters, but some places check for custom
1500 // lowering of the result type. This results in trying to use
1501 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1502 // nothing here and let the illegal result integer be handled normally.
1503 return;
1504 case ISD::FLOG2:
1505 if (SDValue Lowered = LowerFLOG2(Op: SDValue(N, 0), DAG))
1506 Results.push_back(Elt: Lowered);
1507 return;
1508 case ISD::FLOG:
1509 case ISD::FLOG10:
1510 if (SDValue Lowered = LowerFLOGCommon(Op: SDValue(N, 0), DAG))
1511 Results.push_back(Elt: Lowered);
1512 return;
1513 case ISD::FEXP2:
1514 if (SDValue Lowered = lowerFEXP2(Op: SDValue(N, 0), DAG))
1515 Results.push_back(Elt: Lowered);
1516 return;
1517 case ISD::FEXP:
1518 case ISD::FEXP10:
1519 if (SDValue Lowered = lowerFEXP(Op: SDValue(N, 0), DAG))
1520 Results.push_back(Elt: Lowered);
1521 return;
1522 case ISD::CTLZ:
1523 case ISD::CTLZ_ZERO_POISON:
1524 if (auto Lowered = lowerCTLZResults(Op: SDValue(N, 0u), DAG))
1525 Results.push_back(Elt: Lowered);
1526 return;
1527 default:
1528 return;
1529 }
1530}
1531
1532SDValue AMDGPUTargetLowering::LowerBlockAddress(SDValue Op,
1533 SelectionDAG &DAG) const {
1534 BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val&: Op);
1535 SDLoc SL(Op);
1536 EVT VT = Op.getValueType();
1537 return DAG.getTargetBlockAddress(BA: BA->getBlockAddress(), VT, Offset: BA->getOffset(),
1538 TargetFlags: BA->getTargetFlags());
1539}
1540
1541SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
1542 SDValue Op,
1543 SelectionDAG &DAG) const {
1544
1545 const DataLayout &DL = DAG.getDataLayout();
1546 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Val&: Op);
1547 const GlobalValue *GV = G->getGlobal();
1548
1549 if (!MFI->isModuleEntryFunction()) {
1550 bool IsNamedBarrier = AMDGPU::isNamedBarrier(GV: *cast<GlobalVariable>(Val: GV));
1551 std::optional<uint32_t> Address =
1552 AMDGPUMachineFunctionInfo::getLDSAbsoluteAddress(GV: *GV);
1553 if (!Address && IsNamedBarrier)
1554 llvm_unreachable("named barrier should have an assigned address");
1555 if (Address) {
1556 if (IsNamedBarrier) {
1557 unsigned BarCnt = cast<GlobalVariable>(Val: GV)->getGlobalSize(DL) / 16;
1558 MFI->recordNumNamedBarriers(GVAddr: Address.value(), BarCnt);
1559 }
1560 // A constant byte offset (e.g. from a GEP into an array of named
1561 // barriers) folds directly into the fixed LDS address.
1562 return DAG.getConstant(Val: *Address + G->getOffset(), DL: SDLoc(Op),
1563 VT: Op.getValueType());
1564 }
1565 }
1566
1567 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1568 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
1569 if (!MFI->isModuleEntryFunction() &&
1570 GV->getName() != "llvm.amdgcn.module.lds" &&
1571 !AMDGPU::isNamedBarrier(GV: *cast<GlobalVariable>(Val: GV))) {
1572 SDLoc DL(Op);
1573 const Function &Fn = DAG.getMachineFunction().getFunction();
1574 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1575 Fn, "local memory global used by non-kernel function",
1576 DL.getDebugLoc(), DS_Warning));
1577
1578 // We currently don't have a way to correctly allocate LDS objects that
1579 // aren't directly associated with a kernel. We do force inlining of
1580 // functions that use local objects. However, if these dead functions are
1581 // not eliminated, we don't want a compile time error. Just emit a warning
1582 // and a trap, since there should be no callable path here.
1583 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL, VT: MVT::Other, Operand: DAG.getEntryNode());
1584 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
1585 N1: Trap, N2: DAG.getRoot());
1586 DAG.setRoot(OutputChain);
1587 return DAG.getPOISON(VT: Op.getValueType());
1588 }
1589
1590 // TODO: We could emit code to handle the initialization somewhere.
1591 // We ignore the initializer for now and legalize it to allow selection.
1592 // The initializer will anyway get errored out during assembly emission.
1593 unsigned Offset = MFI->allocateLDSGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1594 // A constant byte offset (e.g. from a GEP into an array of named barriers)
1595 // folds directly into the allocated LDS address.
1596 return DAG.getConstant(Val: Offset + G->getOffset(), DL: SDLoc(Op),
1597 VT: Op.getValueType());
1598 }
1599 return SDValue();
1600}
1601
1602SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1603 SelectionDAG &DAG) const {
1604 SmallVector<SDValue, 8> Args;
1605 SDLoc SL(Op);
1606
1607 EVT VT = Op.getValueType();
1608 if (VT.getVectorElementType().getSizeInBits() < 32) {
1609 unsigned OpBitSize = Op.getOperand(i: 0).getValueType().getSizeInBits();
1610 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1611 unsigned NewNumElt = OpBitSize / 32;
1612 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1613 : EVT::getVectorVT(Context&: *DAG.getContext(),
1614 VT: MVT::i32, NumElements: NewNumElt);
1615 for (const SDUse &U : Op->ops()) {
1616 SDValue In = U.get();
1617 SDValue NewIn = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewEltVT, Operand: In);
1618 if (NewNumElt > 1)
1619 DAG.ExtractVectorElements(Op: NewIn, Args);
1620 else
1621 Args.push_back(Elt: NewIn);
1622 }
1623
1624 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
1625 NumElements: NewNumElt * Op.getNumOperands());
1626 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1627 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: BV);
1628 }
1629 }
1630
1631 for (const SDUse &U : Op->ops())
1632 DAG.ExtractVectorElements(Op: U.get(), Args);
1633
1634 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1635}
1636
1637SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1638 SelectionDAG &DAG) const {
1639 SDLoc SL(Op);
1640 SmallVector<SDValue, 8> Args;
1641 unsigned Start = Op.getConstantOperandVal(i: 1);
1642 EVT VT = Op.getValueType();
1643 EVT SrcVT = Op.getOperand(i: 0).getValueType();
1644
1645 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
1646 unsigned NumElt = VT.getVectorNumElements();
1647 unsigned NumSrcElt = SrcVT.getVectorNumElements();
1648 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types");
1649
1650 // Extract 32-bit registers at a time.
1651 EVT NewSrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumSrcElt / 2);
1652 EVT NewVT = NumElt == 2
1653 ? MVT::i32
1654 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElt / 2);
1655 SDValue Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewSrcVT, Operand: Op.getOperand(i: 0));
1656
1657 DAG.ExtractVectorElements(Op: Tmp, Args, Start: Start / 2, Count: NumElt / 2);
1658 if (NumElt == 2)
1659 Tmp = Args[0];
1660 else
1661 Tmp = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1662
1663 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Tmp);
1664 }
1665
1666 DAG.ExtractVectorElements(Op: Op.getOperand(i: 0), Args, Start,
1667 Count: VT.getVectorNumElements());
1668
1669 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1670}
1671
1672// TODO: Handle fabs too
1673static SDValue peekFNeg(SDValue Val) {
1674 if (Val.getOpcode() == ISD::FNEG)
1675 return Val.getOperand(i: 0);
1676
1677 return Val;
1678}
1679
1680static SDValue peekFPSignOps(SDValue Val) {
1681 if (Val.getOpcode() == ISD::FNEG)
1682 Val = Val.getOperand(i: 0);
1683 if (Val.getOpcode() == ISD::FABS)
1684 Val = Val.getOperand(i: 0);
1685 if (Val.getOpcode() == ISD::FCOPYSIGN)
1686 Val = Val.getOperand(i: 0);
1687 return Val;
1688}
1689
1690SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl(
1691 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1692 SDValue False, SDValue CC, DAGCombinerInfo &DCI) const {
1693 SelectionDAG &DAG = DCI.DAG;
1694 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
1695 switch (CCOpcode) {
1696 case ISD::SETOEQ:
1697 case ISD::SETONE:
1698 case ISD::SETUNE:
1699 case ISD::SETNE:
1700 case ISD::SETUEQ:
1701 case ISD::SETEQ:
1702 case ISD::SETFALSE:
1703 case ISD::SETFALSE2:
1704 case ISD::SETTRUE:
1705 case ISD::SETTRUE2:
1706 case ISD::SETUO:
1707 case ISD::SETO:
1708 break;
1709 case ISD::SETULE:
1710 case ISD::SETULT: {
1711 if (LHS == True)
1712 return DAG.getNode(Opcode: AMDGPUISD::FMIN_LEGACY, DL, VT, N1: RHS, N2: LHS);
1713 return DAG.getNode(Opcode: AMDGPUISD::FMAX_LEGACY, DL, VT, N1: LHS, N2: RHS);
1714 }
1715 case ISD::SETOLE:
1716 case ISD::SETOLT:
1717 case ISD::SETLE:
1718 case ISD::SETLT: {
1719 // Ordered. Assume ordered for undefined.
1720
1721 // Only do this after legalization to avoid interfering with other combines
1722 // which might occur.
1723 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1724 !DCI.isCalledByLegalizer())
1725 return SDValue();
1726
1727 // We need to permute the operands to get the correct NaN behavior. The
1728 // selected operand is the second one based on the failing compare with NaN,
1729 // so permute it based on the compare type the hardware uses.
1730 if (LHS == True)
1731 return DAG.getNode(Opcode: AMDGPUISD::FMIN_LEGACY, DL, VT, N1: LHS, N2: RHS);
1732 return DAG.getNode(Opcode: AMDGPUISD::FMAX_LEGACY, DL, VT, N1: RHS, N2: LHS);
1733 }
1734 case ISD::SETUGE:
1735 case ISD::SETUGT: {
1736 if (LHS == True)
1737 return DAG.getNode(Opcode: AMDGPUISD::FMAX_LEGACY, DL, VT, N1: RHS, N2: LHS);
1738 return DAG.getNode(Opcode: AMDGPUISD::FMIN_LEGACY, DL, VT, N1: LHS, N2: RHS);
1739 }
1740 case ISD::SETGT:
1741 case ISD::SETGE:
1742 case ISD::SETOGE:
1743 case ISD::SETOGT: {
1744 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1745 !DCI.isCalledByLegalizer())
1746 return SDValue();
1747
1748 if (LHS == True)
1749 return DAG.getNode(Opcode: AMDGPUISD::FMAX_LEGACY, DL, VT, N1: LHS, N2: RHS);
1750 return DAG.getNode(Opcode: AMDGPUISD::FMIN_LEGACY, DL, VT, N1: RHS, N2: LHS);
1751 }
1752 case ISD::SETCC_INVALID:
1753 llvm_unreachable("Invalid setcc condcode!");
1754 }
1755 return SDValue();
1756}
1757
1758/// Generate Min/Max node
1759SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
1760 SDValue LHS, SDValue RHS,
1761 SDValue True, SDValue False,
1762 SDValue CC,
1763 DAGCombinerInfo &DCI) const {
1764 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
1765 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, DCI);
1766
1767 SelectionDAG &DAG = DCI.DAG;
1768
1769 // If we can't directly match this, try to see if we can fold an fneg to
1770 // match.
1771
1772 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
1773 ConstantFPSDNode *CFalse = dyn_cast<ConstantFPSDNode>(Val&: False);
1774 SDValue NegTrue = peekFNeg(Val: True);
1775
1776 // Undo the combine foldFreeOpFromSelect does if it helps us match the
1777 // fmin/fmax.
1778 //
1779 // select (fcmp olt (lhs, K)), (fneg lhs), -K
1780 // -> fneg (fmin_legacy lhs, K)
1781 //
1782 // TODO: Use getNegatedExpression
1783 if (LHS == NegTrue && CFalse && CRHS) {
1784 APFloat NegRHS = neg(X: CRHS->getValueAPF());
1785 if (NegRHS == CFalse->getValueAPF()) {
1786 SDValue Combined =
1787 combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True: NegTrue, False, CC, DCI);
1788 if (Combined)
1789 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Combined);
1790 return SDValue();
1791 }
1792 }
1793
1794 return SDValue();
1795}
1796
1797std::pair<SDValue, SDValue>
1798AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1799 SDLoc SL(Op);
1800
1801 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1802
1803 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1804 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1805
1806 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1807 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1808
1809 return std::pair(Lo, Hi);
1810}
1811
1812SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1813 SDLoc SL(Op);
1814
1815 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1816 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1817 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1818}
1819
1820SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1821 SDLoc SL(Op);
1822
1823 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1824 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1825 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1826}
1827
1828// Split a vector type into two parts. The first part is a power of two vector.
1829// The second part is whatever is left over, and is a scalar if it would
1830// otherwise be a 1-vector.
1831std::pair<EVT, EVT>
1832AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1833 EVT LoVT, HiVT;
1834 EVT EltVT = VT.getVectorElementType();
1835 unsigned NumElts = VT.getVectorNumElements();
1836 unsigned LoNumElts = PowerOf2Ceil(A: (NumElts + 1) / 2);
1837 LoVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: LoNumElts);
1838 HiVT = NumElts - LoNumElts == 1
1839 ? EltVT
1840 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts - LoNumElts);
1841 return std::pair(LoVT, HiVT);
1842}
1843
1844// Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1845// scalar.
1846std::pair<SDValue, SDValue>
1847AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1848 const EVT &LoVT, const EVT &HiVT,
1849 SelectionDAG &DAG) const {
1850 EVT VT = N.getValueType();
1851 assert(LoVT.getVectorNumElements() +
1852 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
1853 VT.getVectorNumElements() &&
1854 "More vector elements requested than available!");
1855 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N,
1856 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1857
1858 unsigned LoNumElts = LoVT.getVectorNumElements();
1859
1860 if (HiVT.isVector()) {
1861 unsigned HiNumElts = HiVT.getVectorNumElements();
1862 if ((VT.getVectorNumElements() % HiNumElts) == 0) {
1863 // Avoid creating an extract_subvector with an index that isn't a multiple
1864 // of the result type.
1865 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
1866 N2: DAG.getConstant(Val: LoNumElts, DL, VT: MVT::i32));
1867 return {Lo, Hi};
1868 }
1869
1870 SmallVector<SDValue, 8> Elts;
1871 DAG.ExtractVectorElements(Op: N, Args&: Elts, /*Start=*/LoNumElts,
1872 /*Count=*/HiNumElts);
1873 SDValue Hi = DAG.getBuildVector(VT: HiVT, DL, Ops: Elts);
1874 return {Lo, Hi};
1875 }
1876
1877 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: HiVT, N1: N,
1878 N2: DAG.getVectorIdxConstant(Val: LoNumElts, DL));
1879 return {Lo, Hi};
1880}
1881
1882SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1883 SelectionDAG &DAG) const {
1884 LoadSDNode *Load = cast<LoadSDNode>(Val: Op);
1885 EVT VT = Op.getValueType();
1886 SDLoc SL(Op);
1887
1888
1889 // If this is a 2 element vector, we really want to scalarize and not create
1890 // weird 1 element vectors.
1891 if (VT.getVectorNumElements() == 2) {
1892 SDValue Ops[2];
1893 std::tie(args&: Ops[0], args&: Ops[1]) = scalarizeVectorLoad(LD: Load, DAG);
1894 return DAG.getMergeValues(Ops, dl: SL);
1895 }
1896
1897 SDValue BasePtr = Load->getBasePtr();
1898 EVT MemVT = Load->getMemoryVT();
1899
1900 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1901
1902 EVT LoVT, HiVT;
1903 EVT LoMemVT, HiMemVT;
1904 SDValue Lo, Hi;
1905
1906 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1907 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1908 std::tie(args&: Lo, args&: Hi) = splitVector(N: Op, DL: SL, LoVT, HiVT, DAG);
1909
1910 unsigned Size = LoMemVT.getStoreSize();
1911 Align BaseAlign = Load->getAlign();
1912 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1913
1914 SDValue LoLoad = DAG.getExtLoad(
1915 ExtType: Load->getExtensionType(), dl: SL, VT: LoVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1916 MemVT: LoMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags(), AAInfo: Load->getAAInfo());
1917 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Size));
1918 SDValue HiLoad = DAG.getExtLoad(
1919 ExtType: Load->getExtensionType(), dl: SL, VT: HiVT, Chain: Load->getChain(), Ptr: HiPtr,
1920 PtrInfo: SrcValue.getWithOffset(O: LoMemVT.getStoreSize()), MemVT: HiMemVT, Alignment: HiAlign,
1921 MMOFlags: Load->getMemOperand()->getFlags(), AAInfo: Load->getAAInfo());
1922
1923 SDValue Join;
1924 if (LoVT == HiVT) {
1925 // This is the case that the vector is power of two so was evenly split.
1926 Join = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, N1: LoLoad, N2: HiLoad);
1927 } else {
1928 Join = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SL, VT, N1: DAG.getPOISON(VT), N2: LoLoad,
1929 N3: DAG.getVectorIdxConstant(Val: 0, DL: SL));
1930 Join = DAG.getNode(
1931 Opcode: HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, DL: SL,
1932 VT, N1: Join, N2: HiLoad,
1933 N3: DAG.getVectorIdxConstant(Val: LoVT.getVectorNumElements(), DL: SL));
1934 }
1935
1936 SDValue Ops[] = {Join, DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
1937 N1: LoLoad.getValue(R: 1), N2: HiLoad.getValue(R: 1))};
1938
1939 return DAG.getMergeValues(Ops, dl: SL);
1940}
1941
1942SDValue AMDGPUTargetLowering::WidenOrSplitVectorLoad(SDValue Op,
1943 SelectionDAG &DAG) const {
1944 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
1945 EVT VT = Op.getValueType();
1946 SDValue BasePtr = Load->getBasePtr();
1947 EVT MemVT = Load->getMemoryVT();
1948 SDLoc SL(Op);
1949 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1950 Align BaseAlign = Load->getAlign();
1951 unsigned NumElements = MemVT.getVectorNumElements();
1952
1953 // Widen from vec3 to vec4 when the load is at least 8-byte aligned
1954 // or 16-byte fully dereferenceable. Otherwise, split the vector load.
1955 if (NumElements != 3 ||
1956 (BaseAlign < Align(8) &&
1957 !SrcValue.isDereferenceable(Size: 16, C&: *DAG.getContext(), DL: DAG.getDataLayout())))
1958 return SplitVectorLoad(Op, DAG);
1959
1960 assert(NumElements == 3);
1961
1962 EVT WideVT =
1963 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
1964 EVT WideMemVT =
1965 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(), NumElements: 4);
1966 SDValue WideLoad = DAG.getExtLoad(
1967 ExtType: Load->getExtensionType(), dl: SL, VT: WideVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1968 MemVT: WideMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags());
1969 return DAG.getMergeValues(
1970 Ops: {DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT, N1: WideLoad,
1971 N2: DAG.getVectorIdxConstant(Val: 0, DL: SL)),
1972 WideLoad.getValue(R: 1)},
1973 dl: SL);
1974}
1975
1976SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1977 SelectionDAG &DAG) const {
1978 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
1979 SDValue Val = Store->getValue();
1980 EVT VT = Val.getValueType();
1981
1982 // If this is a 2 element vector, we really want to scalarize and not create
1983 // weird 1 element vectors.
1984 if (VT.getVectorNumElements() == 2)
1985 return scalarizeVectorStore(ST: Store, DAG);
1986
1987 EVT MemVT = Store->getMemoryVT();
1988 SDValue Chain = Store->getChain();
1989 SDValue BasePtr = Store->getBasePtr();
1990 SDLoc SL(Op);
1991
1992 EVT LoVT, HiVT;
1993 EVT LoMemVT, HiMemVT;
1994 SDValue Lo, Hi;
1995
1996 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1997 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1998 std::tie(args&: Lo, args&: Hi) = splitVector(N: Val, DL: SL, LoVT, HiVT, DAG);
1999
2000 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: LoMemVT.getStoreSize());
2001
2002 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
2003 Align BaseAlign = Store->getAlign();
2004 unsigned Size = LoMemVT.getStoreSize();
2005 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
2006
2007 SDValue LoStore =
2008 DAG.getTruncStore(Chain, dl: SL, Val: Lo, Ptr: BasePtr, PtrInfo: SrcValue, SVT: LoMemVT, Alignment: BaseAlign,
2009 MMOFlags: Store->getMemOperand()->getFlags(), AAInfo: Store->getAAInfo());
2010 SDValue HiStore = DAG.getTruncStore(
2011 Chain, dl: SL, Val: Hi, Ptr: HiPtr, PtrInfo: SrcValue.getWithOffset(O: Size), SVT: HiMemVT, Alignment: HiAlign,
2012 MMOFlags: Store->getMemOperand()->getFlags(), AAInfo: Store->getAAInfo());
2013
2014 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: LoStore, N2: HiStore);
2015}
2016
2017// This is a shortcut for integer division because we have fast i32<->f32
2018// conversions, and fast f32 reciprocal instructions.
2019SDValue AMDGPUTargetLowering::LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG,
2020 bool Sign) const {
2021 SDLoc DL(Op);
2022 EVT VT = Op.getValueType();
2023 assert(VT == MVT::i32 && "LowerDIVREMToFloat expects an i32");
2024
2025 SDValue LHS = Op.getOperand(i: 0);
2026 SDValue RHS = Op.getOperand(i: 1);
2027 MVT IntVT = MVT::i32;
2028 MVT FltVT = MVT::f32;
2029
2030 unsigned LHSSignBits;
2031 unsigned RHSSignBits;
2032 if (Sign) {
2033 LHSSignBits = DAG.ComputeNumSignBits(Op: LHS);
2034 RHSSignBits = DAG.ComputeNumSignBits(Op: RHS);
2035 if (LHSSignBits < 9 || RHSSignBits < 9)
2036 return SDValue();
2037 } else {
2038 KnownBits LHSKnown = DAG.computeKnownBits(Op: LHS);
2039 KnownBits RHSKnown = DAG.computeKnownBits(Op: RHS);
2040
2041 LHSSignBits = LHSKnown.countMinLeadingZeros();
2042 RHSSignBits = RHSKnown.countMinLeadingZeros();
2043 }
2044
2045 unsigned BitSize = VT.getSizeInBits();
2046 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
2047 unsigned DivBits = BitSize - SignBits;
2048 if (Sign)
2049 ++DivBits;
2050
2051 // In order to avoid problems due to 1 ulp accuracy issues with v_rcp_f32,
2052 // limit LowerDIVREMToFloat to:
2053 // [-0x400000,0x3FFFFF] for Sign
2054 // [ 0x000000,0x3FFFFF] for !Sign
2055 // This matches what is done in expandDivRemToFloatImpl.
2056 if (DivBits > (Sign ? 23 : 22))
2057 return SDValue();
2058
2059 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
2060 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
2061
2062 // int ia = (int)LHS;
2063 SDValue ia = LHS;
2064
2065 // int ib, (int)RHS;
2066 SDValue ib = RHS;
2067
2068 // The calculation:
2069 // fq = fa*recip(fb)
2070 // may be too small due to the 1ulp accuracy in the recip
2071 // operation and rounding issues. Since fq is truncated to produce
2072 // an integer value it may be too small by one. This is
2073 // dealt with by incrementing fa by 1ulp:
2074 // fq = (fa+1ulp)*recip(fb)
2075 // This will increase fa's magnitude by at most 0.5
2076 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
2077 // Thus, this method is safe since fa must be incremented by at least 1.0
2078 // for the quotient to increase by one.
2079 SDValue fa = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ia);
2080 SDValue faAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: fa);
2081 SDValue faIncremented = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: faAsInt,
2082 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
2083 fa = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FltVT, Operand: faIncremented);
2084
2085 // float fb = (float)ib;
2086 SDValue fb = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ib);
2087
2088 SDValue fq = DAG.getNode(Opcode: ISD::FMUL, DL, VT: FltVT,
2089 N1: fa, N2: DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: FltVT, Operand: fb));
2090
2091 // fq = trunc(fq);
2092 fq = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: FltVT, Operand: fq);
2093
2094 // int iq = (int)fq;
2095 SDValue Div = DAG.getNode(Opcode: ToInt, DL, VT: IntVT, Operand: fq);
2096
2097 // Rem needs compensation, it's easier to recompute it
2098 SDValue Rem = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Div, N2: RHS);
2099 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Rem);
2100
2101 return DAG.getMergeValues(Ops: { Div, Rem }, dl: DL);
2102}
2103
2104void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
2105 SelectionDAG &DAG,
2106 SmallVectorImpl<SDValue> &Results) const {
2107 SDLoc DL(Op);
2108 EVT VT = Op.getValueType();
2109
2110 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
2111
2112 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2113
2114 SDValue One = DAG.getConstant(Val: 1, DL, VT: HalfVT);
2115 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HalfVT);
2116
2117 //HiLo split
2118 SDValue LHS_Lo, LHS_Hi;
2119 SDValue LHS = Op.getOperand(i: 0);
2120 std::tie(args&: LHS_Lo, args&: LHS_Hi) = DAG.SplitScalar(N: LHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2121
2122 SDValue RHS_Lo, RHS_Hi;
2123 SDValue RHS = Op.getOperand(i: 1);
2124 std::tie(args&: RHS_Lo, args&: RHS_Hi) = DAG.SplitScalar(N: RHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2125
2126 if (DAG.MaskedValueIsZero(Op: RHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32)) &&
2127 DAG.MaskedValueIsZero(Op: LHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32))) {
2128
2129 SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2130 N1: LHS_Lo, N2: RHS_Lo);
2131
2132 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 0), Zero});
2133 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 1), Zero});
2134
2135 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV));
2136 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM));
2137 return;
2138 }
2139
2140 if (isTypeLegal(VT: MVT::i64)) {
2141 // The algorithm here is based on ideas from "Software Integer Division",
2142 // Tom Rodeheffer, August 2008.
2143
2144 MachineFunction &MF = DAG.getMachineFunction();
2145 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2146
2147 // Compute denominator reciprocal.
2148 unsigned FMAD =
2149 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA
2150 : MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign()
2151 ? (unsigned)ISD::FMAD
2152 : (unsigned)AMDGPUISD::FMAD_FTZ;
2153
2154 SDValue Cvt_Lo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Lo);
2155 SDValue Cvt_Hi = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Hi);
2156 SDValue Mad1 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Cvt_Hi,
2157 N2: DAG.getConstantFP(Val: APInt(32, 0x4f800000).bitsToFloat(), DL, VT: MVT::f32),
2158 N3: Cvt_Lo);
2159 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: MVT::f32, Operand: Mad1);
2160 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Rcp,
2161 N2: DAG.getConstantFP(Val: APInt(32, 0x5f7ffffc).bitsToFloat(), DL, VT: MVT::f32));
2162 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Mul1,
2163 N2: DAG.getConstantFP(Val: APInt(32, 0x2f800000).bitsToFloat(), DL, VT: MVT::f32));
2164 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: MVT::f32, Operand: Mul2);
2165 SDValue Mad2 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Trunc,
2166 N2: DAG.getConstantFP(Val: APInt(32, 0xcf800000).bitsToFloat(), DL, VT: MVT::f32),
2167 N3: Mul1);
2168 SDValue Rcp_Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Mad2);
2169 SDValue Rcp_Hi = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Trunc);
2170 SDValue Rcp64 = DAG.getBitcast(VT,
2171 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Rcp_Lo, Rcp_Hi}));
2172
2173 SDValue Zero64 = DAG.getConstant(Val: 0, DL, VT);
2174 SDValue One64 = DAG.getConstant(Val: 1, DL, VT);
2175 SDValue Zero1 = DAG.getConstant(Val: 0, DL, VT: MVT::i1);
2176 SDVTList HalfCarryVT = DAG.getVTList(VT1: HalfVT, VT2: MVT::i1);
2177
2178 // First round of UNR (Unsigned integer Newton-Raphson).
2179 SDValue Neg_RHS = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero64, N2: RHS);
2180 SDValue Mullo1 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Rcp64);
2181 SDValue Mulhi1 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Rcp64, N2: Mullo1);
2182 SDValue Mulhi1_Lo, Mulhi1_Hi;
2183 std::tie(args&: Mulhi1_Lo, args&: Mulhi1_Hi) =
2184 DAG.SplitScalar(N: Mulhi1, DL, LoVT: HalfVT, HiVT: HalfVT);
2185 SDValue Add1_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Lo,
2186 N2: Mulhi1_Lo, N3: Zero1);
2187 SDValue Add1_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Hi,
2188 N2: Mulhi1_Hi, N3: Add1_Lo.getValue(R: 1));
2189 SDValue Add1 = DAG.getBitcast(VT,
2190 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add1_Lo, Add1_Hi}));
2191
2192 // Second round of UNR.
2193 SDValue Mullo2 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Add1);
2194 SDValue Mulhi2 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Add1, N2: Mullo2);
2195 SDValue Mulhi2_Lo, Mulhi2_Hi;
2196 std::tie(args&: Mulhi2_Lo, args&: Mulhi2_Hi) =
2197 DAG.SplitScalar(N: Mulhi2, DL, LoVT: HalfVT, HiVT: HalfVT);
2198 SDValue Add2_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Lo,
2199 N2: Mulhi2_Lo, N3: Zero1);
2200 SDValue Add2_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Hi,
2201 N2: Mulhi2_Hi, N3: Add2_Lo.getValue(R: 1));
2202 SDValue Add2 = DAG.getBitcast(VT,
2203 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add2_Lo, Add2_Hi}));
2204
2205 SDValue Mulhi3 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: LHS, N2: Add2);
2206
2207 SDValue Mul3 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: RHS, N2: Mulhi3);
2208
2209 SDValue Mul3_Lo, Mul3_Hi;
2210 std::tie(args&: Mul3_Lo, args&: Mul3_Hi) = DAG.SplitScalar(N: Mul3, DL, LoVT: HalfVT, HiVT: HalfVT);
2211 SDValue Sub1_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Lo,
2212 N2: Mul3_Lo, N3: Zero1);
2213 SDValue Sub1_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Hi,
2214 N2: Mul3_Hi, N3: Sub1_Lo.getValue(R: 1));
2215 SDValue Sub1_Mi = DAG.getNode(Opcode: ISD::SUB, DL, VT: HalfVT, N1: LHS_Hi, N2: Mul3_Hi);
2216 SDValue Sub1 = DAG.getBitcast(VT,
2217 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub1_Lo, Sub1_Hi}));
2218
2219 SDValue MinusOne = DAG.getConstant(Val: 0xffffffffu, DL, VT: HalfVT);
2220 SDValue C1 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2221 Cond: ISD::SETUGE);
2222 SDValue C2 = DAG.getSelectCC(DL, LHS: Sub1_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2223 Cond: ISD::SETUGE);
2224 SDValue C3 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: C2, False: C1, Cond: ISD::SETEQ);
2225
2226 // TODO: Here and below portions of the code can be enclosed into if/endif.
2227 // Currently control flow is unconditional and we have 4 selects after
2228 // potential endif to substitute PHIs.
2229
2230 // if C3 != 0 ...
2231 SDValue Sub2_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Lo,
2232 N2: RHS_Lo, N3: Zero1);
2233 SDValue Sub2_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Mi,
2234 N2: RHS_Hi, N3: Sub1_Lo.getValue(R: 1));
2235 SDValue Sub2_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2236 N2: Zero, N3: Sub2_Lo.getValue(R: 1));
2237 SDValue Sub2 = DAG.getBitcast(VT,
2238 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub2_Lo, Sub2_Hi}));
2239
2240 SDValue Add3 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mulhi3, N2: One64);
2241
2242 SDValue C4 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2243 Cond: ISD::SETUGE);
2244 SDValue C5 = DAG.getSelectCC(DL, LHS: Sub2_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2245 Cond: ISD::SETUGE);
2246 SDValue C6 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: C5, False: C4, Cond: ISD::SETEQ);
2247
2248 // if (C6 != 0)
2249 SDValue Add4 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Add3, N2: One64);
2250
2251 SDValue Sub3_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Lo,
2252 N2: RHS_Lo, N3: Zero1);
2253 SDValue Sub3_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2254 N2: RHS_Hi, N3: Sub2_Lo.getValue(R: 1));
2255 SDValue Sub3_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub3_Mi,
2256 N2: Zero, N3: Sub3_Lo.getValue(R: 1));
2257 SDValue Sub3 = DAG.getBitcast(VT,
2258 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub3_Lo, Sub3_Hi}));
2259
2260 // endif C6
2261 // endif C3
2262
2263 SDValue Sel1 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Add4, False: Add3, Cond: ISD::SETNE);
2264 SDValue Div = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel1, False: Mulhi3, Cond: ISD::SETNE);
2265
2266 SDValue Sel2 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Sub3, False: Sub2, Cond: ISD::SETNE);
2267 SDValue Rem = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel2, False: Sub1, Cond: ISD::SETNE);
2268
2269 Results.push_back(Elt: Div);
2270 Results.push_back(Elt: Rem);
2271
2272 return;
2273 }
2274
2275 // r600 expandion.
2276 // Get Speculative values
2277 SDValue DIV_Part = DAG.getNode(Opcode: ISD::UDIV, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2278 SDValue REM_Part = DAG.getNode(Opcode: ISD::UREM, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2279
2280 SDValue REM_Lo = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: REM_Part, False: LHS_Hi, Cond: ISD::SETEQ);
2281 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {REM_Lo, Zero});
2282 REM = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM);
2283
2284 SDValue DIV_Hi = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: DIV_Part, False: Zero, Cond: ISD::SETEQ);
2285 SDValue DIV_Lo = Zero;
2286
2287 const unsigned halfBitWidth = HalfVT.getSizeInBits();
2288
2289 for (unsigned i = 0; i < halfBitWidth; ++i) {
2290 const unsigned bitPos = halfBitWidth - i - 1;
2291 SDValue POS = DAG.getConstant(Val: bitPos, DL, VT: HalfVT);
2292 // Get value of high bit
2293 SDValue HBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: HalfVT, N1: LHS_Lo, N2: POS);
2294 HBit = DAG.getNode(Opcode: ISD::AND, DL, VT: HalfVT, N1: HBit, N2: One);
2295 HBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: HBit);
2296
2297 // Shift
2298 REM = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: REM, N2: DAG.getConstant(Val: 1, DL, VT));
2299 // Add LHS high bit
2300 REM = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: REM, N2: HBit);
2301
2302 SDValue BIT = DAG.getConstant(Val: 1ULL << bitPos, DL, VT: HalfVT);
2303 SDValue realBIT = DAG.getSelectCC(DL, LHS: REM, RHS, True: BIT, False: Zero, Cond: ISD::SETUGE);
2304
2305 DIV_Lo = DAG.getNode(Opcode: ISD::OR, DL, VT: HalfVT, N1: DIV_Lo, N2: realBIT);
2306
2307 // Update REM
2308 SDValue REM_sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: REM, N2: RHS);
2309 REM = DAG.getSelectCC(DL, LHS: REM, RHS, True: REM_sub, False: REM, Cond: ISD::SETUGE);
2310 }
2311
2312 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {DIV_Lo, DIV_Hi});
2313 DIV = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV);
2314 Results.push_back(Elt: DIV);
2315 Results.push_back(Elt: REM);
2316}
2317
2318SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
2319 SelectionDAG &DAG) const {
2320 SDLoc DL(Op);
2321 EVT VT = Op.getValueType();
2322
2323 if (VT == MVT::i64) {
2324 SmallVector<SDValue, 2> Results;
2325 LowerUDIVREM64(Op, DAG, Results);
2326 return DAG.getMergeValues(Ops: Results, dl: DL);
2327 }
2328
2329 if (VT == MVT::i32) {
2330 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: false))
2331 return Res;
2332 }
2333
2334 SDValue X = Op.getOperand(i: 0);
2335 SDValue Y = Op.getOperand(i: 1);
2336
2337 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
2338 // algorithm used here.
2339
2340 // Initial estimate of inv(y).
2341 SDValue Z = DAG.getNode(Opcode: AMDGPUISD::URECIP, DL, VT, Operand: Y);
2342
2343 // One round of UNR.
2344 SDValue NegY = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Y);
2345 SDValue NegYZ = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: NegY, N2: Z);
2346 Z = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Z,
2347 N2: DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Z, N2: NegYZ));
2348
2349 // Quotient/remainder estimate.
2350 SDValue Q = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: X, N2: Z);
2351 SDValue R =
2352 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: X, N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Q, N2: Y));
2353
2354 // First quotient/remainder refinement.
2355 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2356 SDValue One = DAG.getConstant(Val: 1, DL, VT);
2357 SDValue Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2358 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2359 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2360 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2361 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2362
2363 // Second quotient/remainder refinement.
2364 Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2365 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2366 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2367 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2368 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2369
2370 return DAG.getMergeValues(Ops: {Q, R}, dl: DL);
2371}
2372
2373SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
2374 SelectionDAG &DAG) const {
2375 SDLoc DL(Op);
2376 EVT VT = Op.getValueType();
2377
2378 SDValue LHS = Op.getOperand(i: 0);
2379 SDValue RHS = Op.getOperand(i: 1);
2380
2381 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
2382 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2383
2384 if (VT == MVT::i32) {
2385 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: true))
2386 return Res;
2387 }
2388
2389 // LHS must have > 33 sign-bits to ensure that LHS != -2147483648
2390 // Otherwise 32-bit division cannot be used safely.
2391 // -2147483648/1 and -2147483648/-1 are not equal,
2392 // but they produce the same lower 32-bit result.
2393 if (VT == MVT::i64 && DAG.ComputeNumSignBits(Op: LHS) > 33 &&
2394 DAG.ComputeNumSignBits(Op: RHS) > 32) {
2395 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2396
2397 //HiLo split
2398 SDValue LHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: LHS, N2: Zero);
2399 SDValue RHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: RHS, N2: Zero);
2400 SDValue DIVREM = DAG.getNode(Opcode: ISD::SDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2401 N1: LHS_Lo, N2: RHS_Lo);
2402 SDValue Res[2] = {
2403 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 0)),
2404 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 1))
2405 };
2406 return DAG.getMergeValues(Ops: Res, dl: DL);
2407 }
2408
2409 SDValue LHSign = DAG.getSelectCC(DL, LHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2410 SDValue RHSign = DAG.getSelectCC(DL, LHS: RHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2411 SDValue DSign = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHSign, N2: RHSign);
2412 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2413
2414 LHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: LHSign);
2415 RHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: RHS, N2: RHSign);
2416
2417 LHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHS, N2: LHSign);
2418 RHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: RHS, N2: RHSign);
2419
2420 SDValue Div = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
2421 SDValue Rem = Div.getValue(R: 1);
2422
2423 Div = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Div, N2: DSign);
2424 Rem = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Rem, N2: RSign);
2425
2426 Div = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Div, N2: DSign);
2427 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Rem, N2: RSign);
2428
2429 SDValue Res[2] = {
2430 Div,
2431 Rem
2432 };
2433 return DAG.getMergeValues(Ops: Res, dl: DL);
2434}
2435
2436SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2437 SDLoc SL(Op);
2438 SDValue Src = Op.getOperand(i: 0);
2439
2440 // result = trunc(src)
2441 // if (src > 0.0 && src != result)
2442 // result += 1.0
2443
2444 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2445
2446 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2447 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
2448
2449 EVT SetCCVT =
2450 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2451
2452 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOGT);
2453 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2454 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2455
2456 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: One, N3: Zero);
2457 // TODO: Should this propagate fast-math-flags?
2458 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2459}
2460
2461static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2462 SelectionDAG &DAG) {
2463 const unsigned FractBits = 52;
2464 const unsigned ExpBits = 11;
2465
2466 SDValue ExpPart = DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32,
2467 N1: Hi,
2468 N2: DAG.getConstant(Val: FractBits - 32, DL: SL, VT: MVT::i32),
2469 N3: DAG.getConstant(Val: ExpBits, DL: SL, VT: MVT::i32));
2470 SDValue Exp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ExpPart,
2471 N2: DAG.getConstant(Val: 1023, DL: SL, VT: MVT::i32));
2472
2473 return Exp;
2474}
2475
2476SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2477 SDLoc SL(Op);
2478 SDValue Src = Op.getOperand(i: 0);
2479
2480 assert(Op.getValueType() == MVT::f64);
2481
2482 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
2483
2484 // Extract the upper half, since this is where we will find the sign and
2485 // exponent.
2486 SDValue Hi = getHiHalf64(Op: Src, DAG);
2487
2488 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2489
2490 const unsigned FractBits = 52;
2491
2492 // Extract the sign bit.
2493 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, DL: SL, VT: MVT::i32);
2494 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Hi, N2: SignBitMask);
2495
2496 // Extend back to 64-bits.
2497 SDValue SignBit64 = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Zero, SignBit});
2498 SignBit64 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: SignBit64);
2499
2500 SDValue BcInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Src);
2501 const SDValue FractMask
2502 = DAG.getConstant(Val: (UINT64_C(1) << FractBits) - 1, DL: SL, VT: MVT::i64);
2503
2504 SDValue Shr = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: FractMask, N2: Exp);
2505 SDValue Not = DAG.getNOT(DL: SL, Val: Shr, VT: MVT::i64);
2506 SDValue Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i64, N1: BcInt, N2: Not);
2507
2508 EVT SetCCVT =
2509 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i32);
2510
2511 const SDValue FiftyOne = DAG.getConstant(Val: FractBits - 1, DL: SL, VT: MVT::i32);
2512
2513 SDValue ExpLt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: Zero, Cond: ISD::SETLT);
2514 SDValue ExpGt51 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: FiftyOne, Cond: ISD::SETGT);
2515
2516 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpLt0, N2: SignBit64, N3: Tmp0);
2517 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpGt51, N2: BcInt, N3: Tmp1);
2518
2519 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f64, Operand: Tmp2);
2520}
2521
2522SDValue AMDGPUTargetLowering::LowerFROUNDEVEN(SDValue Op,
2523 SelectionDAG &DAG) const {
2524 SDLoc SL(Op);
2525 SDValue Src = Op.getOperand(i: 0);
2526
2527 assert(Op.getValueType() == MVT::f64);
2528
2529 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2530 SDValue C1 = DAG.getConstantFP(Val: C1Val, DL: SL, VT: MVT::f64);
2531 SDValue CopySign = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MVT::f64, N1: C1, N2: Src);
2532
2533 // TODO: Should this propagate fast-math-flags?
2534
2535 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Src, N2: CopySign);
2536 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT: MVT::f64, N1: Tmp1, N2: CopySign);
2537
2538 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f64, Operand: Src);
2539
2540 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2541 SDValue C2 = DAG.getConstantFP(Val: C2Val, DL: SL, VT: MVT::f64);
2542
2543 EVT SetCCVT =
2544 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2545 SDValue Cond = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Fabs, RHS: C2, Cond: ISD::SETOGT);
2546
2547 return DAG.getSelect(DL: SL, VT: MVT::f64, Cond, LHS: Src, RHS: Tmp2);
2548}
2549
2550SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op,
2551 SelectionDAG &DAG) const {
2552 // FNEARBYINT and FRINT are the same, except in their handling of FP
2553 // exceptions. Those aren't really meaningful for us, and OpenCL only has
2554 // rint, so just treat them as equivalent.
2555 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT: Op.getValueType(),
2556 Operand: Op.getOperand(i: 0));
2557}
2558
2559SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2560 auto VT = Op.getValueType();
2561 auto Arg = Op.getOperand(i: 0u);
2562 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT, Operand: Arg);
2563}
2564
2565// XXX - May require not supporting f32 denormals?
2566
2567// Don't handle v2f16. The extra instructions to scalarize and repack around the
2568// compare and vselect end up producing worse code than scalarizing the whole
2569// operation.
2570SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2571 SDLoc SL(Op);
2572 SDValue X = Op.getOperand(i: 0);
2573 EVT VT = Op.getValueType();
2574
2575 SDValue T = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: X);
2576
2577 // TODO: Should this propagate fast-math-flags?
2578
2579 SDValue Diff = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: T);
2580
2581 SDValue AbsDiff = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Diff);
2582
2583 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2584 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2585
2586 EVT SetCCVT =
2587 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2588
2589 const SDValue Half = DAG.getConstantFP(Val: 0.5, DL: SL, VT);
2590 SDValue Cmp = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsDiff, RHS: Half, Cond: ISD::SETOGE);
2591 SDValue OneOrZeroFP = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cmp, N2: One, N3: Zero);
2592
2593 SDValue SignedOffset = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: OneOrZeroFP, N2: X);
2594 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: T, N2: SignedOffset);
2595}
2596
2597SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2598 SDLoc SL(Op);
2599 SDValue Src = Op.getOperand(i: 0);
2600
2601 // result = trunc(src);
2602 // if (src < 0.0 && src != result)
2603 // result += -1.0.
2604
2605 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2606
2607 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2608 const SDValue NegOne = DAG.getConstantFP(Val: -1.0, DL: SL, VT: MVT::f64);
2609
2610 EVT SetCCVT =
2611 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2612
2613 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOLT);
2614 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2615 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2616
2617 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: NegOne, N3: Zero);
2618 // TODO: Should this propagate fast-math-flags?
2619 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2620}
2621
2622/// Return true if it's known that \p Src can never be an f32 denormal value.
2623static bool valueIsKnownNeverF32Denorm(SDValue Src) {
2624 switch (Src.getOpcode()) {
2625 case ISD::FP_EXTEND:
2626 return Src.getOperand(i: 0).getValueType() == MVT::f16;
2627 case ISD::FP16_TO_FP:
2628 case ISD::FFREXP:
2629 case ISD::FSQRT:
2630 case AMDGPUISD::LOG:
2631 case AMDGPUISD::EXP:
2632 return true;
2633 case ISD::INTRINSIC_WO_CHAIN: {
2634 unsigned IntrinsicID = Src.getConstantOperandVal(i: 0);
2635 switch (IntrinsicID) {
2636 case Intrinsic::amdgcn_frexp_mant:
2637 case Intrinsic::amdgcn_log:
2638 case Intrinsic::amdgcn_log_clamp:
2639 case Intrinsic::amdgcn_exp2:
2640 case Intrinsic::amdgcn_sqrt:
2641 return true;
2642 default:
2643 return false;
2644 }
2645 }
2646 default:
2647 return false;
2648 }
2649
2650 llvm_unreachable("covered opcode switch");
2651}
2652
2653bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG,
2654 SDNodeFlags Flags) {
2655 return Flags.hasApproximateFuncs();
2656}
2657
2658bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG,
2659 SDValue Src,
2660 SDNodeFlags Flags) {
2661 return !valueIsKnownNeverF32Denorm(Src) &&
2662 DAG.getMachineFunction()
2663 .getDenormalMode(FPType: APFloat::IEEEsingle())
2664 .Input != DenormalMode::PreserveSign;
2665}
2666
2667SDValue AMDGPUTargetLowering::getIsLtSmallestNormal(SelectionDAG &DAG,
2668 SDValue Src,
2669 SDNodeFlags Flags) const {
2670 SDLoc SL(Src);
2671 EVT VT = Src.getValueType();
2672 const fltSemantics &Semantics = VT.getFltSemantics();
2673 SDValue SmallestNormal =
2674 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2675
2676 // Want to scale denormals up, but negatives and 0 work just as well on the
2677 // scaled path.
2678 SDValue IsLtSmallestNormal = DAG.getSetCC(
2679 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2680 RHS: SmallestNormal, Cond: ISD::SETOLT);
2681
2682 return IsLtSmallestNormal;
2683}
2684
2685SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src,
2686 SDNodeFlags Flags) const {
2687 SDLoc SL(Src);
2688 EVT VT = Src.getValueType();
2689 const fltSemantics &Semantics = VT.getFltSemantics();
2690 SDValue Inf = DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL: SL, VT);
2691
2692 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Src, Flags);
2693 SDValue IsFinite = DAG.getSetCC(
2694 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Fabs,
2695 RHS: Inf, Cond: ISD::SETOLT);
2696 return IsFinite;
2697}
2698
2699/// If denormal handling is required return the scaled input to FLOG2, and the
2700/// check for denormal range. Otherwise, return null values.
2701std::pair<SDValue, SDValue>
2702AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL,
2703 SDValue Src, SDNodeFlags Flags) const {
2704 if (!needsDenormHandlingF32(DAG, Src, Flags))
2705 return {};
2706
2707 MVT VT = MVT::f32;
2708 const fltSemantics &Semantics = APFloat::IEEEsingle();
2709 SDValue SmallestNormal =
2710 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2711
2712 SDValue IsLtSmallestNormal = DAG.getSetCC(
2713 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2714 RHS: SmallestNormal, Cond: ISD::SETOLT);
2715
2716 SDValue Scale32 = DAG.getConstantFP(Val: 0x1.0p+32, DL: SL, VT);
2717 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2718 SDValue ScaleFactor =
2719 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: Scale32, N3: One, Flags);
2720
2721 SDValue ScaledInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Src, N2: ScaleFactor, Flags);
2722 return {ScaledInput, IsLtSmallestNormal};
2723}
2724
2725SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const {
2726 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
2727 // If we have to handle denormals, scale up the input and adjust the result.
2728
2729 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
2730 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
2731
2732 SDLoc SL(Op);
2733 EVT VT = Op.getValueType();
2734 SDValue Src = Op.getOperand(i: 0);
2735 SDNodeFlags Flags = Op->getFlags();
2736
2737 if (VT == MVT::f16) {
2738 // Nothing in half is a denormal when promoted to f32.
2739 assert(!isTypeLegal(VT));
2740 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
2741 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
2742 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
2743 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
2744 }
2745
2746 auto [ScaledInput, IsLtSmallestNormal] =
2747 getScaledLogInput(DAG, SL, Src, Flags);
2748 if (!ScaledInput)
2749 return DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: Src, Flags);
2750
2751 SDValue Log2 = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2752
2753 SDValue ThirtyTwo = DAG.getConstantFP(Val: 32.0, DL: SL, VT);
2754 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2755 SDValue ResultOffset =
2756 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: ThirtyTwo, N3: Zero);
2757 return DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: Log2, N2: ResultOffset, Flags);
2758}
2759
2760static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2761 SDValue Y, SDValue C, SDNodeFlags Flags = SDNodeFlags()) {
2762 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Y, Flags);
2763 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: C, Flags);
2764}
2765
2766SDValue AMDGPUTargetLowering::LowerFLOGCommon(SDValue Op,
2767 SelectionDAG &DAG) const {
2768 SDValue X = Op.getOperand(i: 0);
2769 EVT VT = Op.getValueType();
2770 SDNodeFlags Flags = Op->getFlags();
2771 SDLoc DL(Op);
2772 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10;
2773 assert(IsLog10 || Op.getOpcode() == ISD::FLOG);
2774
2775 if (VT == MVT::f16 || Flags.hasApproximateFuncs()) {
2776 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
2777 // depending on !fpmath metadata.
2778
2779 bool PromoteToF32 = VT == MVT::f16 && (!Flags.hasApproximateFuncs() ||
2780 !isTypeLegal(VT: MVT::f16));
2781
2782 if (PromoteToF32) {
2783 // Log and multiply in f32 is always good enough for f16.
2784 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2785 }
2786
2787 SDValue Lowered = LowerFLOGUnsafe(Op: X, SL: DL, DAG, IsLog10, Flags);
2788 if (PromoteToF32) {
2789 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Lowered,
2790 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32), Flags);
2791 }
2792
2793 return Lowered;
2794 }
2795
2796 SDValue ScaledInput, IsScaled;
2797 if (VT == MVT::f16)
2798 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2799 else {
2800 std::tie(args&: ScaledInput, args&: IsScaled) = getScaledLogInput(DAG, SL: DL, Src: X, Flags);
2801 if (ScaledInput)
2802 X = ScaledInput;
2803 }
2804
2805 SDValue Y = DAG.getNode(Opcode: AMDGPUISD::LOG, DL, VT, Operand: X, Flags);
2806
2807 SDValue R;
2808 if (Subtarget->hasFastFMAF32()) {
2809 // c+cc are ln(2)/ln(10) to more than 49 bits
2810 const float c_log10 = 0x1.344134p-2f;
2811 const float cc_log10 = 0x1.09f79ep-26f;
2812
2813 // c + cc is ln(2) to more than 49 bits
2814 const float c_log = 0x1.62e42ep-1f;
2815 const float cc_log = 0x1.efa39ep-25f;
2816
2817 SDValue C = DAG.getConstantFP(Val: IsLog10 ? c_log10 : c_log, DL, VT);
2818 SDValue CC = DAG.getConstantFP(Val: IsLog10 ? cc_log10 : cc_log, DL, VT);
2819 // This adds correction terms for which contraction may lead to an increase
2820 // in the error of the approximation, so disable it.
2821 Flags.setAllowContract(false);
2822 R = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Y, N2: C, Flags);
2823 SDValue NegR = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: R, Flags);
2824 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: C, N3: NegR, Flags);
2825 SDValue FMA1 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: CC, N3: FMA0, Flags);
2826 R = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: R, N2: FMA1, Flags);
2827 } else {
2828 // ch+ct is ln(2)/ln(10) to more than 36 bits
2829 const float ch_log10 = 0x1.344000p-2f;
2830 const float ct_log10 = 0x1.3509f6p-18f;
2831
2832 // ch + ct is ln(2) to more than 36 bits
2833 const float ch_log = 0x1.62e000p-1f;
2834 const float ct_log = 0x1.0bfbe8p-15f;
2835
2836 SDValue CH = DAG.getConstantFP(Val: IsLog10 ? ch_log10 : ch_log, DL, VT);
2837 SDValue CT = DAG.getConstantFP(Val: IsLog10 ? ct_log10 : ct_log, DL, VT);
2838
2839 SDValue YAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Y);
2840 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL, VT: MVT::i32);
2841 SDValue YHInt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: YAsInt, N2: MaskConst);
2842 SDValue YH = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: YHInt);
2843 SDValue YT = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: YH, Flags);
2844 // This adds correction terms for which contraction may lead to an increase
2845 // in the error of the approximation, so disable it.
2846 Flags.setAllowContract(false);
2847 SDValue YTCT = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: YT, N2: CT, Flags);
2848 SDValue Mad0 = getMad(DAG, SL: DL, VT, X: YH, Y: CT, C: YTCT, Flags);
2849 SDValue Mad1 = getMad(DAG, SL: DL, VT, X: YT, Y: CH, C: Mad0, Flags);
2850 R = getMad(DAG, SL: DL, VT, X: YH, Y: CH, C: Mad1);
2851 }
2852
2853 const bool IsFiniteOnly = Flags.hasNoNaNs() && Flags.hasNoInfs();
2854
2855 // TODO: Check if known finite from source value.
2856 if (!IsFiniteOnly) {
2857 SDValue IsFinite = getIsFinite(DAG, Src: Y, Flags);
2858 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsFinite, N2: R, N3: Y, Flags);
2859 }
2860
2861 if (IsScaled) {
2862 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
2863 SDValue ShiftK =
2864 DAG.getConstantFP(Val: IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2865 SDValue Shift =
2866 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsScaled, N2: ShiftK, N3: Zero, Flags);
2867 R = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: R, N2: Shift, Flags);
2868 }
2869
2870 return R;
2871}
2872
2873SDValue AMDGPUTargetLowering::LowerFLOG10(SDValue Op, SelectionDAG &DAG) const {
2874 return LowerFLOGCommon(Op, DAG);
2875}
2876
2877// Do f32 fast math expansion for flog2 or flog10. This is accurate enough for a
2878// promote f16 operation.
2879SDValue AMDGPUTargetLowering::LowerFLOGUnsafe(SDValue Src, const SDLoc &SL,
2880 SelectionDAG &DAG, bool IsLog10,
2881 SDNodeFlags Flags) const {
2882 EVT VT = Src.getValueType();
2883 unsigned LogOp =
2884 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2;
2885
2886 double Log2BaseInverted =
2887 IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2888
2889 if (VT == MVT::f32) {
2890 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags);
2891 if (ScaledInput) {
2892 SDValue LogSrc = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2893 SDValue ScaledResultOffset =
2894 DAG.getConstantFP(Val: -32.0 * Log2BaseInverted, DL: SL, VT);
2895
2896 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL: SL, VT);
2897
2898 SDValue ResultOffset = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsScaled,
2899 N2: ScaledResultOffset, N3: Zero, Flags);
2900
2901 SDValue Log2Inv = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2902
2903 if (Subtarget->hasFastFMAF32())
2904 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: LogSrc, N2: Log2Inv, N3: ResultOffset,
2905 Flags);
2906 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LogSrc, N2: Log2Inv, Flags);
2907 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: ResultOffset);
2908 }
2909 }
2910
2911 SDValue Log2Operand = DAG.getNode(Opcode: LogOp, DL: SL, VT, Operand: Src, Flags);
2912 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2913
2914 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Log2Operand, N2: Log2BaseInvertedOperand,
2915 Flags);
2916}
2917
2918// This expansion gives a result slightly better than 1ulp.
2919SDValue AMDGPUTargetLowering::lowerFEXPF64(SDValue Op,
2920 SelectionDAG &DAG) const {
2921 SDLoc DL(Op);
2922 SDValue X = Op.getOperand(i: 0);
2923
2924 // TODO: Check if reassoc is safe. There is an output change in exp2 and
2925 // exp10, which slightly increases ulp.
2926 SDNodeFlags Flags = Op->getFlags() & ~SDNodeFlags::AllowReassociation;
2927
2928 SDValue DN, F, T;
2929
2930 if (Op.getOpcode() == ISD::FEXP2) {
2931 // dn = rint(x)
2932 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: X, Flags);
2933 // f = x - dn
2934 F = DAG.getNode(Opcode: ISD::FSUB, DL, VT: MVT::f64, N1: X, N2: DN, Flags);
2935 // t = f*C1 + f*C2
2936 SDValue C1 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2937 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2938 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C2, Flags);
2939 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C1, N3: Mul2, Flags);
2940 } else if (Op.getOpcode() == ISD::FEXP10) {
2941 // dn = rint(x * C1)
2942 SDValue C1 = DAG.getConstantFP(Val: 0x1.a934f0979a371p+1, DL, VT: MVT::f64);
2943 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2944 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2945
2946 // f = FMA(-dn, C2, FMA(-dn, C3, x))
2947 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2948 SDValue C2 = DAG.getConstantFP(Val: -0x1.9dc1da994fd21p-59, DL, VT: MVT::f64);
2949 SDValue C3 = DAG.getConstantFP(Val: 0x1.34413509f79ffp-2, DL, VT: MVT::f64);
2950 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2951 F = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2952
2953 // t = FMA(f, C4, f*C5)
2954 SDValue C4 = DAG.getConstantFP(Val: 0x1.26bb1bbb55516p+1, DL, VT: MVT::f64);
2955 SDValue C5 = DAG.getConstantFP(Val: -0x1.f48ad494ea3e9p-53, DL, VT: MVT::f64);
2956 SDValue MulF = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C5, Flags);
2957 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C4, N3: MulF, Flags);
2958 } else { // ISD::FEXP
2959 // dn = rint(x * C1)
2960 SDValue C1 = DAG.getConstantFP(Val: 0x1.71547652b82fep+0, DL, VT: MVT::f64);
2961 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2962 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2963
2964 // t = FMA(-dn, C2, FMA(-dn, C3, x))
2965 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2966 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2967 SDValue C3 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2968 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2969 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2970 }
2971
2972 // Polynomial expansion for p
2973 SDValue P = DAG.getConstantFP(Val: 0x1.ade156a5dcb37p-26, DL, VT: MVT::f64);
2974 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2975 N3: DAG.getConstantFP(Val: 0x1.28af3fca7ab0cp-22, DL, VT: MVT::f64),
2976 Flags);
2977 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2978 N3: DAG.getConstantFP(Val: 0x1.71dee623fde64p-19, DL, VT: MVT::f64),
2979 Flags);
2980 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2981 N3: DAG.getConstantFP(Val: 0x1.a01997c89e6b0p-16, DL, VT: MVT::f64),
2982 Flags);
2983 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2984 N3: DAG.getConstantFP(Val: 0x1.a01a014761f6ep-13, DL, VT: MVT::f64),
2985 Flags);
2986 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2987 N3: DAG.getConstantFP(Val: 0x1.6c16c1852b7b0p-10, DL, VT: MVT::f64),
2988 Flags);
2989 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2990 N3: DAG.getConstantFP(Val: 0x1.1111111122322p-7, DL, VT: MVT::f64), Flags);
2991 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2992 N3: DAG.getConstantFP(Val: 0x1.55555555502a1p-5, DL, VT: MVT::f64), Flags);
2993 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2994 N3: DAG.getConstantFP(Val: 0x1.5555555555511p-3, DL, VT: MVT::f64), Flags);
2995 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2996 N3: DAG.getConstantFP(Val: 0x1.000000000000bp-1, DL, VT: MVT::f64), Flags);
2997
2998 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT: MVT::f64);
2999
3000 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
3001 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
3002
3003 // z = ldexp(p, (int)dn)
3004 SDValue DNInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: MVT::i32, Operand: DN);
3005 SDValue Z = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: P, N2: DNInt, Flags);
3006
3007 // Overflow/underflow guards
3008 SDValue CondHi = DAG.getSetCC(
3009 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: 1024.0, DL, VT: MVT::f64), Cond: ISD::SETULE);
3010
3011 if (!Flags.hasNoInfs()) {
3012 SDValue PInf = DAG.getConstantFP(Val: std::numeric_limits<double>::infinity(),
3013 DL, VT: MVT::f64);
3014 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondHi, LHS: Z, RHS: PInf, Flags);
3015 }
3016
3017 SDValue CondLo = DAG.getSetCC(
3018 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: -1075.0, DL, VT: MVT::f64), Cond: ISD::SETUGE);
3019 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
3020 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondLo, LHS: Z, RHS: Zero, Flags);
3021
3022 return Z;
3023}
3024
3025SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const {
3026 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
3027 // If we have to handle denormals, scale up the input and adjust the result.
3028
3029 EVT VT = Op.getValueType();
3030 if (VT == MVT::f64)
3031 return lowerFEXPF64(Op, DAG);
3032
3033 SDLoc SL(Op);
3034 SDValue Src = Op.getOperand(i: 0);
3035 SDNodeFlags Flags = Op->getFlags();
3036
3037 if (VT == MVT::f16) {
3038 // Nothing in half is a denormal when promoted to f32.
3039 assert(!isTypeLegal(MVT::f16));
3040 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3041 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
3042 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
3043 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3044 }
3045
3046 assert(VT == MVT::f32);
3047
3048 if (!needsDenormHandlingF32(DAG, Src, Flags))
3049 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3050
3051 // bool needs_scaling = x < -0x1.f80000p+6f;
3052 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3053
3054 // -nextafter(128.0, -1)
3055 SDValue RangeCheckConst = DAG.getConstantFP(Val: -0x1.f80000p+6f, DL: SL, VT);
3056
3057 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3058
3059 SDValue NeedsScaling =
3060 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: RangeCheckConst, Cond: ISD::SETOLT);
3061
3062 SDValue SixtyFour = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3063 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3064
3065 SDValue AddOffset =
3066 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: SixtyFour, N3: Zero);
3067
3068 SDValue AddInput = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Src, N2: AddOffset, Flags);
3069 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: AddInput, Flags);
3070
3071 SDValue TwoExpNeg64 = DAG.getConstantFP(Val: 0x1.0p-64f, DL: SL, VT);
3072 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
3073 SDValue ResultScale =
3074 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: TwoExpNeg64, N3: One);
3075
3076 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScale, Flags);
3077}
3078
3079SDValue AMDGPUTargetLowering::lowerFEXPUnsafeImpl(SDValue X, const SDLoc &SL,
3080 SelectionDAG &DAG,
3081 SDNodeFlags Flags,
3082 bool IsExp10) const {
3083 // exp(x) -> exp2(M_LOG2E_F * x);
3084 // exp10(x) -> exp2(log2(10) * x);
3085 EVT VT = X.getValueType();
3086 SDValue Const =
3087 DAG.getConstantFP(Val: IsExp10 ? 0x1.a934f0p+1f : numbers::log2e, DL: SL, VT);
3088
3089 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Const, Flags);
3090 return DAG.getNode(Opcode: VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
3091 : (unsigned)ISD::FEXP2,
3092 DL: SL, VT, Operand: Mul, Flags);
3093}
3094
3095SDValue AMDGPUTargetLowering::lowerFEXPUnsafe(SDValue X, const SDLoc &SL,
3096 SelectionDAG &DAG,
3097 SDNodeFlags Flags) const {
3098 EVT VT = X.getValueType();
3099 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags))
3100 return lowerFEXPUnsafeImpl(X, SL, DAG, Flags, /*IsExp10=*/false);
3101
3102 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3103
3104 SDValue Threshold = DAG.getConstantFP(Val: -0x1.5d58a0p+6f, DL: SL, VT);
3105 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3106
3107 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3108
3109 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3110
3111 SDValue AdjustedX =
3112 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3113
3114 const SDValue Log2E = DAG.getConstantFP(Val: numbers::log2e, DL: SL, VT);
3115 SDValue ExpInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: Log2E, Flags);
3116
3117 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: ExpInput, Flags);
3118
3119 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.969d48p-93f, DL: SL, VT);
3120 SDValue AdjustedResult =
3121 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScaleFactor, Flags);
3122
3123 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: Exp2,
3124 Flags);
3125}
3126
3127/// Emit approx-funcs appropriate lowering for exp10. inf/nan should still be
3128/// handled correctly.
3129SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL,
3130 SelectionDAG &DAG,
3131 SDNodeFlags Flags) const {
3132 const EVT VT = X.getValueType();
3133
3134 const unsigned Exp2Op = VT == MVT::f32 ? static_cast<unsigned>(AMDGPUISD::EXP)
3135 : static_cast<unsigned>(ISD::FEXP2);
3136
3137 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags)) {
3138 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
3139 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3140 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3141
3142 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K0, Flags);
3143 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3144 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K1, Flags);
3145 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3146 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1);
3147 }
3148
3149 // bool s = x < -0x1.2f7030p+5f;
3150 // x += s ? 0x1.0p+5f : 0.0f;
3151 // exp10 = exp2(x * 0x1.a92000p+1f) *
3152 // exp2(x * 0x1.4f0978p-11f) *
3153 // (s ? 0x1.9f623ep-107f : 1.0f);
3154
3155 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3156
3157 SDValue Threshold = DAG.getConstantFP(Val: -0x1.2f7030p+5f, DL: SL, VT);
3158 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3159
3160 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+5f, DL: SL, VT);
3161 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3162 SDValue AdjustedX =
3163 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3164
3165 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3166 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3167
3168 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K0, Flags);
3169 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3170 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K1, Flags);
3171 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3172
3173 SDValue MulExps = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1, Flags);
3174
3175 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.9f623ep-107f, DL: SL, VT);
3176 SDValue AdjustedResult =
3177 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: MulExps, N2: ResultScaleFactor, Flags);
3178
3179 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: MulExps,
3180 Flags);
3181}
3182
3183SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
3184 EVT VT = Op.getValueType();
3185
3186 if (VT == MVT::f64)
3187 return lowerFEXPF64(Op, DAG);
3188
3189 SDLoc SL(Op);
3190 SDValue X = Op.getOperand(i: 0);
3191 SDNodeFlags Flags = Op->getFlags();
3192 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10;
3193
3194 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
3195 // library behavior. Also, is known-not-daz source sufficient?
3196 if (allowApproxFunc(DAG, Flags)) { // TODO: Does this really require fast?
3197 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags)
3198 : lowerFEXPUnsafe(X, SL, DAG, Flags);
3199 }
3200
3201 if (VT.getScalarType() == MVT::f16) {
3202 if (VT.isVector())
3203 return SDValue();
3204
3205 // Nothing in half is a denormal when promoted to f32.
3206 //
3207 // exp(f16 x) ->
3208 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
3209 //
3210 // exp10(f16 x) ->
3211 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
3212 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: X, Flags);
3213 SDValue Lowered = lowerFEXPUnsafeImpl(X: Ext, SL, DAG, Flags, IsExp10);
3214 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Lowered,
3215 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3216 }
3217
3218 assert(VT == MVT::f32);
3219
3220 // Algorithm:
3221 //
3222 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
3223 //
3224 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
3225 // n = 64*m + j, 0 <= j < 64
3226 //
3227 // e^x = 2^((64*m + j + f)/64)
3228 // = (2^m) * (2^(j/64)) * 2^(f/64)
3229 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
3230 //
3231 // f = x*(64/ln(2)) - n
3232 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
3233 //
3234 // e^x = (2^m) * (2^(j/64)) * e^r
3235 //
3236 // (2^(j/64)) is precomputed
3237 //
3238 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3239 // e^r = 1 + q
3240 //
3241 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3242 //
3243 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
3244 SDNodeFlags FlagsNoContract = Flags;
3245 FlagsNoContract.setAllowContract(false);
3246
3247 SDValue PH, PL;
3248 if (Subtarget->hasFastFMAF32()) {
3249 const float c_exp = numbers::log2ef;
3250 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
3251 const float c_exp10 = 0x1.a934f0p+1f;
3252 const float cc_exp10 = 0x1.2f346ep-24f;
3253
3254 SDValue C = DAG.getConstantFP(Val: IsExp10 ? c_exp10 : c_exp, DL: SL, VT);
3255 SDValue CC = DAG.getConstantFP(Val: IsExp10 ? cc_exp10 : cc_exp, DL: SL, VT);
3256
3257 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: C, Flags);
3258 SDValue NegPH = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: PH, Flags);
3259 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: C, N3: NegPH, Flags);
3260 PL = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: CC, N3: FMA0, Flags);
3261 } else {
3262 const float ch_exp = 0x1.714000p+0f;
3263 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
3264
3265 const float ch_exp10 = 0x1.a92000p+1f;
3266 const float cl_exp10 = 0x1.4f0978p-11f;
3267
3268 SDValue CH = DAG.getConstantFP(Val: IsExp10 ? ch_exp10 : ch_exp, DL: SL, VT);
3269 SDValue CL = DAG.getConstantFP(Val: IsExp10 ? cl_exp10 : cl_exp, DL: SL, VT);
3270
3271 SDValue XAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: X);
3272 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL: SL, VT: MVT::i32);
3273 SDValue XHAsInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: XAsInt, N2: MaskConst);
3274 SDValue XH = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: XHAsInt);
3275 SDValue XL = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: XH, Flags);
3276
3277 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XH, N2: CH, Flags);
3278
3279 SDValue XLCL = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XL, N2: CL, Flags);
3280 SDValue Mad0 = getMad(DAG, SL, VT, X: XL, Y: CH, C: XLCL, Flags);
3281 PL = getMad(DAG, SL, VT, X: XH, Y: CL, C: Mad0, Flags);
3282 }
3283
3284 SDValue E = DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SL, VT, Operand: PH, Flags);
3285
3286 // It is unsafe to contract this fsub into the PH multiply.
3287 SDValue PHSubE = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: PH, N2: E, Flags: FlagsNoContract);
3288
3289 SDValue A = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: PHSubE, N2: PL, Flags);
3290 SDValue IntE = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: SL, VT: MVT::i32, Operand: E);
3291 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: A, Flags);
3292
3293 SDValue R = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: Exp2, N2: IntE, Flags);
3294
3295 SDValue UnderflowCheckConst =
3296 DAG.getConstantFP(Val: IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, DL: SL, VT);
3297
3298 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3299 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3300 SDValue Underflow =
3301 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: UnderflowCheckConst, Cond: ISD::SETOLT);
3302
3303 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Underflow, N2: Zero, N3: R);
3304
3305 if (!Flags.hasNoInfs()) {
3306 SDValue OverflowCheckConst =
3307 DAG.getConstantFP(Val: IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, DL: SL, VT);
3308 SDValue Overflow =
3309 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: OverflowCheckConst, Cond: ISD::SETOGT);
3310 SDValue Inf =
3311 DAG.getConstantFP(Val: APFloat::getInf(Sem: APFloat::IEEEsingle()), DL: SL, VT);
3312 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Overflow, N2: Inf, N3: R);
3313 }
3314
3315 return R;
3316}
3317
3318static bool isCtlzOpc(unsigned Opc) {
3319 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
3320}
3321
3322static bool isCttzOpc(unsigned Opc) {
3323 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_POISON;
3324}
3325
3326SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op,
3327 SelectionDAG &DAG) const {
3328 auto SL = SDLoc(Op);
3329 auto Opc = Op.getOpcode();
3330 auto Arg = Op.getOperand(i: 0u);
3331 auto ResultVT = Op.getValueType();
3332
3333 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3334 return {};
3335
3336 assert(isCtlzOpc(Opc));
3337 assert(ResultVT == Arg.getValueType());
3338
3339 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3340 SDValue NumExtBits = DAG.getConstant(Val: 32u - NumBits, DL: SL, VT: MVT::i32);
3341 SDValue NewOp;
3342
3343 if (Opc == ISD::CTLZ_ZERO_POISON) {
3344 NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3345 NewOp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3346 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3347 } else {
3348 NewOp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3349 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3350 NewOp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3351 }
3352
3353 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ResultVT, Operand: NewOp);
3354}
3355
3356SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
3357 SDLoc SL(Op);
3358 SDValue Src = Op.getOperand(i: 0);
3359
3360 assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode()));
3361 bool Ctlz = isCtlzOpc(Opc: Op.getOpcode());
3362 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;
3363
3364 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_POISON ||
3365 Op.getOpcode() == ISD::CTTZ_ZERO_POISON;
3366 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3367
3368 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3369 // (ctlz hi:lo) -> (umin (ffbh src), 32)
3370 // (cttz hi:lo) -> (umin (ffbl src), 32)
3371 // (ctlz_zero_poison src) -> (ffbh src)
3372 // (cttz_zero_poison src) -> (ffbl src)
3373
3374 // 64-bit scalar version produce 32-bit result
3375 // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64)
3376 // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64)
3377 // (ctlz_zero_poison src) -> (S_FLBIT_I32_B64 src)
3378 // (cttz_zero_poison src) -> (S_FF1_I32_B64 src)
3379 SDValue NewOpr = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Src);
3380 if (!ZeroUndef) {
3381 const SDValue ConstVal = DAG.getConstant(
3382 Val: Op.getValueType().getScalarSizeInBits(), DL: SL, VT: MVT::i32);
3383 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: ConstVal);
3384 }
3385 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: Src.getValueType(), Operand: NewOpr);
3386 }
3387
3388 SDValue Lo, Hi;
3389 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3390
3391 SDValue OprLo = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Lo);
3392 SDValue OprHi = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Hi);
3393
3394 // (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64)
3395 // (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64)
3396 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
3397 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
3398
3399 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT;
3400 const SDValue Const32 = DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32);
3401 if (Ctlz)
3402 OprLo = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprLo, N2: Const32);
3403 else
3404 OprHi = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprHi, N2: Const32);
3405
3406 SDValue NewOpr;
3407 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: OprLo, N2: OprHi);
3408 if (!ZeroUndef) {
3409 const SDValue Const64 = DAG.getConstant(Val: 64, DL: SL, VT: MVT::i32);
3410 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: Const64);
3411 }
3412
3413 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: NewOpr);
3414}
3415
3416SDValue AMDGPUTargetLowering::LowerCTLS(SDValue Op, SelectionDAG &DAG) const {
3417 SDLoc SL(Op);
3418 SDValue Src = Op.getOperand(i: 0);
3419 assert(Src.getValueType() == MVT::i32 && "LowerCTLS only supports i32");
3420 SDValue Ffbh = DAG.getNode(
3421 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3422 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Src);
3423 SDValue Clamped = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: Ffbh,
3424 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3425 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: Clamped,
3426 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i32));
3427}
3428
3429SDValue AMDGPUTargetLowering::LowerINT_TO_FP16(SDValue Op, SelectionDAG &DAG,
3430 EVT FP16Ty) const {
3431 assert(FP16Ty == MVT::f16 || FP16Ty == MVT::bf16);
3432 SDLoc SL(Op);
3433 SDValue Src = Op.getOperand(i: 0);
3434 SDValue ToF32 = DAG.getNode(Opcode: Op.getOpcode(), DL: SL, VT: MVT::f32, Operand: Src);
3435 SDValue FPRoundFlag = DAG.getIntPtrConstant(Val: 0, DL: SL, /*isTarget=*/true);
3436 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: FP16Ty, N1: ToF32, N2: FPRoundFlag);
3437}
3438
3439SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
3440 bool Signed) const {
3441 // The regular method converting a 64-bit integer to float roughly consists of
3442 // 2 steps: normalization and rounding. In fact, after normalization, the
3443 // conversion from a 64-bit integer to a float is essentially the same as the
3444 // one from a 32-bit integer. The only difference is that it has more
3445 // trailing bits to be rounded. To leverage the native 32-bit conversion, a
3446 // 64-bit integer could be preprocessed and fit into a 32-bit integer then
3447 // converted into the correct float number. The basic steps for the unsigned
3448 // conversion are illustrated in the following pseudo code:
3449 //
3450 // f32 uitofp(i64 u) {
3451 // i32 hi, lo = split(u);
3452 // // Only count the leading zeros in hi as we have native support of the
3453 // // conversion from i32 to f32. If hi is all 0s, the conversion is
3454 // // reduced to a 32-bit one automatically.
3455 // i32 shamt = clz(hi); // Return 32 if hi is all 0s.
3456 // u <<= shamt;
3457 // hi, lo = split(u);
3458 // hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo.
3459 // // convert it as a 32-bit integer and scale the result back.
3460 // return uitofp(hi) * 2^(32 - shamt);
3461 // }
3462 //
3463 // The signed one follows the same principle but uses 'ffbh_i32' to count its
3464 // sign bits instead. If 'ffbh_i32' is not available, its absolute value is
3465 // converted instead followed by negation based its sign bit.
3466
3467 SDLoc SL(Op);
3468 SDValue Src = Op.getOperand(i: 0);
3469
3470 SDValue Lo, Hi;
3471 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3472 SDValue Sign;
3473 SDValue ShAmt;
3474 if (Signed && Subtarget->isGCN()) {
3475 // We also need to consider the sign bit in Lo if Hi has just sign bits,
3476 // i.e. Hi is 0 or -1. However, that only needs to take the MSB into
3477 // account. That is, the maximal shift is
3478 // - 32 if Lo and Hi have opposite signs;
3479 // - 33 if Lo and Hi have the same sign.
3480 //
3481 // Or, MaxShAmt = 33 + OppositeSign, where
3482 //
3483 // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is
3484 // - -1 if Lo and Hi have opposite signs; and
3485 // - 0 otherwise.
3486 //
3487 // All in all, ShAmt is calculated as
3488 //
3489 // umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1.
3490 //
3491 // or
3492 //
3493 // umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31).
3494 //
3495 // to reduce the critical path.
3496 SDValue OppositeSign = DAG.getNode(
3497 Opcode: ISD::SRA, DL: SL, VT: MVT::i32, N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: Lo, N2: Hi),
3498 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3499 SDValue MaxShAmt =
3500 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3501 N2: OppositeSign);
3502 // Count the leading sign bits.
3503 ShAmt = DAG.getNode(
3504 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3505 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Hi);
3506 // Different from unsigned conversion, the shift should be one bit less to
3507 // preserve the sign bit.
3508 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ShAmt,
3509 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
3510 ShAmt = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: ShAmt, N2: MaxShAmt);
3511 } else {
3512 if (Signed) {
3513 // Without 'ffbh_i32', only leading zeros could be counted. Take the
3514 // absolute value first.
3515 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: Src,
3516 N2: DAG.getConstant(Val: 63, DL: SL, VT: MVT::i64));
3517 SDValue Abs =
3518 DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64,
3519 N1: DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: Src, N2: Sign), N2: Sign);
3520 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Abs, DAG);
3521 }
3522 // Count the leading zeros.
3523 ShAmt = DAG.getNode(Opcode: ISD::CTLZ, DL: SL, VT: MVT::i32, Operand: Hi);
3524 // The shift amount for signed integers is [0, 32].
3525 }
3526 // Normalize the given 64-bit integer.
3527 SDValue Norm = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i64, N1: Src, N2: ShAmt);
3528 // Split it again.
3529 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Norm, DAG);
3530 // Calculate the adjust bit for rounding.
3531 // (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo)
3532 SDValue Adjust = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32,
3533 N1: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32), N2: Lo);
3534 // Get the 32-bit normalized integer.
3535 Norm = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Hi, N2: Adjust);
3536 // Convert the normalized 32-bit integer into f32.
3537
3538 bool UseLDEXP = isOperationLegal(Op: ISD::FLDEXP, VT: MVT::f32);
3539 unsigned Opc = Signed && UseLDEXP ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3540 SDValue FVal = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::f32, Operand: Norm);
3541
3542 // Finally, need to scale back the converted floating number as the original
3543 // 64-bit integer is converted as a 32-bit one.
3544 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3545 N2: ShAmt);
3546 // On GCN, use LDEXP directly.
3547 if (UseLDEXP)
3548 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f32, N1: FVal, N2: ShAmt);
3549
3550 // Otherwise, align 'ShAmt' to the exponent part and add it into the exponent
3551 // part directly to emulate the multiplication of 2^ShAmt. That 8-bit
3552 // exponent is enough to avoid overflowing into the sign bit.
3553 SDValue Exp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ShAmt,
3554 N2: DAG.getConstant(Val: 23, DL: SL, VT: MVT::i32));
3555 SDValue IVal =
3556 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32,
3557 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: FVal), N2: Exp);
3558 if (Signed) {
3559 // Set the sign bit.
3560 Sign = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32,
3561 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Sign),
3562 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3563 IVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: IVal, N2: Sign);
3564 }
3565 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: IVal);
3566}
3567
3568SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
3569 bool Signed) const {
3570 SDLoc SL(Op);
3571 SDValue Src = Op.getOperand(i: 0);
3572
3573 SDValue Lo, Hi;
3574 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3575
3576 SDValue CvtHi = DAG.getNode(Opcode: Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
3577 DL: SL, VT: MVT::f64, Operand: Hi);
3578
3579 SDValue CvtLo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: SL, VT: MVT::f64, Operand: Lo);
3580
3581 SDValue LdExp = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f64, N1: CvtHi,
3582 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3583 // TODO: Should this propagate fast-math-flags?
3584 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: LdExp, N2: CvtLo);
3585}
3586
3587SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
3588 SelectionDAG &DAG) const {
3589 // TODO: Factor out code common with LowerSINT_TO_FP.
3590 EVT DestVT = Op.getValueType();
3591 SDValue Src = Op.getOperand(i: 0);
3592 EVT SrcVT = Src.getValueType();
3593
3594 if (SrcVT == MVT::i16) {
3595 if (DestVT == MVT::f16)
3596 return Op;
3597 SDLoc DL(Op);
3598
3599 // Promote src to i32
3600 SDValue Ext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: Src);
3601 return DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: DestVT, Operand: Ext);
3602 }
3603
3604 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3605 return LowerINT_TO_FP16(Op, DAG, FP16Ty: DestVT);
3606
3607 if (SrcVT != MVT::i64)
3608 return Op;
3609
3610 if (DestVT == MVT::f32)
3611 return LowerINT_TO_FP32(Op, DAG, Signed: false);
3612
3613 assert(DestVT == MVT::f64);
3614 return LowerINT_TO_FP64(Op, DAG, Signed: false);
3615}
3616
3617SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
3618 SelectionDAG &DAG) const {
3619 EVT DestVT = Op.getValueType();
3620
3621 SDValue Src = Op.getOperand(i: 0);
3622 EVT SrcVT = Src.getValueType();
3623
3624 if (SrcVT == MVT::i16) {
3625 if (DestVT == MVT::f16)
3626 return Op;
3627
3628 SDLoc DL(Op);
3629 // Promote src to i32
3630 SDValue Ext = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i32, Operand: Src);
3631 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT: DestVT, Operand: Ext);
3632 }
3633
3634 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3635 return LowerINT_TO_FP16(Op, DAG, FP16Ty: DestVT);
3636
3637 if (SrcVT != MVT::i64)
3638 return Op;
3639
3640 // TODO: Factor out code common with LowerUINT_TO_FP.
3641
3642 if (DestVT == MVT::f32)
3643 return LowerINT_TO_FP32(Op, DAG, Signed: true);
3644
3645 assert(DestVT == MVT::f64);
3646 return LowerINT_TO_FP64(Op, DAG, Signed: true);
3647}
3648
3649SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
3650 bool Signed) const {
3651 SDLoc SL(Op);
3652
3653 SDValue Src = Op.getOperand(i: 0);
3654 EVT SrcVT = Src.getValueType();
3655
3656 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3657
3658 // The basic idea of converting a floating point number into a pair of 32-bit
3659 // integers is illustrated as follows:
3660 //
3661 // tf := trunc(val);
3662 // hif := floor(tf * 2^-32);
3663 // lof := tf - hif * 2^32; // lof is always positive due to floor.
3664 // hi := fptoi(hif);
3665 // lo := fptoi(lof);
3666 //
3667 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: SrcVT, Operand: Src);
3668 SDValue Sign;
3669 if (Signed && SrcVT == MVT::f32) {
3670 // However, a 32-bit floating point number has only 23 bits mantissa and
3671 // it's not enough to hold all the significant bits of `lof` if val is
3672 // negative. To avoid the loss of precision, We need to take the absolute
3673 // value after truncating and flip the result back based on the original
3674 // signedness.
3675 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i32,
3676 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Trunc),
3677 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3678 Trunc = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: SrcVT, Operand: Trunc);
3679 }
3680
3681 SDValue K0, K1;
3682 if (SrcVT == MVT::f64) {
3683 K0 = DAG.getConstantFP(
3684 Val: llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), DL: SL,
3685 VT: SrcVT);
3686 K1 = DAG.getConstantFP(
3687 Val: llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), DL: SL,
3688 VT: SrcVT);
3689 } else {
3690 K0 = DAG.getConstantFP(
3691 Val: llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), DL: SL, VT: SrcVT);
3692 K1 = DAG.getConstantFP(
3693 Val: llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), DL: SL, VT: SrcVT);
3694 }
3695 // TODO: Should this propagate fast-math-flags?
3696 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: SrcVT, N1: Trunc, N2: K0);
3697
3698 SDValue FloorMul = DAG.getNode(Opcode: ISD::FFLOOR, DL: SL, VT: SrcVT, Operand: Mul);
3699
3700 SDValue Fma = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: SrcVT, N1: FloorMul, N2: K1, N3: Trunc);
3701
3702 SDValue Hi = DAG.getNode(Opcode: (Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
3703 : ISD::FP_TO_UINT,
3704 DL: SL, VT: MVT::i32, Operand: FloorMul);
3705 SDValue Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SL, VT: MVT::i32, Operand: Fma);
3706
3707 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3708 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Lo, Hi}));
3709
3710 if (Signed && SrcVT == MVT::f32) {
3711 assert(Sign);
3712 // Flip the result based on the signedness, which is either all 0s or 1s.
3713 Sign = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3714 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Sign, Sign}));
3715 // r := xor(r, sign) - sign;
3716 Result =
3717 DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i64,
3718 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64, N1: Result, N2: Sign), N2: Sign);
3719 }
3720
3721 return Result;
3722}
3723
3724SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
3725 SDLoc DL(Op);
3726 SDValue N0 = Op.getOperand(i: 0);
3727
3728 // Convert to target node to get known bits
3729 if (N0.getValueType() == MVT::f32)
3730 return DAG.getNode(Opcode: AMDGPUISD::FP_TO_FP16, DL, VT: Op.getValueType(), Operand: N0);
3731
3732 if (Op->getFlags().hasApproximateFuncs()) {
3733 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
3734 return SDValue();
3735 }
3736
3737 return LowerF64ToF16Safe(Src: N0, DL, DAG);
3738}
3739
3740// return node in i32
3741SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
3742 SelectionDAG &DAG) const {
3743 assert(Src.getSimpleValueType() == MVT::f64);
3744
3745 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
3746 // TODO: We can generate better code for True16.
3747 const unsigned ExpMask = 0x7ff;
3748 const unsigned ExpBiasf64 = 1023;
3749 const unsigned ExpBiasf16 = 15;
3750 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3751 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
3752 SDValue U = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Src);
3753 SDValue UH = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: U,
3754 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3755 UH = DAG.getZExtOrTrunc(Op: UH, DL, VT: MVT::i32);
3756 U = DAG.getZExtOrTrunc(Op: U, DL, VT: MVT::i32);
3757 SDValue E = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3758 N2: DAG.getConstant(Val: 20, DL, VT: MVT::i64));
3759 E = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: E,
3760 N2: DAG.getConstant(Val: ExpMask, DL, VT: MVT::i32));
3761 // Subtract the fp64 exponent bias (1023) to get the real exponent and
3762 // add the f16 bias (15) to get the biased exponent for the f16 format.
3763 E = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: E,
3764 N2: DAG.getConstant(Val: -ExpBiasf64 + ExpBiasf16, DL, VT: MVT::i32));
3765
3766 SDValue M = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3767 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32));
3768 M = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M,
3769 N2: DAG.getConstant(Val: 0xffe, DL, VT: MVT::i32));
3770
3771 SDValue MaskedSig = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: UH,
3772 N2: DAG.getConstant(Val: 0x1ff, DL, VT: MVT::i32));
3773 MaskedSig = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: MaskedSig, N2: U);
3774
3775 SDValue Lo40Set = DAG.getSelectCC(DL, LHS: MaskedSig, RHS: Zero, True: Zero, False: One, Cond: ISD::SETEQ);
3776 M = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M, N2: Lo40Set);
3777
3778 // (M != 0 ? 0x0200 : 0) | 0x7c00;
3779 SDValue I = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
3780 N1: DAG.getSelectCC(DL, LHS: M, RHS: Zero, True: DAG.getConstant(Val: 0x0200, DL, VT: MVT::i32),
3781 False: Zero, Cond: ISD::SETNE), N2: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32));
3782
3783 // N = M | (E << 12);
3784 SDValue N = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3785 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: E,
3786 N2: DAG.getConstant(Val: 12, DL, VT: MVT::i32)));
3787
3788 // B = clamp(1-E, 0, 13);
3789 SDValue OneSubExp = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
3790 N1: One, N2: E);
3791 SDValue B = DAG.getNode(Opcode: ISD::SMAX, DL, VT: MVT::i32, N1: OneSubExp, N2: Zero);
3792 B = DAG.getNode(Opcode: ISD::SMIN, DL, VT: MVT::i32, N1: B,
3793 N2: DAG.getConstant(Val: 13, DL, VT: MVT::i32));
3794
3795 SDValue SigSetHigh = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3796 N2: DAG.getConstant(Val: 0x1000, DL, VT: MVT::i32));
3797
3798 SDValue D = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SigSetHigh, N2: B);
3799 SDValue D0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: D, N2: B);
3800 SDValue D1 = DAG.getSelectCC(DL, LHS: D0, RHS: SigSetHigh, True: One, False: Zero, Cond: ISD::SETNE);
3801 D = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: D, N2: D1);
3802
3803 SDValue V = DAG.getSelectCC(DL, LHS: E, RHS: One, True: D, False: N, Cond: ISD::SETLT);
3804 SDValue VLow3 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: V,
3805 N2: DAG.getConstant(Val: 0x7, DL, VT: MVT::i32));
3806 V = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: V,
3807 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3808 SDValue V0 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 3, DL, VT: MVT::i32),
3809 True: One, False: Zero, Cond: ISD::SETEQ);
3810 SDValue V1 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 5, DL, VT: MVT::i32),
3811 True: One, False: Zero, Cond: ISD::SETGT);
3812 V1 = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: V0, N2: V1);
3813 V = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: V, N2: V1);
3814
3815 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 30, DL, VT: MVT::i32),
3816 True: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32), False: V, Cond: ISD::SETGT);
3817 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 1039, DL, VT: MVT::i32),
3818 True: I, False: V, Cond: ISD::SETEQ);
3819
3820 // Extract the sign bit.
3821 SDValue Sign = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3822 N2: DAG.getConstant(Val: 16, DL, VT: MVT::i32));
3823 Sign = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Sign,
3824 N2: DAG.getConstant(Val: 0x8000, DL, VT: MVT::i32));
3825
3826 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Sign, N2: V);
3827}
3828
3829SDValue AMDGPUTargetLowering::LowerFP_TO_INT(const SDValue Op,
3830 SelectionDAG &DAG) const {
3831 SDValue Src = Op.getOperand(i: 0);
3832 unsigned OpOpcode = Op.getOpcode();
3833 EVT SrcVT = Src.getValueType();
3834 EVT DestVT = Op.getValueType();
3835
3836 // Will be selected natively
3837 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3838 return Op;
3839
3840 if (SrcVT == MVT::bf16 || (SrcVT == MVT::f16 && DestVT == MVT::i32)) {
3841 SDLoc DL(Op);
3842 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3843 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DestVT, Operand: PromotedSrc);
3844 }
3845
3846 // Promote i16 to i32
3847 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3848 SDLoc DL(Op);
3849
3850 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3851 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToInt32);
3852 }
3853
3854 if (DestVT != MVT::i64)
3855 return Op;
3856
3857 if (SrcVT == MVT::f16 ||
3858 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
3859 SDLoc DL(Op);
3860
3861 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3862 unsigned Ext =
3863 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3864 return DAG.getNode(Opcode: Ext, DL, VT: MVT::i64, Operand: FpToInt32);
3865 }
3866
3867 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3868 return LowerFP_TO_INT64(Op, DAG, Signed: OpOpcode == ISD::FP_TO_SINT);
3869
3870 return SDValue();
3871}
3872
3873SDValue AMDGPUTargetLowering::LowerFP_TO_INT_SAT(const SDValue Op,
3874 SelectionDAG &DAG) const {
3875 SDValue Src = Op.getOperand(i: 0);
3876 unsigned OpOpcode = Op.getOpcode();
3877 EVT SrcVT = Src.getValueType();
3878 EVT DstVT = Op.getValueType();
3879 SDValue SatVTOp = Op.getNode()->getOperand(Num: 1);
3880 EVT SatVT = cast<VTSDNode>(Val&: SatVTOp)->getVT();
3881 SDLoc DL(Op);
3882
3883 uint64_t DstWidth = DstVT.getScalarSizeInBits();
3884 uint64_t SatWidth = SatVT.getScalarSizeInBits();
3885 assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
3886
3887 // Scalar cases will be selected natively to v_cvt_/s_cvt_ instructions.
3888 // v2f32 -> v2i16 will be selected natively to v_cvt_pk_[iu]16_f32.
3889 if (SatWidth == DstWidth) {
3890 if ((DstVT == MVT::i32 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
3891 (DstVT == MVT::i16 && (SrcVT == MVT::f16 || SrcVT == MVT::f32)) ||
3892 (DstVT == MVT::v2i16 && SrcVT == MVT::v2f32))
3893 return Op;
3894 }
3895
3896 // Vectors can only be selected natively.
3897 if (DstVT.isVector())
3898 return SDValue();
3899
3900 // Perform all saturation at selected width (i16 or i32) and truncate
3901 if (SatWidth < DstWidth && SatWidth <= 32) {
3902 // For f16 conversion with sub-i16 saturation perform saturation
3903 // at i16, if available in the target. This removes the need for extra f16
3904 // to f32 conversion. For all the others use i32.
3905 MVT ResultVT =
3906 Subtarget->has16BitInsts() && SrcVT == MVT::f16 && SatWidth < 16
3907 ? MVT::i16
3908 : MVT::i32;
3909
3910 const SDValue ResultVTOp = DAG.getValueType(ResultVT);
3911 const uint64_t ResultWidth = ResultVT.getScalarSizeInBits();
3912
3913 // First, convert input float into selected integer (i16 or i32)
3914 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: ResultVT, N1: Src, N2: ResultVTOp);
3915 SDValue IntSatVal;
3916
3917 // Then, clamp at the saturation width using either i16 or i32 instructions
3918 if (OpOpcode == ISD::FP_TO_SINT_SAT) {
3919 SDValue MinConst = DAG.getConstant(
3920 Val: APInt::getSignedMaxValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3921 SDValue MaxConst = DAG.getConstant(
3922 Val: APInt::getSignedMinValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3923 SDValue MinVal = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3924 IntSatVal = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ResultVT, N1: MinVal, N2: MaxConst);
3925 } else {
3926 SDValue MinConst = DAG.getConstant(
3927 Val: APInt::getMaxValue(numBits: SatWidth).zext(width: ResultWidth), DL, VT: ResultVT);
3928 IntSatVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3929 }
3930
3931 // Finally, after saturating at i16 or i32 fit into the destination type
3932 return DAG.getExtOrTrunc(IsSigned: OpOpcode == ISD::FP_TO_SINT_SAT, Op: IntSatVal, DL,
3933 VT: DstVT);
3934 }
3935
3936 // SatWidth == DstWidth or SatWidth > 32
3937
3938 // Saturate at i32 for i64 dst and f16/bf16 src (will invoke f16 promotion
3939 // below)
3940 if (DstVT == MVT::i64 &&
3941 (SrcVT == MVT::f16 || SrcVT == MVT::bf16 ||
3942 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP))) {
3943 const SDValue Int32VTOp = DAG.getValueType(MVT::i32);
3944 return DAG.getNode(Opcode: OpOpcode, DL, VT: DstVT, N1: Src, N2: Int32VTOp);
3945 }
3946
3947 // Promote f16/bf16 src to f32 for i32 conversion
3948 if (DstVT == MVT::i32 && (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
3949 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3950 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DstVT, N1: PromotedSrc, N2: SatVTOp);
3951 }
3952
3953 // For DstWidth < 16, promote i1 and i8 dst to i16 (if legal) with sub-i16
3954 // saturation. For DstWidth == 16, promote i16 dst to i32 with sub-i32
3955 // saturation; this covers i16.f32 and i16.f64
3956 if (DstWidth < 32) {
3957 // Note: this triggers SatWidth < DstWidth above to generate saturated
3958 // truncate by requesting MVT::i16/i32 destination with SatWidth < 16/32.
3959 MVT PromoteVT =
3960 (DstWidth < 16 && Subtarget->has16BitInsts()) ? MVT::i16 : MVT::i32;
3961 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: PromoteVT, N1: Src, N2: SatVTOp);
3962 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: FpToInt);
3963 }
3964
3965 // TODO: can we implement i64 dst for f32/f64?
3966
3967 return SDValue();
3968}
3969
3970SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
3971 SelectionDAG &DAG) const {
3972 EVT ExtraVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3973 MVT VT = Op.getSimpleValueType();
3974 MVT ScalarVT = VT.getScalarType();
3975
3976 assert(VT.isVector());
3977
3978 SDValue Src = Op.getOperand(i: 0);
3979 SDLoc DL(Op);
3980
3981 // TODO: Don't scalarize on Evergreen?
3982 unsigned NElts = VT.getVectorNumElements();
3983 SmallVector<SDValue, 8> Args;
3984 DAG.ExtractVectorElements(Op: Src, Args, Start: 0, Count: NElts);
3985
3986 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
3987 for (unsigned I = 0; I < NElts; ++I)
3988 Args[I] = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: ScalarVT, N1: Args[I], N2: VTOp);
3989
3990 return DAG.getBuildVector(VT, DL, Ops: Args);
3991}
3992
3993//===----------------------------------------------------------------------===//
3994// Custom DAG optimizations
3995//===----------------------------------------------------------------------===//
3996
3997static bool isU24(SDValue Op, SelectionDAG &DAG) {
3998 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
3999}
4000
4001static bool isI24(SDValue Op, SelectionDAG &DAG) {
4002 EVT VT = Op.getValueType();
4003 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
4004 // as unsigned 24-bit values.
4005 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24;
4006}
4007
4008static SDValue simplifyMul24(SDNode *Node24,
4009 TargetLowering::DAGCombinerInfo &DCI) {
4010 SelectionDAG &DAG = DCI.DAG;
4011 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4012 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
4013
4014 SDValue LHS = IsIntrin ? Node24->getOperand(Num: 1) : Node24->getOperand(Num: 0);
4015 SDValue RHS = IsIntrin ? Node24->getOperand(Num: 2) : Node24->getOperand(Num: 1);
4016 unsigned NewOpcode = Node24->getOpcode();
4017 if (IsIntrin) {
4018 unsigned IID = Node24->getConstantOperandVal(Num: 0);
4019 switch (IID) {
4020 case Intrinsic::amdgcn_mul_i24:
4021 NewOpcode = AMDGPUISD::MUL_I24;
4022 break;
4023 case Intrinsic::amdgcn_mul_u24:
4024 NewOpcode = AMDGPUISD::MUL_U24;
4025 break;
4026 case Intrinsic::amdgcn_mulhi_i24:
4027 NewOpcode = AMDGPUISD::MULHI_I24;
4028 break;
4029 case Intrinsic::amdgcn_mulhi_u24:
4030 NewOpcode = AMDGPUISD::MULHI_U24;
4031 break;
4032 default:
4033 llvm_unreachable("Expected 24-bit mul intrinsic");
4034 }
4035 }
4036
4037 APInt Demanded = APInt::getLowBitsSet(numBits: LHS.getValueSizeInBits(), loBitsSet: 24);
4038
4039 // First try to simplify using SimplifyMultipleUseDemandedBits which allows
4040 // the operands to have other uses, but will only perform simplifications that
4041 // involve bypassing some nodes for this user.
4042 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(Op: LHS, DemandedBits: Demanded, DAG);
4043 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(Op: RHS, DemandedBits: Demanded, DAG);
4044 if (DemandedLHS || DemandedRHS)
4045 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(Node24), VTList: Node24->getVTList(),
4046 N1: DemandedLHS ? DemandedLHS : LHS,
4047 N2: DemandedRHS ? DemandedRHS : RHS);
4048
4049 // Now try SimplifyDemandedBits which can simplify the nodes used by our
4050 // operands if this node is the only user.
4051 if (TLI.SimplifyDemandedBits(Op: LHS, DemandedBits: Demanded, DCI))
4052 return SDValue(Node24, 0);
4053 if (TLI.SimplifyDemandedBits(Op: RHS, DemandedBits: Demanded, DCI))
4054 return SDValue(Node24, 0);
4055
4056 return SDValue();
4057}
4058
4059template <typename IntTy>
4060static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
4061 uint32_t Width, const SDLoc &DL) {
4062 if (Width + Offset < 32) {
4063 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
4064 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
4065 if constexpr (std::is_signed_v<IntTy>) {
4066 return DAG.getSignedConstant(Val: Result, DL, VT: MVT::i32);
4067 } else {
4068 return DAG.getConstant(Result, DL, MVT::i32);
4069 }
4070 }
4071
4072 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
4073}
4074
4075static bool hasVolatileUser(SDNode *Val) {
4076 for (SDNode *U : Val->users()) {
4077 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: U)) {
4078 if (M->isVolatile())
4079 return true;
4080 }
4081 }
4082
4083 return false;
4084}
4085
4086bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
4087 // i32 vectors are the canonical memory type.
4088 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
4089 return false;
4090
4091 if (!VT.isByteSized())
4092 return false;
4093
4094 unsigned Size = VT.getStoreSize();
4095
4096 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
4097 return false;
4098
4099 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
4100 return false;
4101
4102 return true;
4103}
4104
4105// Replace load of an illegal type with a bitcast from a load of a friendlier
4106// type.
4107SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
4108 DAGCombinerInfo &DCI) const {
4109 if (!DCI.isBeforeLegalize())
4110 return SDValue();
4111
4112 LoadSDNode *LN = cast<LoadSDNode>(Val: N);
4113 if (!LN->isSimple() || !ISD::isNormalLoad(N: LN) || hasVolatileUser(Val: LN))
4114 return SDValue();
4115
4116 SDLoc SL(N);
4117 SelectionDAG &DAG = DCI.DAG;
4118 EVT VT = LN->getMemoryVT();
4119
4120 unsigned Size = VT.getStoreSize();
4121 Align Alignment = LN->getAlign();
4122 if (Alignment < Size && isTypeLegal(VT)) {
4123 unsigned IsFast;
4124 unsigned AS = LN->getAddressSpace();
4125
4126 // Expand unaligned loads earlier than legalization. Due to visitation order
4127 // problems during legalization, the emitted instructions to pack and unpack
4128 // the bytes again are not eliminated in the case of an unaligned copy.
4129 if (!allowsMisalignedMemoryAccesses(
4130 VT, AddrSpace: AS, Alignment, Flags: LN->getMemOperand()->getFlags(), &IsFast)) {
4131 if (VT.isVector())
4132 return SplitVectorLoad(Op: SDValue(LN, 0), DAG);
4133
4134 SDValue Ops[2];
4135 std::tie(args&: Ops[0], args&: Ops[1]) = expandUnalignedLoad(LD: LN, DAG);
4136
4137 return DAG.getMergeValues(Ops, dl: SDLoc(N));
4138 }
4139
4140 if (!IsFast)
4141 return SDValue();
4142 }
4143
4144 if (!shouldCombineMemoryType(VT))
4145 return SDValue();
4146
4147 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4148
4149 SDValue NewLoad
4150 = DAG.getLoad(VT: NewVT, dl: SL, Chain: LN->getChain(),
4151 Ptr: LN->getBasePtr(), MMO: LN->getMemOperand());
4152
4153 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewLoad);
4154 DCI.CombineTo(N, Res0: BC, Res1: NewLoad.getValue(R: 1));
4155 return SDValue(N, 0);
4156}
4157
4158// Replace store of an illegal type with a store of a bitcast to a friendlier
4159// type.
4160SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
4161 DAGCombinerInfo &DCI) const {
4162 if (!DCI.isBeforeLegalize())
4163 return SDValue();
4164
4165 StoreSDNode *SN = cast<StoreSDNode>(Val: N);
4166 if (!SN->isSimple() || !ISD::isNormalStore(N: SN))
4167 return SDValue();
4168
4169 EVT VT = SN->getMemoryVT();
4170 unsigned Size = VT.getStoreSize();
4171
4172 SDLoc SL(N);
4173 SelectionDAG &DAG = DCI.DAG;
4174 Align Alignment = SN->getAlign();
4175 if (Alignment < Size && isTypeLegal(VT)) {
4176 unsigned IsFast;
4177 unsigned AS = SN->getAddressSpace();
4178
4179 // Expand unaligned stores earlier than legalization. Due to visitation
4180 // order problems during legalization, the emitted instructions to pack and
4181 // unpack the bytes again are not eliminated in the case of an unaligned
4182 // copy.
4183 if (!allowsMisalignedMemoryAccesses(
4184 VT, AddrSpace: AS, Alignment, Flags: SN->getMemOperand()->getFlags(), &IsFast)) {
4185 if (VT.isVector())
4186 return SplitVectorStore(Op: SDValue(SN, 0), DAG);
4187
4188 return expandUnalignedStore(ST: SN, DAG);
4189 }
4190
4191 if (!IsFast)
4192 return SDValue();
4193 }
4194
4195 if (!shouldCombineMemoryType(VT))
4196 return SDValue();
4197
4198 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4199 SDValue Val = SN->getValue();
4200
4201 // DCI.AddToWorklist(Val.getNode());
4202
4203 bool OtherUses = !Val.hasOneUse();
4204 SDValue CastVal = DAG.getBitcast(VT: NewVT, V: Val);
4205 if (OtherUses) {
4206 SDValue CastBack = DAG.getBitcast(VT, V: CastVal);
4207 DAG.ReplaceAllUsesOfValueWith(From: Val, To: CastBack);
4208 }
4209
4210 return DAG.getStore(Chain: SN->getChain(), dl: SL, Val: CastVal,
4211 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
4212}
4213
4214// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
4215// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
4216// issues.
4217SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
4218 DAGCombinerInfo &DCI) const {
4219 SelectionDAG &DAG = DCI.DAG;
4220 SDValue N0 = N->getOperand(Num: 0);
4221
4222 // (vt2 (assertzext (truncate vt0:x), vt1)) ->
4223 // (vt2 (truncate (assertzext vt0:x, vt1)))
4224 if (N0.getOpcode() == ISD::TRUNCATE) {
4225 SDValue N1 = N->getOperand(Num: 1);
4226 EVT ExtVT = cast<VTSDNode>(Val&: N1)->getVT();
4227 SDLoc SL(N);
4228
4229 SDValue Src = N0.getOperand(i: 0);
4230 EVT SrcVT = Src.getValueType();
4231 if (SrcVT.bitsGE(VT: ExtVT)) {
4232 SDValue NewInReg = DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: SrcVT, N1: Src, N2: N1);
4233 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: N->getValueType(ResNo: 0), Operand: NewInReg);
4234 }
4235 }
4236
4237 return SDValue();
4238}
4239
4240SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
4241 SDNode *N, DAGCombinerInfo &DCI) const {
4242 unsigned IID = N->getConstantOperandVal(Num: 0);
4243 switch (IID) {
4244 case Intrinsic::amdgcn_mul_i24:
4245 case Intrinsic::amdgcn_mul_u24:
4246 case Intrinsic::amdgcn_mulhi_i24:
4247 case Intrinsic::amdgcn_mulhi_u24:
4248 return simplifyMul24(Node24: N, DCI);
4249 case Intrinsic::amdgcn_fract:
4250 case Intrinsic::amdgcn_rsq:
4251 case Intrinsic::amdgcn_rcp_legacy:
4252 case Intrinsic::amdgcn_rsq_legacy:
4253 case Intrinsic::amdgcn_rsq_clamp:
4254 case Intrinsic::amdgcn_tanh:
4255 case Intrinsic::amdgcn_prng_b32: {
4256 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
4257 SDValue Src = N->getOperand(Num: 1);
4258 return Src.isUndef() ? Src : SDValue();
4259 }
4260 case Intrinsic::amdgcn_frexp_exp: {
4261 // frexp_exp (fneg x) -> frexp_exp x
4262 // frexp_exp (fabs x) -> frexp_exp x
4263 // frexp_exp (fneg (fabs x)) -> frexp_exp x
4264 SDValue Src = N->getOperand(Num: 1);
4265 SDValue PeekSign = peekFPSignOps(Val: Src);
4266 if (PeekSign == Src)
4267 return SDValue();
4268 return SDValue(DCI.DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: PeekSign),
4269 0);
4270 }
4271 default:
4272 return SDValue();
4273 }
4274}
4275
4276/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
4277/// binary operation \p Opc to it with the corresponding constant operands.
4278SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
4279 DAGCombinerInfo &DCI, const SDLoc &SL,
4280 unsigned Opc, SDValue LHS,
4281 uint32_t ValLo, uint32_t ValHi) const {
4282 SelectionDAG &DAG = DCI.DAG;
4283 SDValue Lo, Hi;
4284 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: LHS, DAG);
4285
4286 SDValue LoRHS = DAG.getConstant(Val: ValLo, DL: SL, VT: MVT::i32);
4287 SDValue HiRHS = DAG.getConstant(Val: ValHi, DL: SL, VT: MVT::i32);
4288
4289 SDValue LoAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Lo, N2: LoRHS);
4290 SDValue HiAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Hi, N2: HiRHS);
4291
4292 // Re-visit the ands. It's possible we eliminated one of them and it could
4293 // simplify the vector.
4294 DCI.AddToWorklist(N: Lo.getNode());
4295 DCI.AddToWorklist(N: Hi.getNode());
4296
4297 SDValue Vec = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoAnd, HiAnd});
4298 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
4299}
4300
4301SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
4302 DAGCombinerInfo &DCI) const {
4303 EVT VT = N->getValueType(ResNo: 0);
4304 SDValue LHS = N->getOperand(Num: 0);
4305 SDValue RHS = N->getOperand(Num: 1);
4306 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4307 SDLoc SL(N);
4308 SelectionDAG &DAG = DCI.DAG;
4309
4310 unsigned RHSVal;
4311 if (CRHS) {
4312 RHSVal = CRHS->getZExtValue();
4313 if (!RHSVal)
4314 return LHS;
4315
4316 switch (LHS->getOpcode()) {
4317 default:
4318 break;
4319 case ISD::ZERO_EXTEND:
4320 case ISD::SIGN_EXTEND:
4321 case ISD::ANY_EXTEND: {
4322 SDValue X = LHS->getOperand(Num: 0);
4323
4324 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
4325 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16)) {
4326 // Prefer build_vector as the canonical form if packed types are legal.
4327 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
4328 SDValue Vec = DAG.getBuildVector(
4329 VT: MVT::v2i16, DL: SL,
4330 Ops: {DAG.getConstant(Val: 0, DL: SL, VT: MVT::i16), LHS->getOperand(Num: 0)});
4331 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Vec);
4332 }
4333
4334 // shl (ext x) => zext (shl x), if shift does not overflow int
4335 if (VT != MVT::i64)
4336 break;
4337 KnownBits Known = DAG.computeKnownBits(Op: X);
4338 unsigned LZ = Known.countMinLeadingZeros();
4339 if (LZ < RHSVal)
4340 break;
4341 EVT XVT = X.getValueType();
4342 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: XVT, N1: X, N2: SDValue(CRHS, 0));
4343 return DAG.getZExtOrTrunc(Op: Shl, DL: SL, VT);
4344 }
4345 }
4346 }
4347
4348 if (VT.getScalarType() != MVT::i64)
4349 return SDValue();
4350
4351 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4352 // common case, splitting this into a move and a 32-bit shift is faster and
4353 // the same code size.
4354 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4355
4356 EVT ElementType = VT.getScalarType();
4357 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4358 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4359
4360 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4361 return SDValue();
4362 SDValue ShiftAmt;
4363
4364 if (CRHS) {
4365 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4366 VT: TargetType);
4367 } else {
4368 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4369 const SDValue ShiftMask =
4370 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4371 // This AND instruction will clamp out of bounds shift values.
4372 // It will also be removed during later instruction selection.
4373 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4374 }
4375
4376 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: LHS);
4377 SDValue NewShift =
4378 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: TargetType, N1: Lo, N2: ShiftAmt, Flags: N->getFlags());
4379
4380 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4381 SDValue Vec;
4382
4383 if (VT.isVector()) {
4384 EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4385 unsigned NElts = TargetType.getVectorNumElements();
4386 SmallVector<SDValue, 8> HiOps;
4387 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4388
4389 DAG.ExtractVectorElements(Op: NewShift, Args&: HiOps, Start: 0, Count: NElts);
4390 for (unsigned I = 0; I != NElts; ++I)
4391 HiAndLoOps[2 * I + 1] = HiOps[I];
4392 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4393 } else {
4394 EVT ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4395 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {Zero, NewShift});
4396 }
4397 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4398}
4399
4400SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
4401 DAGCombinerInfo &DCI) const {
4402 SDValue RHS = N->getOperand(Num: 1);
4403 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4404 EVT VT = N->getValueType(ResNo: 0);
4405 SDValue LHS = N->getOperand(Num: 0);
4406 SelectionDAG &DAG = DCI.DAG;
4407 SDLoc SL(N);
4408
4409 if (VT.getScalarType() != MVT::i64)
4410 return SDValue();
4411
4412 // For C >= 32
4413 // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
4414
4415 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4416 // common case, splitting this into a move and a 32-bit shift is faster and
4417 // the same code size.
4418 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4419
4420 EVT ElementType = VT.getScalarType();
4421 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4422 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4423
4424 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4425 return SDValue();
4426
4427 SDValue ShiftFullAmt =
4428 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4429 SDValue ShiftAmt;
4430 if (CRHS) {
4431 unsigned RHSVal = CRHS->getZExtValue();
4432 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4433 VT: TargetType);
4434 } else if (Known.getMinValue().getZExtValue() ==
4435 (ElementType.getSizeInBits() - 1)) {
4436 ShiftAmt = ShiftFullAmt;
4437 } else {
4438 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4439 const SDValue ShiftMask =
4440 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4441 // This AND instruction will clamp out of bounds shift values.
4442 // It will also be removed during later instruction selection.
4443 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4444 }
4445
4446 EVT ConcatType;
4447 SDValue Hi;
4448 SDLoc LHSSL(LHS);
4449 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4450 if (VT.isVector()) {
4451 unsigned NElts = TargetType.getVectorNumElements();
4452 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4453 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4454 SmallVector<SDValue, 8> HiOps(NElts);
4455 SmallVector<SDValue, 16> HiAndLoOps;
4456
4457 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, Start: 0, Count: NElts * 2);
4458 for (unsigned I = 0; I != NElts; ++I) {
4459 HiOps[I] = HiAndLoOps[2 * I + 1];
4460 }
4461 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4462 } else {
4463 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4464 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4465 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4466 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4467 }
4468
4469 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
4470 SDValue NewShift, HiShift;
4471 if (KnownLHS.isNegative()) {
4472 HiShift = DAG.getAllOnesConstant(DL: SL, VT: TargetType);
4473 NewShift =
4474 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4475 } else if (CRHS &&
4476 CRHS->getZExtValue() == (ElementType.getSizeInBits() - 1)) {
4477 NewShift = HiShift =
4478 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4479 } else {
4480 Hi = DAG.getFreeze(V: Hi);
4481 HiShift = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftFullAmt);
4482 NewShift =
4483 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4484 }
4485
4486 SDValue Vec;
4487 if (VT.isVector()) {
4488 unsigned NElts = TargetType.getVectorNumElements();
4489 SmallVector<SDValue, 8> HiOps;
4490 SmallVector<SDValue, 8> LoOps;
4491 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
4492
4493 DAG.ExtractVectorElements(Op: HiShift, Args&: HiOps, Start: 0, Count: NElts);
4494 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4495 for (unsigned I = 0; I != NElts; ++I) {
4496 HiAndLoOps[2 * I + 1] = HiOps[I];
4497 HiAndLoOps[2 * I] = LoOps[I];
4498 }
4499 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4500 } else {
4501 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, HiShift});
4502 }
4503 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4504}
4505
4506SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
4507 DAGCombinerInfo &DCI) const {
4508 SDValue RHS = N->getOperand(Num: 1);
4509 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4510 EVT VT = N->getValueType(ResNo: 0);
4511 SDValue LHS = N->getOperand(Num: 0);
4512 SelectionDAG &DAG = DCI.DAG;
4513 SDLoc SL(N);
4514 unsigned RHSVal;
4515
4516 if (CRHS) {
4517 RHSVal = CRHS->getZExtValue();
4518
4519 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
4520 // this improves the ability to match BFE patterns in isel.
4521 if (LHS.getOpcode() == ISD::AND) {
4522 if (auto *Mask = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
4523 unsigned MaskIdx, MaskLen;
4524 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4525 MaskIdx == RHSVal) {
4526 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT,
4527 N1: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 0),
4528 N2: N->getOperand(Num: 1)),
4529 N2: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 1),
4530 N2: N->getOperand(Num: 1)));
4531 }
4532 }
4533 }
4534 }
4535
4536 if (VT.getScalarType() != MVT::i64)
4537 return SDValue();
4538
4539 // for C >= 32
4540 // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
4541
4542 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4543 // common case, splitting this into a move and a 32-bit shift is faster and
4544 // the same code size.
4545 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4546
4547 EVT ElementType = VT.getScalarType();
4548 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4549 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4550
4551 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4552 return SDValue();
4553
4554 SDValue ShiftAmt;
4555 if (CRHS) {
4556 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4557 VT: TargetType);
4558 } else {
4559 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4560 const SDValue ShiftMask =
4561 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4562 // This AND instruction will clamp out of bounds shift values.
4563 // It will also be removed during later instruction selection.
4564 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4565 }
4566
4567 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4568 EVT ConcatType;
4569 SDValue Hi;
4570 SDLoc LHSSL(LHS);
4571 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4572 if (VT.isVector()) {
4573 unsigned NElts = TargetType.getVectorNumElements();
4574 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4575 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4576 SmallVector<SDValue, 8> HiOps(NElts);
4577 SmallVector<SDValue, 16> HiAndLoOps;
4578
4579 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, /*Start=*/0, Count: NElts * 2);
4580 for (unsigned I = 0; I != NElts; ++I)
4581 HiOps[I] = HiAndLoOps[2 * I + 1];
4582 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4583 } else {
4584 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4585 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4586 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4587 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4588 }
4589
4590 SDValue NewShift =
4591 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4592
4593 SDValue Vec;
4594 if (VT.isVector()) {
4595 unsigned NElts = TargetType.getVectorNumElements();
4596 SmallVector<SDValue, 8> LoOps;
4597 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4598
4599 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4600 for (unsigned I = 0; I != NElts; ++I)
4601 HiAndLoOps[2 * I] = LoOps[I];
4602 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4603 } else {
4604 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, Zero});
4605 }
4606 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4607}
4608
4609SDValue AMDGPUTargetLowering::performTruncateCombine(
4610 SDNode *N, DAGCombinerInfo &DCI) const {
4611 SDLoc SL(N);
4612 SelectionDAG &DAG = DCI.DAG;
4613 EVT VT = N->getValueType(ResNo: 0);
4614 SDValue Src = N->getOperand(Num: 0);
4615
4616 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
4617 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
4618 SDValue Vec = Src.getOperand(i: 0);
4619 if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
4620 SDValue Elt0 = Vec.getOperand(i: 0);
4621 EVT EltVT = Elt0.getValueType();
4622 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
4623 if (EltVT.isFloatingPoint()) {
4624 Elt0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL,
4625 VT: EltVT.changeTypeToInteger(), Operand: Elt0);
4626 }
4627
4628 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Elt0);
4629 }
4630 }
4631 }
4632
4633 // Equivalent of above for accessing the high element of a vector as an
4634 // integer operation.
4635 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
4636 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
4637 if (auto *K = isConstOrConstSplat(N: Src.getOperand(i: 1))) {
4638 SDValue BV = stripBitcast(Val: Src.getOperand(i: 0));
4639 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
4640 EVT SrcEltVT = BV.getOperand(i: 0).getValueType();
4641 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
4642 unsigned BitIndex = K->getZExtValue();
4643 unsigned PartIndex = BitIndex / SrcEltSize;
4644
4645 if (PartIndex * SrcEltSize == BitIndex &&
4646 PartIndex < BV.getNumOperands()) {
4647 if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
4648 SDValue SrcElt =
4649 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcEltVT.changeTypeToInteger(),
4650 Operand: BV.getOperand(i: PartIndex));
4651 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: SrcElt);
4652 }
4653 }
4654 }
4655 }
4656 }
4657
4658 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
4659 //
4660 // i16 (trunc (srl i64:x, K)), K <= 16 ->
4661 // i16 (trunc (srl (i32 (trunc x), K)))
4662 if (VT.getScalarSizeInBits() < 32) {
4663 EVT SrcVT = Src.getValueType();
4664 if (SrcVT.getScalarSizeInBits() > 32 &&
4665 (Src.getOpcode() == ISD::SRL ||
4666 Src.getOpcode() == ISD::SRA ||
4667 Src.getOpcode() == ISD::SHL)) {
4668 SDValue Amt = Src.getOperand(i: 1);
4669 KnownBits Known = DAG.computeKnownBits(Op: Amt);
4670
4671 // - For left shifts, do the transform as long as the shift
4672 // amount is still legal for i32, so when ShiftAmt < 32 (<= 31)
4673 // - For right shift, do it if ShiftAmt <= (32 - Size) to avoid
4674 // losing information stored in the high bits when truncating.
4675 const unsigned MaxCstSize =
4676 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
4677 if (Known.getMaxValue().ule(RHS: MaxCstSize)) {
4678 EVT MidVT = VT.isVector() ?
4679 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
4680 NumElements: VT.getVectorNumElements()) : MVT::i32;
4681
4682 EVT NewShiftVT = getShiftAmountTy(LHSTy: MidVT, DL: DAG.getDataLayout());
4683 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MidVT,
4684 Operand: Src.getOperand(i: 0));
4685 DCI.AddToWorklist(N: Trunc.getNode());
4686
4687 if (Amt.getValueType() != NewShiftVT) {
4688 Amt = DAG.getZExtOrTrunc(Op: Amt, DL: SL, VT: NewShiftVT);
4689 DCI.AddToWorklist(N: Amt.getNode());
4690 }
4691
4692 SDValue ShrunkShift = DAG.getNode(Opcode: Src.getOpcode(), DL: SL, VT: MidVT,
4693 N1: Trunc, N2: Amt);
4694 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: ShrunkShift);
4695 }
4696 }
4697 }
4698
4699 return SDValue();
4700}
4701
4702// We need to specifically handle i64 mul here to avoid unnecessary conversion
4703// instructions. If we only match on the legalized i64 mul expansion,
4704// SimplifyDemandedBits will be unable to remove them because there will be
4705// multiple uses due to the separate mul + mulh[su].
4706static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
4707 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
4708 if (Size <= 32) {
4709 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4710 return DAG.getNode(Opcode: MulOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4711 }
4712
4713 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4714 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
4715
4716 SDValue MulLo = DAG.getNode(Opcode: MulLoOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4717 SDValue MulHi = DAG.getNode(Opcode: MulHiOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4718
4719 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: MulLo, N2: MulHi);
4720}
4721
4722/// If \p V is an add of a constant 1, returns the other operand. Otherwise
4723/// return SDValue().
4724static SDValue getAddOneOp(const SDNode *V) {
4725 if (V->getOpcode() != ISD::ADD)
4726 return SDValue();
4727
4728 return isOneConstant(V: V->getOperand(Num: 1)) ? V->getOperand(Num: 0) : SDValue();
4729}
4730
4731SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
4732 DAGCombinerInfo &DCI) const {
4733 assert(N->getOpcode() == ISD::MUL);
4734 EVT VT = N->getValueType(ResNo: 0);
4735
4736 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4737 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4738 // unnecessarily). isDivergent() is used as an approximation of whether the
4739 // value is in an SGPR.
4740 if (!N->isDivergent())
4741 return SDValue();
4742
4743 unsigned Size = VT.getSizeInBits();
4744 if (VT.isVector() || Size > 64)
4745 return SDValue();
4746
4747 SelectionDAG &DAG = DCI.DAG;
4748 SDLoc DL(N);
4749
4750 SDValue N0 = N->getOperand(Num: 0);
4751 SDValue N1 = N->getOperand(Num: 1);
4752
4753 // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
4754 // matching.
4755
4756 // mul x, (add y, 1) -> add (mul x, y), x
4757 auto IsFoldableAdd = [](SDValue V) -> SDValue {
4758 SDValue AddOp = getAddOneOp(V: V.getNode());
4759 if (!AddOp)
4760 return SDValue();
4761
4762 if (V.hasOneUse() || all_of(Range: V->users(), P: [](const SDNode *U) -> bool {
4763 return U->getOpcode() == ISD::MUL;
4764 }))
4765 return AddOp;
4766
4767 return SDValue();
4768 };
4769
4770 // FIXME: The selection pattern is not properly checking for commuted
4771 // operands, so we have to place the mul in the LHS
4772 if (SDValue MulOper = IsFoldableAdd(N0)) {
4773 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1, N2: MulOper);
4774 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N1);
4775 }
4776
4777 if (SDValue MulOper = IsFoldableAdd(N1)) {
4778 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N0, N2: MulOper);
4779 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N0);
4780 }
4781
4782 // There are i16 integer mul/mad.
4783 if (isTypeLegal(VT: MVT::i16) && VT.getScalarType().bitsLE(VT: MVT::i16))
4784 return SDValue();
4785
4786 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4787 // in the source into any_extends if the result of the mul is truncated. Since
4788 // we can assume the high bits are whatever we want, use the underlying value
4789 // to avoid the unknown high bits from interfering.
4790 if (N0.getOpcode() == ISD::ANY_EXTEND)
4791 N0 = N0.getOperand(i: 0);
4792
4793 if (N1.getOpcode() == ISD::ANY_EXTEND)
4794 N1 = N1.getOperand(i: 0);
4795
4796 SDValue Mul;
4797
4798 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4799 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4800 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4801 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: false);
4802 } else if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4803 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4804 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4805 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: true);
4806 } else {
4807 return SDValue();
4808 }
4809
4810 // We need to use sext even for MUL_U24, because MUL_U24 is used
4811 // for signed multiply of 8 and 16-bit types.
4812 return DAG.getSExtOrTrunc(Op: Mul, DL, VT);
4813}
4814
4815SDValue
4816AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N,
4817 DAGCombinerInfo &DCI) const {
4818 if (N->getValueType(ResNo: 0) != MVT::i32)
4819 return SDValue();
4820
4821 SelectionDAG &DAG = DCI.DAG;
4822 SDLoc DL(N);
4823
4824 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
4825 SDValue N0 = N->getOperand(Num: 0);
4826 SDValue N1 = N->getOperand(Num: 1);
4827
4828 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4829 // in the source into any_extends if the result of the mul is truncated. Since
4830 // we can assume the high bits are whatever we want, use the underlying value
4831 // to avoid the unknown high bits from interfering.
4832 if (N0.getOpcode() == ISD::ANY_EXTEND)
4833 N0 = N0.getOperand(i: 0);
4834 if (N1.getOpcode() == ISD::ANY_EXTEND)
4835 N1 = N1.getOperand(i: 0);
4836
4837 // Try to use two fast 24-bit multiplies (one for each half of the result)
4838 // instead of one slow extending multiply.
4839 unsigned LoOpcode = 0;
4840 unsigned HiOpcode = 0;
4841 if (Signed) {
4842 if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4843 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4844 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4845 LoOpcode = AMDGPUISD::MUL_I24;
4846 HiOpcode = AMDGPUISD::MULHI_I24;
4847 }
4848 } else {
4849 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4850 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4851 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4852 LoOpcode = AMDGPUISD::MUL_U24;
4853 HiOpcode = AMDGPUISD::MULHI_U24;
4854 }
4855 }
4856 if (!LoOpcode)
4857 return SDValue();
4858
4859 SDValue Lo = DAG.getNode(Opcode: LoOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4860 SDValue Hi = DAG.getNode(Opcode: HiOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4861 DCI.CombineTo(N, Res0: Lo, Res1: Hi);
4862 return SDValue(N, 0);
4863}
4864
4865SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
4866 DAGCombinerInfo &DCI) const {
4867 EVT VT = N->getValueType(ResNo: 0);
4868
4869 if (!Subtarget->hasMulI24() || VT.isVector())
4870 return SDValue();
4871
4872 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4873 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4874 // unnecessarily). isDivergent() is used as an approximation of whether the
4875 // value is in an SGPR.
4876 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4877 // valu op anyway)
4878 if (Subtarget->hasSMulHi() && !N->isDivergent())
4879 return SDValue();
4880
4881 SelectionDAG &DAG = DCI.DAG;
4882 SDLoc DL(N);
4883
4884 SDValue N0 = N->getOperand(Num: 0);
4885 SDValue N1 = N->getOperand(Num: 1);
4886
4887 if (!isI24(Op: N0, DAG) || !isI24(Op: N1, DAG))
4888 return SDValue();
4889
4890 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4891 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4892
4893 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_I24, DL, VT: MVT::i32, N1: N0, N2: N1);
4894 DCI.AddToWorklist(N: Mulhi.getNode());
4895 return DAG.getSExtOrTrunc(Op: Mulhi, DL, VT);
4896}
4897
4898SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
4899 DAGCombinerInfo &DCI) const {
4900 EVT VT = N->getValueType(ResNo: 0);
4901
4902 if (VT.isVector() || VT.getSizeInBits() > 32 || !Subtarget->hasMulU24())
4903 return SDValue();
4904
4905 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4906 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4907 // unnecessarily). isDivergent() is used as an approximation of whether the
4908 // value is in an SGPR.
4909 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4910 // valu op anyway)
4911 if (!N->isDivergent() && Subtarget->hasSMulHi())
4912 return SDValue();
4913
4914 SelectionDAG &DAG = DCI.DAG;
4915 SDLoc DL(N);
4916
4917 SDValue N0 = N->getOperand(Num: 0);
4918 SDValue N1 = N->getOperand(Num: 1);
4919
4920 if (!isU24(Op: N0, DAG) || !isU24(Op: N1, DAG))
4921 return SDValue();
4922
4923 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4924 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4925
4926 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_U24, DL, VT: MVT::i32, N1: N0, N2: N1);
4927 DCI.AddToWorklist(N: Mulhi.getNode());
4928 return DAG.getZExtOrTrunc(Op: Mulhi, DL, VT);
4929}
4930
4931SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
4932 SDValue Op,
4933 const SDLoc &DL,
4934 unsigned Opc) const {
4935 EVT VT = Op.getValueType();
4936 if (VT.bitsGT(VT: MVT::i32))
4937 return SDValue();
4938
4939 if (VT != MVT::i32)
4940 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: Op);
4941
4942 SDValue FFBX = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, Operand: Op);
4943 if (VT != MVT::i32)
4944 FFBX = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FFBX);
4945
4946 return FFBX;
4947}
4948
4949// The native instructions return -1 on 0 input. Optimize out a select that
4950// produces -1 on 0.
4951//
4952// TODO: If zero is not undef, we could also do this if the output is compared
4953// against the bitwidth.
4954//
4955// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
4956SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
4957 SDValue LHS, SDValue RHS,
4958 DAGCombinerInfo &DCI) const {
4959 if (!isNullConstant(V: Cond.getOperand(i: 1)))
4960 return SDValue();
4961
4962 SelectionDAG &DAG = DCI.DAG;
4963 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
4964 SDValue CmpLHS = Cond.getOperand(i: 0);
4965
4966 // select (setcc x, 0, eq), -1, (ctlz_zero_poison x) -> ffbh_u32 x
4967 // select (setcc x, 0, eq), -1, (cttz_zero_poison x) -> ffbl_u32 x
4968 if (CCOpcode == ISD::SETEQ &&
4969 (isCtlzOpc(Opc: RHS.getOpcode()) || isCttzOpc(Opc: RHS.getOpcode())) &&
4970 RHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: LHS)) {
4971 unsigned Opc =
4972 isCttzOpc(Opc: RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4973 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4974 }
4975
4976 // select (setcc x, 0, ne), (ctlz_zero_poison x), -1 -> ffbh_u32 x
4977 // select (setcc x, 0, ne), (cttz_zero_poison x), -1 -> ffbl_u32 x
4978 if (CCOpcode == ISD::SETNE &&
4979 (isCtlzOpc(Opc: LHS.getOpcode()) || isCttzOpc(Opc: LHS.getOpcode())) &&
4980 LHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: RHS)) {
4981 unsigned Opc =
4982 isCttzOpc(Opc: LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4983
4984 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4985 }
4986
4987 return SDValue();
4988}
4989
4990static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
4991 unsigned Op,
4992 const SDLoc &SL,
4993 SDValue Cond,
4994 SDValue N1,
4995 SDValue N2) {
4996 SelectionDAG &DAG = DCI.DAG;
4997 EVT VT = N1.getValueType();
4998
4999 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cond,
5000 N2: N1.getOperand(i: 0), N3: N2.getOperand(i: 0));
5001 DCI.AddToWorklist(N: NewSelect.getNode());
5002 return DAG.getNode(Opcode: Op, DL: SL, VT, Operand: NewSelect);
5003}
5004
5005// Pull a free FP operation out of a select so it may fold into uses.
5006//
5007// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
5008// select c, (fneg x), k -> fneg (select c, x, (fneg k))
5009//
5010// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
5011// select c, (fabs x), +k -> fabs (select c, x, k)
5012SDValue
5013AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
5014 SDValue N) const {
5015 SelectionDAG &DAG = DCI.DAG;
5016 SDValue Cond = N.getOperand(i: 0);
5017 SDValue LHS = N.getOperand(i: 1);
5018 SDValue RHS = N.getOperand(i: 2);
5019
5020 EVT VT = N.getValueType();
5021 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
5022 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
5023 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5024 return SDValue();
5025
5026 return distributeOpThroughSelect(DCI, Op: LHS.getOpcode(),
5027 SL: SDLoc(N), Cond, N1: LHS, N2: RHS);
5028 }
5029
5030 bool Inv = false;
5031 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
5032 std::swap(a&: LHS, b&: RHS);
5033 Inv = true;
5034 }
5035
5036 // TODO: Support vector constants.
5037 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
5038 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS &&
5039 !selectSupportsSourceMods(N: N.getNode())) {
5040 SDLoc SL(N);
5041 // If one side is an fneg/fabs and the other is a constant, we can push the
5042 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
5043 SDValue NewLHS = LHS.getOperand(i: 0);
5044 SDValue NewRHS = RHS;
5045
5046 // Careful: if the neg can be folded up, don't try to pull it back down.
5047 bool ShouldFoldNeg = true;
5048
5049 if (NewLHS.hasOneUse()) {
5050 unsigned Opc = NewLHS.getOpcode();
5051 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(N: NewLHS.getNode()))
5052 ShouldFoldNeg = false;
5053 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
5054 ShouldFoldNeg = false;
5055 }
5056
5057 if (ShouldFoldNeg) {
5058 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative())
5059 return SDValue();
5060
5061 // We're going to be forced to use a source modifier anyway, there's no
5062 // point to pulling the negate out unless we can get a size reduction by
5063 // negating the constant.
5064 //
5065 // TODO: Generalize to use getCheaperNegatedExpression which doesn't know
5066 // about cheaper constants.
5067 if (NewLHS.getOpcode() == ISD::FABS &&
5068 getConstantNegateCost(C: CRHS) != NegatibleCost::Cheaper)
5069 return SDValue();
5070
5071 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5072 return SDValue();
5073
5074 if (LHS.getOpcode() == ISD::FNEG)
5075 NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5076
5077 if (Inv)
5078 std::swap(a&: NewLHS, b&: NewRHS);
5079
5080 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT,
5081 N1: Cond, N2: NewLHS, N3: NewRHS);
5082 DCI.AddToWorklist(N: NewSelect.getNode());
5083 return DAG.getNode(Opcode: LHS.getOpcode(), DL: SL, VT, Operand: NewSelect);
5084 }
5085 }
5086
5087 return SDValue();
5088}
5089
5090SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
5091 DAGCombinerInfo &DCI) const {
5092 if (SDValue Folded = foldFreeOpFromSelect(DCI, N: SDValue(N, 0)))
5093 return Folded;
5094
5095 SDValue Cond = N->getOperand(Num: 0);
5096 if (Cond.getOpcode() != ISD::SETCC)
5097 return SDValue();
5098
5099 EVT VT = N->getValueType(ResNo: 0);
5100 SDValue LHS = Cond.getOperand(i: 0);
5101 SDValue RHS = Cond.getOperand(i: 1);
5102 SDValue CC = Cond.getOperand(i: 2);
5103
5104 SDValue True = N->getOperand(Num: 1);
5105 SDValue False = N->getOperand(Num: 2);
5106
5107 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
5108 SelectionDAG &DAG = DCI.DAG;
5109 if (DAG.isConstantValueOfAnyType(N: True) &&
5110 !DAG.isConstantValueOfAnyType(N: False)) {
5111 // Swap cmp + select pair to move constant to false input.
5112 // This will allow using VOPC cndmasks more often.
5113 // select (setcc x, y), k, x -> select (setccinv x, y), x, k
5114
5115 SDLoc SL(N);
5116 ISD::CondCode NewCC =
5117 getSetCCInverse(Operation: cast<CondCodeSDNode>(Val&: CC)->get(), Type: LHS.getValueType());
5118
5119 SDValue NewCond = DAG.getSetCC(DL: SL, VT: Cond.getValueType(), LHS, RHS, Cond: NewCC);
5120 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NewCond, N2: False, N3: True);
5121 }
5122
5123 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
5124 SDValue MinMax
5125 = combineFMinMaxLegacy(DL: SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
5126 // Revisit this node so we can catch min3/max3/med3 patterns.
5127 //DCI.AddToWorklist(MinMax.getNode());
5128 return MinMax;
5129 }
5130 }
5131
5132 // There's no reason to not do this if the condition has other uses.
5133 return performCtlz_CttzCombine(SL: SDLoc(N), Cond, LHS: True, RHS: False, DCI);
5134}
5135
5136static bool isInv2Pi(const APFloat &APF) {
5137 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
5138 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
5139 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
5140
5141 return APF.bitwiseIsEqual(RHS: KF16) ||
5142 APF.bitwiseIsEqual(RHS: KF32) ||
5143 APF.bitwiseIsEqual(RHS: KF64);
5144}
5145
5146// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
5147// additional cost to negate them.
5148TargetLowering::NegatibleCost
5149AMDGPUTargetLowering::getConstantNegateCost(const ConstantFPSDNode *C) const {
5150 if (C->isZero())
5151 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5152
5153 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(APF: C->getValueAPF()))
5154 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5155
5156 return NegatibleCost::Neutral;
5157}
5158
5159bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
5160 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5161 return getConstantNegateCost(C) == NegatibleCost::Expensive;
5162 return false;
5163}
5164
5165bool AMDGPUTargetLowering::isConstantCheaperToNegate(SDValue N) const {
5166 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5167 return getConstantNegateCost(C) == NegatibleCost::Cheaper;
5168 return false;
5169}
5170
5171static unsigned inverseMinMax(unsigned Opc) {
5172 switch (Opc) {
5173 case ISD::FMAXNUM:
5174 return ISD::FMINNUM;
5175 case ISD::FMINNUM:
5176 return ISD::FMAXNUM;
5177 case ISD::FMAXNUM_IEEE:
5178 return ISD::FMINNUM_IEEE;
5179 case ISD::FMINNUM_IEEE:
5180 return ISD::FMAXNUM_IEEE;
5181 case ISD::FMAXIMUM:
5182 return ISD::FMINIMUM;
5183 case ISD::FMINIMUM:
5184 return ISD::FMAXIMUM;
5185 case ISD::FMAXIMUMNUM:
5186 return ISD::FMINIMUMNUM;
5187 case ISD::FMINIMUMNUM:
5188 return ISD::FMAXIMUMNUM;
5189 case AMDGPUISD::FMAX_LEGACY:
5190 return AMDGPUISD::FMIN_LEGACY;
5191 case AMDGPUISD::FMIN_LEGACY:
5192 return AMDGPUISD::FMAX_LEGACY;
5193 default:
5194 llvm_unreachable("invalid min/max opcode");
5195 }
5196}
5197
5198/// \return true if it's profitable to try to push an fneg into its source
5199/// instruction.
5200bool AMDGPUTargetLowering::shouldFoldFNegIntoSrc(SDNode *N, SDValue N0) {
5201 // If the input has multiple uses and we can either fold the negate down, or
5202 // the other uses cannot, give up. This both prevents unprofitable
5203 // transformations and infinite loops: we won't repeatedly try to fold around
5204 // a negate that has no 'good' form.
5205 if (N0.hasOneUse()) {
5206 // This may be able to fold into the source, but at a code size cost. Don't
5207 // fold if the fold into the user is free.
5208 if (allUsesHaveSourceMods(N, CostThreshold: 0))
5209 return false;
5210 } else {
5211 if (fnegFoldsIntoOp(N: N0.getNode()) &&
5212 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N: N0.getNode())))
5213 return false;
5214 }
5215
5216 return true;
5217}
5218
5219SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
5220 DAGCombinerInfo &DCI) const {
5221 SelectionDAG &DAG = DCI.DAG;
5222 SDValue N0 = N->getOperand(Num: 0);
5223 EVT VT = N->getValueType(ResNo: 0);
5224
5225 unsigned Opc = N0.getOpcode();
5226
5227 if (!shouldFoldFNegIntoSrc(N, N0))
5228 return SDValue();
5229
5230 SDLoc SL(N);
5231 switch (Opc) {
5232 case ISD::FADD: {
5233 if (!mayIgnoreSignedZero(Op: N0) && !N->getFlags().hasNoSignedZeros())
5234 return SDValue();
5235
5236 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
5237 SDValue LHS = N0.getOperand(i: 0);
5238 SDValue RHS = N0.getOperand(i: 1);
5239
5240 if (LHS.getOpcode() != ISD::FNEG)
5241 LHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5242 else
5243 LHS = LHS.getOperand(i: 0);
5244
5245 if (RHS.getOpcode() != ISD::FNEG)
5246 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5247 else
5248 RHS = RHS.getOperand(i: 0);
5249
5250 SDValue Res = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5251 if (Res.getOpcode() != ISD::FADD)
5252 return SDValue(); // Op got folded away.
5253 if (!N0.hasOneUse())
5254 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5255 return Res;
5256 }
5257 case ISD::FMUL:
5258 case AMDGPUISD::FMUL_LEGACY: {
5259 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
5260 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
5261 SDValue LHS = N0.getOperand(i: 0);
5262 SDValue RHS = N0.getOperand(i: 1);
5263
5264 if (LHS.getOpcode() == ISD::FNEG)
5265 LHS = LHS.getOperand(i: 0);
5266 else if (RHS.getOpcode() == ISD::FNEG)
5267 RHS = RHS.getOperand(i: 0);
5268 else
5269 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5270
5271 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5272 if (Res.getOpcode() != Opc)
5273 return SDValue(); // Op got folded away.
5274 if (!N0.hasOneUse())
5275 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5276 return Res;
5277 }
5278 case ISD::FMA:
5279 case ISD::FMAD: {
5280 // TODO: handle llvm.amdgcn.fma.legacy
5281 if (!mayIgnoreSignedZero(Op: N0) && !N->getFlags().hasNoSignedZeros())
5282 return SDValue();
5283
5284 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
5285 SDValue LHS = N0.getOperand(i: 0);
5286 SDValue MHS = N0.getOperand(i: 1);
5287 SDValue RHS = N0.getOperand(i: 2);
5288
5289 if (LHS.getOpcode() == ISD::FNEG)
5290 LHS = LHS.getOperand(i: 0);
5291 else if (MHS.getOpcode() == ISD::FNEG)
5292 MHS = MHS.getOperand(i: 0);
5293 else
5294 MHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: MHS);
5295
5296 if (RHS.getOpcode() != ISD::FNEG)
5297 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5298 else
5299 RHS = RHS.getOperand(i: 0);
5300
5301 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: MHS, N3: RHS);
5302 if (Res.getOpcode() != Opc)
5303 return SDValue(); // Op got folded away.
5304 if (!N0.hasOneUse())
5305 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5306 return Res;
5307 }
5308 case ISD::FMAXNUM:
5309 case ISD::FMINNUM:
5310 case ISD::FMAXNUM_IEEE:
5311 case ISD::FMINNUM_IEEE:
5312 case ISD::FMINIMUM:
5313 case ISD::FMAXIMUM:
5314 case ISD::FMINIMUMNUM:
5315 case ISD::FMAXIMUMNUM:
5316 case AMDGPUISD::FMAX_LEGACY:
5317 case AMDGPUISD::FMIN_LEGACY: {
5318 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
5319 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
5320 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
5321 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
5322
5323 SDValue LHS = N0.getOperand(i: 0);
5324 SDValue RHS = N0.getOperand(i: 1);
5325
5326 // 0 doesn't have a negated inline immediate.
5327 // TODO: This constant check should be generalized to other operations.
5328 if (isConstantCostlierToNegate(N: RHS))
5329 return SDValue();
5330
5331 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5332 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5333 unsigned Opposite = inverseMinMax(Opc);
5334
5335 SDValue Res = DAG.getNode(Opcode: Opposite, DL: SL, VT, N1: NegLHS, N2: NegRHS, Flags: N0->getFlags());
5336 if (Res.getOpcode() != Opposite)
5337 return SDValue(); // Op got folded away.
5338 if (!N0.hasOneUse())
5339 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5340 return Res;
5341 }
5342 case AMDGPUISD::FMED3: {
5343 // med3 sorts a NaN input as smaller than everything regardless of its sign,
5344 // so negating all operands does not sign-flip the median when an input may
5345 // be NaN.
5346 if (!N0->getFlags().hasNoNaNs())
5347 return SDValue();
5348
5349 SDValue Ops[3];
5350 for (unsigned I = 0; I < 3; ++I)
5351 Ops[I] = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N0->getOperand(Num: I), Flags: N0->getFlags());
5352
5353 SDValue Res = DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT, Ops, Flags: N0->getFlags());
5354 if (Res.getOpcode() != AMDGPUISD::FMED3)
5355 return SDValue(); // Op got folded away.
5356
5357 if (!N0.hasOneUse()) {
5358 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res);
5359 DAG.ReplaceAllUsesWith(From: N0, To: Neg);
5360
5361 for (SDNode *U : Neg->users())
5362 DCI.AddToWorklist(N: U);
5363 }
5364
5365 return Res;
5366 }
5367 case ISD::FP_EXTEND:
5368 case ISD::FTRUNC:
5369 case ISD::FRINT:
5370 case ISD::FNEARBYINT: // XXX - Should fround be handled?
5371 case ISD::FROUNDEVEN:
5372 case ISD::FSIN:
5373 case ISD::FCANONICALIZE:
5374 case AMDGPUISD::RCP:
5375 case AMDGPUISD::RCP_LEGACY:
5376 case AMDGPUISD::RCP_IFLAG:
5377 case AMDGPUISD::SIN_HW: {
5378 SDValue CvtSrc = N0.getOperand(i: 0);
5379 if (CvtSrc.getOpcode() == ISD::FNEG) {
5380 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
5381 // (fneg (rcp (fneg x))) -> (rcp x)
5382 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: CvtSrc.getOperand(i: 0));
5383 }
5384
5385 if (!N0.hasOneUse())
5386 return SDValue();
5387
5388 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
5389 // (fneg (rcp x)) -> (rcp (fneg x))
5390 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5391 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: Neg, Flags: N0->getFlags());
5392 }
5393 case ISD::FP_ROUND: {
5394 SDValue CvtSrc = N0.getOperand(i: 0);
5395
5396 if (CvtSrc.getOpcode() == ISD::FNEG) {
5397 // (fneg (fp_round (fneg x))) -> (fp_round x)
5398 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT,
5399 N1: CvtSrc.getOperand(i: 0), N2: N0.getOperand(i: 1));
5400 }
5401
5402 if (!N0.hasOneUse())
5403 return SDValue();
5404
5405 // (fneg (fp_round x)) -> (fp_round (fneg x))
5406 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5407 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Neg, N2: N0.getOperand(i: 1));
5408 }
5409 case ISD::FP16_TO_FP: {
5410 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
5411 // f16, but legalization of f16 fneg ends up pulling it out of the source.
5412 // Put the fneg back as a legal source operation that can be matched later.
5413 SDLoc SL(N);
5414
5415 SDValue Src = N0.getOperand(i: 0);
5416 EVT SrcVT = Src.getValueType();
5417
5418 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
5419 SDValue IntFNeg = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: SrcVT, N1: Src,
5420 N2: DAG.getConstant(Val: 0x8000, DL: SL, VT: SrcVT));
5421 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFNeg);
5422 }
5423 case ISD::SELECT: {
5424 // fneg (select c, a, b) -> select c, (fneg a), (fneg b)
5425 // TODO: Invert conditions of foldFreeOpFromSelect
5426 return SDValue();
5427 }
5428 case ISD::BITCAST: {
5429 SDLoc SL(N);
5430 SDValue BCSrc = N0.getOperand(i: 0);
5431 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
5432 SDValue HighBits = BCSrc.getOperand(i: BCSrc.getNumOperands() - 1);
5433 if (VT != MVT::f64 || HighBits.getValueType().getSizeInBits() != 32 ||
5434 !fnegFoldsIntoOp(N: HighBits.getNode()))
5435 return SDValue();
5436
5437 // f64 fneg only really needs to operate on the high half of of the
5438 // register, so try to force it to an f32 operation to help make use of
5439 // source modifiers.
5440 //
5441 //
5442 // fneg (f64 (bitcast (build_vector x, y))) ->
5443 // f64 (bitcast (build_vector (bitcast i32:x to f32),
5444 // (fneg (bitcast i32:y to f32)))
5445
5446 SDValue CastHi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: HighBits);
5447 SDValue NegHi = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: CastHi);
5448 SDValue CastBack =
5449 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: HighBits.getValueType(), Operand: NegHi);
5450
5451 SmallVector<SDValue, 8> Ops(BCSrc->ops());
5452 Ops.back() = CastBack;
5453 DCI.AddToWorklist(N: NegHi.getNode());
5454 SDValue Build =
5455 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: BCSrc.getValueType(), Ops);
5456 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Build);
5457
5458 if (!N0.hasOneUse())
5459 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Result));
5460 return Result;
5461 }
5462
5463 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 &&
5464 BCSrc.hasOneUse()) {
5465 // fneg (bitcast (f32 (select cond, i32:lhs, i32:rhs))) ->
5466 // select cond, (bitcast i32:lhs to f32), (bitcast i32:rhs to f32)
5467
5468 // TODO: Cast back result for multiple uses is beneficial in some cases.
5469
5470 SDValue LHS =
5471 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 1));
5472 SDValue RHS =
5473 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 2));
5474
5475 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: LHS);
5476 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHS);
5477
5478 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: BCSrc.getOperand(i: 0), N2: NegLHS,
5479 N3: NegRHS);
5480 }
5481
5482 return SDValue();
5483 }
5484 default:
5485 return SDValue();
5486 }
5487}
5488
5489SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
5490 DAGCombinerInfo &DCI) const {
5491 SelectionDAG &DAG = DCI.DAG;
5492 SDValue N0 = N->getOperand(Num: 0);
5493
5494 if (!N0.hasOneUse())
5495 return SDValue();
5496
5497 switch (N0.getOpcode()) {
5498 case ISD::FP16_TO_FP: {
5499 assert(!isTypeLegal(MVT::f16) && "should only see if f16 is illegal");
5500 SDLoc SL(N);
5501 SDValue Src = N0.getOperand(i: 0);
5502 EVT SrcVT = Src.getValueType();
5503
5504 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
5505 SDValue IntFAbs = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SrcVT, N1: Src,
5506 N2: DAG.getConstant(Val: 0x7fff, DL: SL, VT: SrcVT));
5507 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFAbs);
5508 }
5509 case ISD::FP_ROUND: {
5510 SDLoc SL(N);
5511 SDValue CvtSrc = N0.getOperand(i: 0);
5512
5513 // fabs (fp_round x) -> fp_round (fabs x)
5514 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc,
5515 Flags: N->getFlags());
5516 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: N->getValueType(ResNo: 0), N1: Abs,
5517 N2: N0.getOperand(i: 1), Flags: N0->getFlags());
5518 }
5519 default:
5520 return SDValue();
5521 }
5522}
5523
5524SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
5525 DAGCombinerInfo &DCI) const {
5526 const auto *CFP = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
5527 if (!CFP)
5528 return SDValue();
5529
5530 // XXX - Should this flush denormals?
5531 const APFloat &Val = CFP->getValueAPF();
5532 APFloat One = APFloat::getOne(Sem: Val.getSemantics());
5533 return DCI.DAG.getConstantFP(Val: One / Val, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
5534}
5535
5536bool AMDGPUTargetLowering::isInt64ImmLegal(SDNode *N, SelectionDAG &DAG) const {
5537 if (!Subtarget->isGCN())
5538 return false;
5539
5540 ConstantSDNode *SDConstant = dyn_cast<ConstantSDNode>(Val: N);
5541 ConstantFPSDNode *SDFPConstant = dyn_cast<ConstantFPSDNode>(Val: N);
5542 auto &ST = DAG.getSubtarget<GCNSubtarget>();
5543 const auto *TII = ST.getInstrInfo();
5544
5545 if (!ST.hasVMovB64Inst() || (!SDConstant && !SDFPConstant))
5546 return false;
5547
5548 if (ST.has64BitLiterals())
5549 return true;
5550
5551 if (SDConstant) {
5552 const APInt &APVal = SDConstant->getAPIntValue();
5553 return isUInt<32>(x: APVal.getZExtValue()) || TII->isInlineConstant(Imm: APVal);
5554 }
5555
5556 APInt Val = SDFPConstant->getValueAPF().bitcastToAPInt();
5557 return isUInt<32>(x: Val.getZExtValue()) || TII->isInlineConstant(Imm: Val);
5558}
5559
5560SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
5561 DAGCombinerInfo &DCI) const {
5562 SelectionDAG &DAG = DCI.DAG;
5563 SDLoc DL(N);
5564
5565 switch(N->getOpcode()) {
5566 default:
5567 break;
5568 case ISD::BITCAST: {
5569 EVT DestVT = N->getValueType(ResNo: 0);
5570
5571 // Push casts through vector builds. This helps avoid emitting a large
5572 // number of copies when materializing floating point vector constants.
5573 //
5574 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
5575 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
5576 if (DestVT.isVector()) {
5577 SDValue Src = N->getOperand(Num: 0);
5578 if (Src.getOpcode() == ISD::BUILD_VECTOR &&
5579 (DCI.getDAGCombineLevel() < AfterLegalizeDAG ||
5580 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: DestVT))) {
5581 EVT SrcVT = Src.getValueType();
5582 unsigned NElts = DestVT.getVectorNumElements();
5583
5584 if (SrcVT.getVectorNumElements() == NElts) {
5585 EVT DestEltVT = DestVT.getVectorElementType();
5586
5587 SmallVector<SDValue, 8> CastedElts;
5588 SDLoc SL(N);
5589 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
5590 SDValue Elt = Src.getOperand(i: I);
5591 CastedElts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: DestEltVT, Operand: Elt));
5592 }
5593
5594 return DAG.getBuildVector(VT: DestVT, DL: SL, Ops: CastedElts);
5595 }
5596 }
5597 }
5598
5599 if (DestVT.getSizeInBits() != 64 || !DestVT.isVector())
5600 break;
5601
5602 // Fold bitcasts of constants.
5603 //
5604 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
5605 // TODO: Generalize and move to DAGCombiner
5606 SDValue Src = N->getOperand(Num: 0);
5607 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Src)) {
5608 SDLoc SL(N);
5609 if (isInt64ImmLegal(N: C, DAG))
5610 break;
5611 uint64_t CVal = C->getZExtValue();
5612 SDValue BV = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5613 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5614 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5615 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: BV);
5616 }
5617
5618 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
5619 const APInt &Val = C->getValueAPF().bitcastToAPInt();
5620 SDLoc SL(N);
5621 if (isInt64ImmLegal(N: C, DAG))
5622 break;
5623 uint64_t CVal = Val.getZExtValue();
5624 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5625 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5626 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5627
5628 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: Vec);
5629 }
5630
5631 break;
5632 }
5633 case ISD::SHL:
5634 case ISD::SRA:
5635 case ISD::SRL: {
5636 // Range metadata can be invalidated when loads are converted to legal types
5637 // (e.g. v2i64 -> v4i32).
5638 // Try to convert vector shl/sra/srl before type legalization so that range
5639 // metadata can be utilized.
5640 if (!(N->getValueType(ResNo: 0).isVector() &&
5641 DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
5642 DCI.getDAGCombineLevel() < AfterLegalizeDAG)
5643 break;
5644 if (N->getOpcode() == ISD::SHL)
5645 return performShlCombine(N, DCI);
5646 if (N->getOpcode() == ISD::SRA)
5647 return performSraCombine(N, DCI);
5648 return performSrlCombine(N, DCI);
5649 }
5650 case ISD::TRUNCATE:
5651 return performTruncateCombine(N, DCI);
5652 case ISD::MUL:
5653 return performMulCombine(N, DCI);
5654 case AMDGPUISD::MUL_U24:
5655 case AMDGPUISD::MUL_I24: {
5656 if (SDValue Simplified = simplifyMul24(Node24: N, DCI))
5657 return Simplified;
5658 break;
5659 }
5660 case AMDGPUISD::MULHI_I24:
5661 case AMDGPUISD::MULHI_U24:
5662 return simplifyMul24(Node24: N, DCI);
5663 case ISD::SMUL_LOHI:
5664 case ISD::UMUL_LOHI:
5665 return performMulLoHiCombine(N, DCI);
5666 case ISD::MULHS:
5667 return performMulhsCombine(N, DCI);
5668 case ISD::MULHU:
5669 return performMulhuCombine(N, DCI);
5670 case ISD::SELECT:
5671 return performSelectCombine(N, DCI);
5672 case ISD::FNEG:
5673 return performFNegCombine(N, DCI);
5674 case ISD::FABS:
5675 return performFAbsCombine(N, DCI);
5676 case AMDGPUISD::BFE_I32:
5677 case AMDGPUISD::BFE_U32: {
5678 assert(!N->getValueType(0).isVector() &&
5679 "Vector handling of BFE not implemented");
5680 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
5681 if (!Width)
5682 break;
5683
5684 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
5685 if (WidthVal == 0)
5686 return DAG.getConstant(Val: 0, DL, VT: MVT::i32);
5687
5688 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
5689 if (!Offset)
5690 break;
5691
5692 SDValue BitsFrom = N->getOperand(Num: 0);
5693 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
5694
5695 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
5696
5697 if (OffsetVal == 0) {
5698 // This is already sign / zero extended, so try to fold away extra BFEs.
5699 unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
5700
5701 unsigned OpSignBits = DAG.ComputeNumSignBits(Op: BitsFrom);
5702 if (OpSignBits >= SignBits)
5703 return BitsFrom;
5704
5705 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WidthVal);
5706 if (Signed) {
5707 // This is a sign_extend_inreg. Replace it to take advantage of existing
5708 // DAG Combines. If not eliminated, we will match back to BFE during
5709 // selection.
5710
5711 // TODO: The sext_inreg of extended types ends, although we can could
5712 // handle them in a single BFE.
5713 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32, N1: BitsFrom,
5714 N2: DAG.getValueType(SmallVT));
5715 }
5716
5717 return DAG.getZeroExtendInReg(Op: BitsFrom, DL, VT: SmallVT);
5718 }
5719
5720 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(Val&: BitsFrom)) {
5721 if (Signed) {
5722 return constantFoldBFE<int32_t>(DAG,
5723 Src0: CVal->getSExtValue(),
5724 Offset: OffsetVal,
5725 Width: WidthVal,
5726 DL);
5727 }
5728
5729 return constantFoldBFE<uint32_t>(DAG,
5730 Src0: CVal->getZExtValue(),
5731 Offset: OffsetVal,
5732 Width: WidthVal,
5733 DL);
5734 }
5735
5736 if ((OffsetVal + WidthVal) >= 32 &&
5737 !(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
5738 SDValue ShiftVal = DAG.getConstant(Val: OffsetVal, DL, VT: MVT::i32);
5739 return DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32,
5740 N1: BitsFrom, N2: ShiftVal);
5741 }
5742
5743 if (BitsFrom.hasOneUse()) {
5744 APInt Demanded = APInt::getBitsSet(numBits: 32,
5745 loBit: OffsetVal,
5746 hiBit: OffsetVal + WidthVal);
5747
5748 KnownBits Known;
5749 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
5750 !DCI.isBeforeLegalizeOps());
5751 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5752 if (TLI.ShrinkDemandedConstant(Op: BitsFrom, DemandedBits: Demanded, TLO) ||
5753 TLI.SimplifyDemandedBits(Op: BitsFrom, DemandedBits: Demanded, Known, TLO)) {
5754 DCI.CommitTargetLoweringOpt(TLO);
5755 }
5756 }
5757
5758 break;
5759 }
5760 case ISD::LOAD:
5761 return performLoadCombine(N, DCI);
5762 case ISD::STORE:
5763 return performStoreCombine(N, DCI);
5764 case AMDGPUISD::RCP:
5765 case AMDGPUISD::RCP_IFLAG:
5766 return performRcpCombine(N, DCI);
5767 case ISD::AssertZext:
5768 case ISD::AssertSext:
5769 return performAssertSZExtCombine(N, DCI);
5770 case ISD::INTRINSIC_WO_CHAIN:
5771 return performIntrinsicWOChainCombine(N, DCI);
5772 case AMDGPUISD::FMAD_FTZ: {
5773 SDValue N0 = N->getOperand(Num: 0);
5774 SDValue N1 = N->getOperand(Num: 1);
5775 SDValue N2 = N->getOperand(Num: 2);
5776 EVT VT = N->getValueType(ResNo: 0);
5777
5778 // FMAD_FTZ is a FMAD + flush denormals to zero.
5779 // We flush the inputs, the intermediate step, and the output.
5780 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Val&: N0);
5781 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
5782 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
5783 if (N0CFP && N1CFP && N2CFP) {
5784 const auto FTZ = [](const APFloat &V) {
5785 if (V.isDenormal()) {
5786 APFloat Zero(V.getSemantics(), 0);
5787 return V.isNegative() ? -Zero : Zero;
5788 }
5789 return V;
5790 };
5791
5792 APFloat V0 = FTZ(N0CFP->getValueAPF());
5793 APFloat V1 = FTZ(N1CFP->getValueAPF());
5794 APFloat V2 = FTZ(N2CFP->getValueAPF());
5795 V0.multiply(RHS: V1, RM: APFloat::rmNearestTiesToEven);
5796 V0 = FTZ(V0);
5797 V0.add(RHS: V2, RM: APFloat::rmNearestTiesToEven);
5798 return DAG.getConstantFP(Val: FTZ(V0), DL, VT);
5799 }
5800 break;
5801 }
5802 }
5803 return SDValue();
5804}
5805
5806bool AMDGPUTargetLowering::SimplifyDemandedBitsForTargetNode(
5807 SDValue Op, const APInt &OriginalDemandedBits,
5808 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
5809 unsigned Depth) const {
5810 switch (Op.getOpcode()) {
5811 case ISD::INTRINSIC_WO_CHAIN: {
5812 switch (Op.getConstantOperandVal(i: 0)) {
5813 case Intrinsic::amdgcn_readfirstlane:
5814 case Intrinsic::amdgcn_readlane:
5815 case Intrinsic::amdgcn_wwm: {
5816 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5817 DemandedElts: OriginalDemandedElts, Known, TLO, Depth: Depth + 1))
5818 return true;
5819 break;
5820 }
5821 case Intrinsic::amdgcn_set_inactive:
5822 case Intrinsic::amdgcn_set_inactive_chain_arg: {
5823 // The result is operand 1 in active lanes and operand 2 in inactive
5824 // lanes, so the known bits are the intersection of both operands.
5825 KnownBits KnownValue, KnownInactive;
5826 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5827 DemandedElts: OriginalDemandedElts, Known&: KnownValue, TLO,
5828 Depth: Depth + 1))
5829 return true;
5830 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), DemandedBits: OriginalDemandedBits,
5831 DemandedElts: OriginalDemandedElts, Known&: KnownInactive, TLO,
5832 Depth: Depth + 1))
5833 return true;
5834 Known = KnownValue.intersectWith(RHS: KnownInactive);
5835 break;
5836 }
5837 default:
5838 break;
5839 }
5840 break;
5841 }
5842 default:
5843 break;
5844 }
5845
5846 return false;
5847}
5848
5849//===----------------------------------------------------------------------===//
5850// Helper functions
5851//===----------------------------------------------------------------------===//
5852
5853SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
5854 const TargetRegisterClass *RC,
5855 Register Reg, EVT VT,
5856 const SDLoc &SL,
5857 bool RawReg) const {
5858 MachineFunction &MF = DAG.getMachineFunction();
5859 MachineRegisterInfo &MRI = MF.getRegInfo();
5860 Register VReg;
5861
5862 if (!MRI.isLiveIn(Reg)) {
5863 VReg = MRI.createVirtualRegister(RegClass: RC);
5864 MRI.addLiveIn(Reg, vreg: VReg);
5865 } else {
5866 VReg = MRI.getLiveInVirtReg(PReg: Reg);
5867 }
5868
5869 if (RawReg)
5870 return DAG.getRegister(Reg: VReg, VT);
5871
5872 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: VReg, VT);
5873}
5874
5875// This may be called multiple times, and nothing prevents creating multiple
5876// objects at the same offset. See if we already defined this object.
5877static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
5878 int64_t Offset) {
5879 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
5880 if (MFI.getObjectOffset(ObjectIdx: I) == Offset) {
5881 assert(MFI.getObjectSize(I) == Size);
5882 return I;
5883 }
5884 }
5885
5886 return MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
5887}
5888
5889SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
5890 EVT VT,
5891 const SDLoc &SL,
5892 int64_t Offset) const {
5893 MachineFunction &MF = DAG.getMachineFunction();
5894 MachineFrameInfo &MFI = MF.getFrameInfo();
5895 int FI = getOrCreateFixedStackObject(MFI, Size: VT.getStoreSize(), Offset);
5896
5897 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
5898 SDValue Ptr = DAG.getFrameIndex(FI, VT: MVT::i32);
5899
5900 return DAG.getLoad(VT, dl: SL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: SrcPtrInfo, Alignment: Align(4),
5901 MMOFlags: MachineMemOperand::MODereferenceable |
5902 MachineMemOperand::MOInvariant);
5903}
5904
5905SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
5906 const SDLoc &SL,
5907 SDValue Chain,
5908 SDValue ArgVal,
5909 int64_t Offset) const {
5910 MachineFunction &MF = DAG.getMachineFunction();
5911 MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
5912 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5913
5914 SDValue Ptr = DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32);
5915 // Stores to the argument stack area are relative to the stack pointer.
5916 SDValue SP =
5917 DAG.getCopyFromReg(Chain, dl: SL, Reg: Info->getStackPtrOffsetReg(), VT: MVT::i32);
5918 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: SP, N2: Ptr);
5919 SDValue Store = DAG.getStore(Chain, dl: SL, Val: ArgVal, Ptr, PtrInfo: DstInfo, Alignment: Align(4),
5920 MMOFlags: MachineMemOperand::MODereferenceable);
5921 return Store;
5922}
5923
5924SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
5925 const TargetRegisterClass *RC,
5926 EVT VT, const SDLoc &SL,
5927 const ArgDescriptor &Arg) const {
5928 assert(Arg && "Attempting to load missing argument");
5929
5930 SDValue V = Arg.isRegister() ?
5931 CreateLiveInRegister(DAG, RC, Reg: Arg.getRegister(), VT, SL) :
5932 loadStackInputValue(DAG, VT, SL, Offset: Arg.getStackOffset());
5933
5934 if (!Arg.isMasked())
5935 return V;
5936
5937 unsigned Mask = Arg.getMask();
5938 unsigned Shift = llvm::countr_zero<unsigned>(Val: Mask);
5939 V = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: V,
5940 N2: DAG.getShiftAmountConstant(Val: Shift, VT, DL: SL));
5941 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT, N1: V,
5942 N2: DAG.getConstant(Val: Mask >> Shift, DL: SL, VT));
5943}
5944
5945uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5946 uint64_t ExplicitKernArgSize, const ImplicitParameter Param) const {
5947 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
5948 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr();
5949 uint64_t ArgOffset =
5950 alignTo(Size: ExplicitKernArgSize, A: Alignment) + ExplicitArgOffset;
5951 switch (Param) {
5952 case FIRST_IMPLICIT:
5953 return ArgOffset;
5954 case PRIVATE_BASE:
5955 return ArgOffset + AMDGPU::ImplicitArg::PRIVATE_BASE_OFFSET;
5956 case SHARED_BASE:
5957 return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET;
5958 case QUEUE_PTR:
5959 return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET;
5960 }
5961 llvm_unreachable("unexpected implicit parameter type");
5962}
5963
5964uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5965 const MachineFunction &MF, const ImplicitParameter Param) const {
5966 const AMDGPUMachineFunctionInfo *MFI =
5967 MF.getInfo<AMDGPUMachineFunctionInfo>();
5968 return getImplicitParameterOffset(ExplicitKernArgSize: MFI->getExplicitKernArgSize(), Param);
5969}
5970
5971SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
5972 SelectionDAG &DAG, int Enabled,
5973 int &RefinementSteps,
5974 bool &UseOneConstNR,
5975 bool Reciprocal) const {
5976 EVT VT = Operand.getValueType();
5977
5978 if (VT == MVT::f32) {
5979 RefinementSteps = 0;
5980 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(Operand), VT, Operand);
5981 }
5982
5983 // TODO: There is also f64 rsq instruction, but the documentation is less
5984 // clear on its precision.
5985
5986 return SDValue();
5987}
5988
5989SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
5990 SelectionDAG &DAG, int Enabled,
5991 int &RefinementSteps) const {
5992 EVT VT = Operand.getValueType();
5993
5994 if (VT == MVT::f32) {
5995 // Reciprocal, < 1 ulp error.
5996 //
5997 // This reciprocal approximation converges to < 0.5 ulp error with one
5998 // newton rhapson performed with two fused multiple adds (FMAs).
5999
6000 RefinementSteps = 0;
6001 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SDLoc(Operand), VT, Operand);
6002 }
6003
6004 // TODO: There is also f64 rcp instruction, but the documentation is less
6005 // clear on its precision.
6006
6007 return SDValue();
6008}
6009
6010static unsigned workitemIntrinsicDim(unsigned ID) {
6011 switch (ID) {
6012 case Intrinsic::amdgcn_workitem_id_x:
6013 return 0;
6014 case Intrinsic::amdgcn_workitem_id_y:
6015 return 1;
6016 case Intrinsic::amdgcn_workitem_id_z:
6017 return 2;
6018 default:
6019 llvm_unreachable("not a workitem intrinsic");
6020 }
6021}
6022
6023void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
6024 const SDValue Op, KnownBits &Known,
6025 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
6026
6027 Known.resetAll(); // Don't know anything.
6028
6029 unsigned Opc = Op.getOpcode();
6030
6031 switch (Opc) {
6032 default:
6033 break;
6034 case AMDGPUISD::CARRY:
6035 case AMDGPUISD::BORROW: {
6036 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31);
6037 break;
6038 }
6039
6040 case AMDGPUISD::BFE_I32:
6041 case AMDGPUISD::BFE_U32: {
6042 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6043 if (!CWidth)
6044 return;
6045
6046 uint32_t Width = CWidth->getZExtValue() & 0x1f;
6047
6048 if (Opc == AMDGPUISD::BFE_U32)
6049 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - Width);
6050
6051 break;
6052 }
6053 case AMDGPUISD::FP_TO_FP16: {
6054 unsigned BitWidth = Known.getBitWidth();
6055
6056 // High bits are zero.
6057 Known.Zero = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 16);
6058 break;
6059 }
6060 case AMDGPUISD::MUL_U24:
6061 case AMDGPUISD::MUL_I24: {
6062 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6063 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6064 unsigned BitWidth = Op.getScalarValueSizeInBits();
6065
6066 // Sign/Zero extend from 24 bits.
6067 if (Opc == AMDGPUISD::MUL_I24) {
6068 LHSKnown = LHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6069 RHSKnown = RHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6070 } else {
6071 LHSKnown = LHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6072 RHSKnown = RHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6073 }
6074
6075 // TODO: SelfMultiply can be poison, but not undef.
6076 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
6077 if (SelfMultiply)
6078 SelfMultiply &= DAG.isGuaranteedNotToBeUndefOrPoison(
6079 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
6080 Depth: Depth + 1);
6081
6082 Known = KnownBits::mul(LHS: LHSKnown, RHS: RHSKnown, NoUndefSelfMultiply: SelfMultiply);
6083 break;
6084 }
6085 case AMDGPUISD::PERM: {
6086 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6087 if (!CMask)
6088 return;
6089
6090 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6091 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6092 unsigned Sel = CMask->getZExtValue();
6093
6094 for (unsigned I = 0; I < 32; I += 8) {
6095 unsigned SelBits = Sel & 0xff;
6096 if (SelBits < 4) {
6097 SelBits *= 8;
6098 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6099 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6100 } else if (SelBits < 7) {
6101 SelBits = (SelBits & 3) * 8;
6102 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6103 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6104 } else if (SelBits == 0x0c) {
6105 Known.Zero |= 0xFFull << I;
6106 } else if (SelBits > 0x0c) {
6107 Known.One |= 0xFFull << I;
6108 }
6109 Sel >>= 8;
6110 }
6111 break;
6112 }
6113 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
6114 Known.Zero.setHighBits(24);
6115 break;
6116 }
6117 case AMDGPUISD::BUFFER_LOAD_USHORT: {
6118 Known.Zero.setHighBits(16);
6119 break;
6120 }
6121 case AMDGPUISD::LDS: {
6122 auto *GA = cast<GlobalAddressSDNode>(Val: Op.getOperand(i: 0).getNode());
6123 Align Alignment = GA->getGlobal()->getPointerAlignment(DL: DAG.getDataLayout());
6124
6125 Known.Zero.setHighBits(16);
6126 Known.Zero.setLowBits(Log2(A: Alignment));
6127 break;
6128 }
6129 case AMDGPUISD::SMIN3:
6130 case AMDGPUISD::SMAX3:
6131 case AMDGPUISD::SMED3:
6132 case AMDGPUISD::UMIN3:
6133 case AMDGPUISD::UMAX3:
6134 case AMDGPUISD::UMED3: {
6135 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6136 if (Known2.isUnknown())
6137 break;
6138
6139 KnownBits Known1 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6140 if (Known1.isUnknown())
6141 break;
6142
6143 KnownBits Known0 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6144 if (Known0.isUnknown())
6145 break;
6146
6147 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
6148 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
6149 Known.One = Known0.One & Known1.One & Known2.One;
6150 break;
6151 }
6152 case ISD::INTRINSIC_WO_CHAIN: {
6153 unsigned IID = Op.getConstantOperandVal(i: 0);
6154 switch (IID) {
6155 case Intrinsic::amdgcn_workitem_id_x:
6156 case Intrinsic::amdgcn_workitem_id_y:
6157 case Intrinsic::amdgcn_workitem_id_z: {
6158 unsigned MaxValue = Subtarget->getMaxWorkitemID(
6159 Kernel: DAG.getMachineFunction().getFunction(), Dimension: workitemIntrinsicDim(ID: IID));
6160 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
6161 break;
6162 }
6163 default:
6164 break;
6165 }
6166 }
6167 }
6168}
6169
6170unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
6171 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6172 unsigned Depth) const {
6173 switch (Op.getOpcode()) {
6174 case AMDGPUISD::BFE_I32: {
6175 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6176 if (!Width)
6177 return 1;
6178
6179 unsigned SignBits = 32 - (Width->getZExtValue() & 0x1f) + 1;
6180 if (!isNullConstant(V: Op.getOperand(i: 1)))
6181 return SignBits;
6182
6183 // TODO: Could probably figure something out with non-0 offsets.
6184 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6185 return std::max(a: SignBits, b: Op0SignBits);
6186 }
6187
6188 case AMDGPUISD::BFE_U32: {
6189 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6190 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
6191 }
6192
6193 case AMDGPUISD::CARRY:
6194 case AMDGPUISD::BORROW:
6195 return 31;
6196 case AMDGPUISD::BUFFER_LOAD_BYTE:
6197 return 25;
6198 case AMDGPUISD::BUFFER_LOAD_SHORT:
6199 return 17;
6200 case AMDGPUISD::BUFFER_LOAD_UBYTE:
6201 return 24;
6202 case AMDGPUISD::BUFFER_LOAD_USHORT:
6203 return 16;
6204 case AMDGPUISD::FP_TO_FP16:
6205 return 16;
6206 case AMDGPUISD::SMIN3:
6207 case AMDGPUISD::SMAX3:
6208 case AMDGPUISD::SMED3:
6209 case AMDGPUISD::UMIN3:
6210 case AMDGPUISD::UMAX3:
6211 case AMDGPUISD::UMED3: {
6212 unsigned Tmp2 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6213 if (Tmp2 == 1)
6214 return 1; // Early out.
6215
6216 unsigned Tmp1 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6217 if (Tmp1 == 1)
6218 return 1; // Early out.
6219
6220 unsigned Tmp0 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6221 if (Tmp0 == 1)
6222 return 1; // Early out.
6223
6224 return std::min(l: {Tmp0, Tmp1, Tmp2});
6225 }
6226 default:
6227 return 1;
6228 }
6229}
6230
6231unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
6232 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
6233 const MachineRegisterInfo &MRI, unsigned Depth) const {
6234 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
6235 if (!MI)
6236 return 1;
6237
6238 // TODO: Check range metadata on MMO.
6239 switch (MI->getOpcode()) {
6240 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
6241 return 25;
6242 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
6243 return 17;
6244 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
6245 return 24;
6246 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
6247 return 16;
6248 case AMDGPU::G_AMDGPU_SMED3:
6249 case AMDGPU::G_AMDGPU_UMED3: {
6250 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
6251 unsigned Tmp2 = Analysis.computeNumSignBits(R: Src2, DemandedElts, Depth: Depth + 1);
6252 if (Tmp2 == 1)
6253 return 1;
6254 unsigned Tmp1 = Analysis.computeNumSignBits(R: Src1, DemandedElts, Depth: Depth + 1);
6255 if (Tmp1 == 1)
6256 return 1;
6257 unsigned Tmp0 = Analysis.computeNumSignBits(R: Src0, DemandedElts, Depth: Depth + 1);
6258 if (Tmp0 == 1)
6259 return 1;
6260 return std::min(l: {Tmp0, Tmp1, Tmp2});
6261 }
6262 default:
6263 return 1;
6264 }
6265}
6266
6267bool AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode(
6268 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6269 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
6270 unsigned Opcode = Op.getOpcode();
6271 switch (Opcode) {
6272 case AMDGPUISD::BFE_I32:
6273 case AMDGPUISD::BFE_U32:
6274 return false;
6275 }
6276 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
6277 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
6278}
6279
6280bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
6281 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
6282 unsigned Depth) const {
6283 unsigned Opcode = Op.getOpcode();
6284 switch (Opcode) {
6285 case AMDGPUISD::FMIN_LEGACY:
6286 case AMDGPUISD::FMAX_LEGACY: {
6287 if (SNaN)
6288 return true;
6289
6290 // TODO: Can check no nans on one of the operands for each one, but which
6291 // one?
6292 return false;
6293 }
6294 case AMDGPUISD::FMUL_LEGACY:
6295 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
6296 if (SNaN)
6297 return true;
6298 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6299 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6300 }
6301 case AMDGPUISD::FMED3:
6302 case AMDGPUISD::FMIN3:
6303 case AMDGPUISD::FMAX3:
6304 case AMDGPUISD::FMINIMUM3:
6305 case AMDGPUISD::FMAXIMUM3:
6306 case AMDGPUISD::FMAD_FTZ: {
6307 if (SNaN)
6308 return true;
6309 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6310 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6311 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6312 }
6313 case AMDGPUISD::CVT_F32_UBYTE0:
6314 case AMDGPUISD::CVT_F32_UBYTE1:
6315 case AMDGPUISD::CVT_F32_UBYTE2:
6316 case AMDGPUISD::CVT_F32_UBYTE3:
6317 return true;
6318
6319 case AMDGPUISD::RCP:
6320 case AMDGPUISD::RSQ:
6321 case AMDGPUISD::RCP_LEGACY:
6322 case AMDGPUISD::RSQ_CLAMP: {
6323 if (SNaN)
6324 return true;
6325
6326 // TODO: Need is known positive check.
6327 return false;
6328 }
6329 case ISD::FLDEXP:
6330 case AMDGPUISD::FRACT: {
6331 if (SNaN)
6332 return true;
6333 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6334 }
6335 case AMDGPUISD::DIV_SCALE:
6336 case AMDGPUISD::DIV_FMAS:
6337 case AMDGPUISD::DIV_FIXUP:
6338 // TODO: Refine on operands.
6339 return SNaN;
6340 case AMDGPUISD::SIN_HW:
6341 case AMDGPUISD::COS_HW: {
6342 // TODO: Need check for infinity
6343 return SNaN;
6344 }
6345 case ISD::INTRINSIC_WO_CHAIN: {
6346 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
6347 // TODO: Handle more intrinsics
6348 switch (IntrinsicID) {
6349 case Intrinsic::amdgcn_cubeid:
6350 case Intrinsic::amdgcn_cvt_off_f32_i4:
6351 return true;
6352
6353 case Intrinsic::amdgcn_frexp_mant: {
6354 if (SNaN)
6355 return true;
6356 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6357 }
6358 case Intrinsic::amdgcn_cvt_pkrtz: {
6359 if (SNaN)
6360 return true;
6361 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6362 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6363 }
6364 case Intrinsic::amdgcn_rcp:
6365 case Intrinsic::amdgcn_rsq:
6366 case Intrinsic::amdgcn_rcp_legacy:
6367 case Intrinsic::amdgcn_rsq_legacy:
6368 case Intrinsic::amdgcn_rsq_clamp:
6369 case Intrinsic::amdgcn_tanh: {
6370 if (SNaN)
6371 return true;
6372
6373 // TODO: Need is known positive check.
6374 return false;
6375 }
6376 case Intrinsic::amdgcn_trig_preop:
6377 case Intrinsic::amdgcn_fdot2:
6378 // TODO: Refine on operand
6379 return SNaN;
6380 case Intrinsic::amdgcn_fma_legacy:
6381 if (SNaN)
6382 return true;
6383 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6384 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1) &&
6385 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 3), SNaN, Depth: Depth + 1);
6386 default:
6387 return false;
6388 }
6389 }
6390 default:
6391 return false;
6392 }
6393}
6394
6395bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
6396 Register N0, Register N1) const {
6397 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
6398}
6399