1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This is the parent TargetLowering class for hardware code gen
11/// targets.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUISelLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUMachineFunctionInfo.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUSelectionDAGInfo.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/Analysis.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/Support/AMDGPUAddrSpace.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/KnownBits.h"
30#include "llvm/Target/TargetMachine.h"
31
32using namespace llvm;
33
34#define GET_CALLING_CONV_IMPL
35#include "AMDGPUGenCallingConv.inc"
36
37static cl::opt<bool> AMDGPUBypassSlowDiv(
38 "amdgpu-bypass-slow-div",
39 cl::desc("Skip 64-bit divide for dynamic 32-bit values"),
40 cl::init(Val: true));
41
42// Find a larger type to do a load / store of a vector with.
43EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
44 unsigned StoreSize = VT.getStoreSizeInBits();
45 if (StoreSize <= 32)
46 return EVT::getIntegerVT(Context&: Ctx, BitWidth: StoreSize);
47
48 if (StoreSize % 32 == 0)
49 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i32, NumElements: StoreSize / 32);
50
51 return VT;
52}
53
54unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
55 return DAG.computeKnownBits(Op).countMaxActiveBits();
56}
57
58unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
59 // In order for this to be a signed 24-bit value, bit 23, must
60 // be a sign bit.
61 return DAG.ComputeMaxSignificantBits(Op);
62}
63
64AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
65 const TargetSubtargetInfo &STI,
66 const AMDGPUSubtarget &AMDGPUSTI)
67 : TargetLowering(TM, STI), Subtarget(&AMDGPUSTI) {
68 // Always lower memset, memcpy, and memmove intrinsics to load/store
69 // instructions, rather then generating calls to memset, mempcy or memmove.
70 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U;
71 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U;
72 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U;
73
74 // Enable ganging up loads and stores in the memcpy DAG lowering.
75 MaxGluedStoresPerMemcpy = 16;
76
77 // Lower floating point store/load to integer store/load to reduce the number
78 // of patterns in tablegen.
79 setOperationAction(Op: ISD::LOAD, VT: MVT::f32, Action: Promote);
80 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
81
82 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Promote);
83 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
84
85 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f32, Action: Promote);
86 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
87
88 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f32, Action: Promote);
89 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
90
91 setOperationAction(Op: ISD::LOAD, VT: MVT::v5f32, Action: Promote);
92 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
93
94 setOperationAction(Op: ISD::LOAD, VT: MVT::v6f32, Action: Promote);
95 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
96
97 setOperationAction(Op: ISD::LOAD, VT: MVT::v7f32, Action: Promote);
98 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
99
100 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f32, Action: Promote);
101 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
102
103 setOperationAction(Op: ISD::LOAD, VT: MVT::v9f32, Action: Promote);
104 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
105
106 setOperationAction(Op: ISD::LOAD, VT: MVT::v10f32, Action: Promote);
107 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
108
109 setOperationAction(Op: ISD::LOAD, VT: MVT::v11f32, Action: Promote);
110 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
111
112 setOperationAction(Op: ISD::LOAD, VT: MVT::v12f32, Action: Promote);
113 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
114
115 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f32, Action: Promote);
116 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
117
118 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f32, Action: Promote);
119 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
120
121 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Promote);
122 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i64, DestVT: MVT::v2i32);
123
124 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
125 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
126
127 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Promote);
128 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f64, DestVT: MVT::v2i32);
129
130 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Promote);
131 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
132
133 setOperationAction(Op: ISD::LOAD, VT: MVT::v3i64, Action: Promote);
134 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
135
136 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i64, Action: Promote);
137 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
138
139 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f64, Action: Promote);
140 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
141
142 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f64, Action: Promote);
143 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
144
145 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i64, Action: Promote);
146 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
147
148 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f64, Action: Promote);
149 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
150
151 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i64, Action: Promote);
152 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
153
154 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f64, Action: Promote);
155 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
156
157 setOperationAction(Op: ISD::LOAD, VT: MVT::i128, Action: Promote);
158 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i128, DestVT: MVT::v4i32);
159
160 // TODO: Would be better to consume as directly legal
161 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f32, Action: Promote);
162 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
163
164 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f64, Action: Promote);
165 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f64, DestVT: MVT::i64);
166
167 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f16, Action: Promote);
168 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
169
170 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::bf16, Action: Promote);
171 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
172
173 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::v2f32, Action: Promote);
174 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::v2f32, DestVT: MVT::i64);
175
176 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f32, Action: Promote);
177 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
178
179 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f64, Action: Promote);
180 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f64, DestVT: MVT::i64);
181
182 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f16, Action: Promote);
183 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
184
185 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::bf16, Action: Promote);
186 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
187
188 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::v2f32, Action: Promote);
189 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::v2f32, DestVT: MVT::i64);
190
191 // There are no 64-bit extloads. These should be done as a 32-bit extload and
192 // an extension to 64-bit.
193 for (MVT VT : MVT::integer_valuetypes())
194 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64, MemVT: VT,
195 Action: Expand);
196
197 for (MVT VT : MVT::integer_valuetypes()) {
198 if (VT == MVT::i64)
199 continue;
200
201 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) {
202 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i1, Action: Promote);
203 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i8, Action: Legal);
204 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i16, Action: Legal);
205 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i32, Action: Expand);
206 }
207 }
208
209 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
210 for (auto MemVT :
211 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
212 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT,
213 Action: Expand);
214
215 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
216 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
218 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
220 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
221 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
222 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
223 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
224 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
225 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
226 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
227 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
228 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
229
230 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
231 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
232 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
233 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
234 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
235 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
236
237 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
238 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
239 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
240 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
241 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
242 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
243 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
244 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
245 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
246 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
247 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
248 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
249
250 setOperationAction(Op: ISD::STORE, VT: MVT::f32, Action: Promote);
251 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
252
253 setOperationAction(Op: ISD::STORE, VT: MVT::v2f32, Action: Promote);
254 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
255
256 setOperationAction(Op: ISD::STORE, VT: MVT::v3f32, Action: Promote);
257 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
258
259 setOperationAction(Op: ISD::STORE, VT: MVT::v4f32, Action: Promote);
260 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
261
262 setOperationAction(Op: ISD::STORE, VT: MVT::v5f32, Action: Promote);
263 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
264
265 setOperationAction(Op: ISD::STORE, VT: MVT::v6f32, Action: Promote);
266 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
267
268 setOperationAction(Op: ISD::STORE, VT: MVT::v7f32, Action: Promote);
269 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
270
271 setOperationAction(Op: ISD::STORE, VT: MVT::v8f32, Action: Promote);
272 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
273
274 setOperationAction(Op: ISD::STORE, VT: MVT::v9f32, Action: Promote);
275 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
276
277 setOperationAction(Op: ISD::STORE, VT: MVT::v10f32, Action: Promote);
278 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
279
280 setOperationAction(Op: ISD::STORE, VT: MVT::v11f32, Action: Promote);
281 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
282
283 setOperationAction(Op: ISD::STORE, VT: MVT::v12f32, Action: Promote);
284 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
285
286 setOperationAction(Op: ISD::STORE, VT: MVT::v16f32, Action: Promote);
287 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
288
289 setOperationAction(Op: ISD::STORE, VT: MVT::v32f32, Action: Promote);
290 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
291
292 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Promote);
293 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i64, DestVT: MVT::v2i32);
294
295 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
296 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
297
298 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Promote);
299 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f64, DestVT: MVT::v2i32);
300
301 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Promote);
302 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
303
304 setOperationAction(Op: ISD::STORE, VT: MVT::v3i64, Action: Promote);
305 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
306
307 setOperationAction(Op: ISD::STORE, VT: MVT::v3f64, Action: Promote);
308 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
309
310 setOperationAction(Op: ISD::STORE, VT: MVT::v4i64, Action: Promote);
311 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
312
313 setOperationAction(Op: ISD::STORE, VT: MVT::v4f64, Action: Promote);
314 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
315
316 setOperationAction(Op: ISD::STORE, VT: MVT::v8i64, Action: Promote);
317 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
318
319 setOperationAction(Op: ISD::STORE, VT: MVT::v8f64, Action: Promote);
320 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
321
322 setOperationAction(Op: ISD::STORE, VT: MVT::v16i64, Action: Promote);
323 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
324
325 setOperationAction(Op: ISD::STORE, VT: MVT::v16f64, Action: Promote);
326 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
327
328 setOperationAction(Op: ISD::STORE, VT: MVT::i128, Action: Promote);
329 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i128, DestVT: MVT::v4i32);
330
331 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i1, Action: Expand);
332 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i8, Action: Expand);
333 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
334 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i32, Action: Expand);
335
336 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i1, Action: Expand);
337 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i8, Action: Expand);
338 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i16, Action: Expand);
339 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i32, Action: Expand);
340
341 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
342 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
343 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
344 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
345 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
346 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
347 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
348 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
349 setTruncStoreAction(ValVT: MVT::v6f32, MemVT: MVT::v6f16, Action: Expand);
350 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
351 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
352 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
353 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
354 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
355 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
356
357 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
358 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
359 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
360
361 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
362 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
363 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
364
365 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i8, Action: Expand);
366
367 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
368 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
369 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i8, Action: Expand);
370 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i1, Action: Expand);
371 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
372 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
373 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
374
375 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i32, Action: Expand);
376 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i16, Action: Expand);
377 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
378 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
379 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
380
381 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i1, Action: Expand);
382 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i8, Action: Expand);
383 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i16, Action: Expand);
384
385 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i1, Action: Expand);
386 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i8, Action: Expand);
387 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i16, Action: Expand);
388
389 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i1, Action: Expand);
390 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i8, Action: Expand);
391 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i16, Action: Expand);
392
393 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
394 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
395 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
396
397 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
398 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
399 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
400 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i16, Action: Expand);
401 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
402 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
403 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i1, Action: Expand);
404
405 setOperationAction(Ops: ISD::Constant, VTs: {MVT::i32, MVT::i64}, Action: Legal);
406 setOperationAction(Ops: ISD::ConstantFP, VTs: {MVT::f32, MVT::f64}, Action: Legal);
407
408 setOperationAction(Ops: {ISD::BR_JT, ISD::BRIND}, VT: MVT::Other, Action: Expand);
409
410 // For R600, this is totally unsupported, just custom lower to produce an
411 // error.
412 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
413
414 // Library functions. These default to Expand, but we have instructions
415 // for them.
416 setOperationAction(
417 Ops: {ISD::FCEIL, ISD::FABS, ISD::FFLOOR, ISD::FROUNDEVEN, ISD::FTRUNC},
418 VTs: {MVT::f16, MVT::f32}, Action: Legal);
419 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM}, VT: MVT::f32, Action: Legal);
420
421 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Custom);
422 setOperationAction(Ops: ISD::FROUND, VTs: {MVT::f32, MVT::f64}, Action: Custom);
423 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND},
424 VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
425
426 setOperationAction(
427 Ops: {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10, ISD::FPOW},
428 VT: MVT::f32, Action: Custom);
429 setOperationAction(Ops: {ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, VT: MVT::f64, Action: Custom);
430
431 setOperationAction(Ops: ISD::FNEARBYINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
432
433 setOperationAction(Ops: ISD::FRINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
434
435 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VTs: {MVT::f16, MVT::f32, MVT::f64},
436 Action: Expand);
437
438 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
439 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
440 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Custom);
441
442 setOperationAction(Ops: {ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, VT: MVT::f16,
443 Action: Custom);
444
445 setOperationAction(Ops: ISD::FCANONICALIZE, VTs: {MVT::f32, MVT::f64}, Action: Legal);
446
447 // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
448 // scalarization code. Can be removed when IS_FPCLASS expand isn't called by
449 // default unless marked custom/legal.
450 setOperationAction(Ops: ISD::IS_FPCLASS,
451 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
452 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
453 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64,
454 MVT::v16f64},
455 Action: Custom);
456
457 // Expand to fneg + fadd.
458 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
459
460 setOperationAction(Ops: ISD::CONCAT_VECTORS,
461 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
462 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
463 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
464 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
465 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
466 Action: Custom);
467
468 setOperationAction(
469 Ops: ISD::EXTRACT_SUBVECTOR,
470 VTs: {MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32,
471 MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32,
472 MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32,
473 MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32,
474 MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
475 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
476 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64},
477 Action: Custom);
478
479 setOperationAction(Ops: {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP}, VT: MVT::f64,
480 Action: Expand);
481 setOperationAction(Ops: ISD::FP_TO_FP16, VTs: {MVT::f64, MVT::f32}, Action: Custom);
482
483 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
484 for (MVT VT : ScalarIntVTs) {
485 // These should use [SU]DIVREM, so set them to expand
486 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT,
487 Action: Expand);
488
489 // GPU does not have divrem function for signed or unsigned.
490 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT, Action: Custom);
491
492 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
493 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
494
495 setOperationAction(Ops: {ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Action: Expand);
496
497 setOperationAction(Ops: {ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT,
498 Action: Expand);
499 }
500
501 // The hardware supports 32-bit FSHR, but not FSHL.
502 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
503
504 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VTs: {MVT::i32, MVT::i64}, Action: Expand);
505
506 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT: MVT::i16, Action: Expand);
507
508 setOperationAction(Ops: {ISD::MUL, ISD::MULHU, ISD::MULHS}, VT: MVT::i64, Action: Expand);
509 setOperationAction(Ops: {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT,
510 ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
511 ISD::FP_TO_UINT_SAT},
512 VT: MVT::i64, Action: Custom);
513 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: Expand);
514
515 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i32,
516 Action: Legal);
517
518 setOperationAction(
519 Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON},
520 VT: MVT::i64, Action: Custom);
521
522 for (auto VT : {MVT::i8, MVT::i16})
523 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT, Action: Custom);
524
525 static const MVT::SimpleValueType VectorIntTypes[] = {
526 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
527 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
528
529 for (MVT VT : VectorIntTypes) {
530 // Expand the following operations for the current type by default.
531 // clang-format off
532 setOperationAction(Ops: {ISD::ADD, ISD::AND,
533 ISD::FP_TO_SINT, ISD::FP_TO_UINT,
534 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
535 ISD::MUL, ISD::MULHU,
536 ISD::MULHS, ISD::OR,
537 ISD::SHL, ISD::SRA,
538 ISD::SRL, ISD::ROTL,
539 ISD::ROTR, ISD::SUB,
540 ISD::SINT_TO_FP, ISD::UINT_TO_FP,
541 ISD::SDIV, ISD::UDIV,
542 ISD::SREM, ISD::UREM,
543 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
544 ISD::SDIVREM, ISD::UDIVREM,
545 ISD::SELECT, ISD::VSELECT,
546 ISD::SELECT_CC, ISD::XOR,
547 ISD::BSWAP, ISD::CTPOP,
548 ISD::CTTZ, ISD::CTLZ,
549 ISD::VECTOR_SHUFFLE, ISD::SETCC,
550 ISD::ADDRSPACECAST},
551 VT, Action: Expand);
552 // clang-format on
553 }
554
555 static const MVT::SimpleValueType FloatVectorTypes[] = {
556 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
557 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
558
559 for (MVT VT : FloatVectorTypes) {
560 setOperationAction(
561 Ops: {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM,
562 ISD::FADD, ISD::FCEIL, ISD::FCOS,
563 ISD::FDIV, ISD::FEXP2, ISD::FEXP,
564 ISD::FEXP10, ISD::FLOG2, ISD::FREM,
565 ISD::FLOG, ISD::FLOG10, ISD::FPOW,
566 ISD::FFLOOR, ISD::FTRUNC, ISD::FMUL,
567 ISD::FMA, ISD::FRINT, ISD::FNEARBYINT,
568 ISD::FSQRT, ISD::FSIN, ISD::FSUB,
569 ISD::FNEG, ISD::VSELECT, ISD::SELECT_CC,
570 ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, ISD::SETCC,
571 ISD::FCANONICALIZE, ISD::FROUNDEVEN},
572 VT, Action: Expand);
573 }
574
575 // This causes using an unrolled select operation rather than expansion with
576 // bit operations. This is in general better, but the alternative using BFI
577 // instructions may be better if the select sources are SGPRs.
578 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f32, Action: Promote);
579 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
580
581 setOperationAction(Op: ISD::SELECT, VT: MVT::v3f32, Action: Promote);
582 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
583
584 setOperationAction(Op: ISD::SELECT, VT: MVT::v4f32, Action: Promote);
585 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
586
587 setOperationAction(Op: ISD::SELECT, VT: MVT::v5f32, Action: Promote);
588 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
589
590 setOperationAction(Op: ISD::SELECT, VT: MVT::v6f32, Action: Promote);
591 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
592
593 setOperationAction(Op: ISD::SELECT, VT: MVT::v7f32, Action: Promote);
594 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
595
596 setOperationAction(Op: ISD::SELECT, VT: MVT::v9f32, Action: Promote);
597 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
598
599 setOperationAction(Op: ISD::SELECT, VT: MVT::v10f32, Action: Promote);
600 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
601
602 setOperationAction(Op: ISD::SELECT, VT: MVT::v11f32, Action: Promote);
603 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
604
605 setOperationAction(Op: ISD::SELECT, VT: MVT::v12f32, Action: Promote);
606 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
607
608 setSchedulingPreference(Sched::RegPressure);
609 setJumpIsExpensive(true);
610
611 setMinCmpXchgSizeInBits(32);
612 setSupportsUnalignedAtomics(false);
613
614 PredictableSelectIsExpensive = false;
615
616 // We want to find all load dependencies for long chains of stores to enable
617 // merging into very wide vectors. The problem is with vectors with > 4
618 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
619 // vectors are a legal type, even though we have to split the loads
620 // usually. When we can more precisely specify load legality per address
621 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
622 // smarter so that they can figure out what to do in 2 iterations without all
623 // N > 4 stores on the same chain.
624 GatherAllAliasesMaxDepth = 16;
625
626 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
627 // about these during lowering.
628 MaxStoresPerMemcpy = 0xffffffff;
629 MaxStoresPerMemmove = 0xffffffff;
630 MaxStoresPerMemset = 0xffffffff;
631
632 // The expansion for 64-bit division is enormous.
633 if (AMDGPUBypassSlowDiv)
634 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
635
636 setTargetDAGCombine({ISD::BITCAST, ISD::SHL,
637 ISD::SRA, ISD::SRL,
638 ISD::TRUNCATE, ISD::MUL,
639 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
640 ISD::MULHU, ISD::MULHS,
641 ISD::SELECT, ISD::SELECT_CC,
642 ISD::STORE, ISD::FADD,
643 ISD::FSUB, ISD::FNEG,
644 ISD::FABS, ISD::AssertZext,
645 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN});
646
647 setMaxAtomicSizeInBitsSupported(64);
648 setMaxDivRemBitWidthSupported(64);
649 setMaxLargeFPConvertBitWidthSupported(64);
650}
651
652//===----------------------------------------------------------------------===//
653// Target Information
654//===----------------------------------------------------------------------===//
655
656LLVM_READNONE
657static bool fnegFoldsIntoOpcode(unsigned Opc) {
658 switch (Opc) {
659 case ISD::FADD:
660 case ISD::FSUB:
661 case ISD::FMUL:
662 case ISD::FMA:
663 case ISD::FMAD:
664 case ISD::FMINNUM:
665 case ISD::FMAXNUM:
666 case ISD::FMINNUM_IEEE:
667 case ISD::FMAXNUM_IEEE:
668 case ISD::FMINIMUM:
669 case ISD::FMAXIMUM:
670 case ISD::FMINIMUMNUM:
671 case ISD::FMAXIMUMNUM:
672 case ISD::SELECT:
673 case ISD::FSIN:
674 case ISD::FTRUNC:
675 case ISD::FRINT:
676 case ISD::FNEARBYINT:
677 case ISD::FROUNDEVEN:
678 case ISD::FCANONICALIZE:
679 case AMDGPUISD::RCP:
680 case AMDGPUISD::RCP_LEGACY:
681 case AMDGPUISD::RCP_IFLAG:
682 case AMDGPUISD::SIN_HW:
683 case AMDGPUISD::FMUL_LEGACY:
684 case AMDGPUISD::FMIN_LEGACY:
685 case AMDGPUISD::FMAX_LEGACY:
686 case AMDGPUISD::FMED3:
687 // TODO: handle llvm.amdgcn.fma.legacy
688 return true;
689 case ISD::BITCAST:
690 llvm_unreachable("bitcast is special cased");
691 default:
692 return false;
693 }
694}
695
696static bool fnegFoldsIntoOp(const SDNode *N) {
697 unsigned Opc = N->getOpcode();
698 if (Opc == ISD::BITCAST) {
699 // TODO: Is there a benefit to checking the conditions performFNegCombine
700 // does? We don't for the other cases.
701 SDValue BCSrc = N->getOperand(Num: 0);
702 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
703 return BCSrc.getNumOperands() == 2 &&
704 BCSrc.getOperand(i: 1).getValueSizeInBits() == 32;
705 }
706
707 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32;
708 }
709
710 return fnegFoldsIntoOpcode(Opc);
711}
712
713/// \p returns true if the operation will definitely need to use a 64-bit
714/// encoding, and thus will use a VOP3 encoding regardless of the source
715/// modifiers.
716LLVM_READONLY
717static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
718 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) ||
719 VT == MVT::f64;
720}
721
722/// Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the
723/// type for ISD::SELECT.
724LLVM_READONLY
725static bool selectSupportsSourceMods(const SDNode *N) {
726 // TODO: Only applies if select will be vector
727 return N->getValueType(ResNo: 0) == MVT::f32;
728}
729
730// Most FP instructions support source modifiers, but this could be refined
731// slightly.
732LLVM_READONLY
733static bool hasSourceMods(const SDNode *N) {
734 if (isa<MemSDNode>(Val: N))
735 return false;
736
737 switch (N->getOpcode()) {
738 case ISD::CopyToReg:
739 case ISD::FDIV:
740 case ISD::FREM:
741 case ISD::INLINEASM:
742 case ISD::INLINEASM_BR:
743 case AMDGPUISD::DIV_SCALE:
744 case ISD::INTRINSIC_W_CHAIN:
745
746 // TODO: Should really be looking at the users of the bitcast. These are
747 // problematic because bitcasts are used to legalize all stores to integer
748 // types.
749 case ISD::BITCAST:
750 return false;
751 case ISD::INTRINSIC_WO_CHAIN: {
752 switch (N->getConstantOperandVal(Num: 0)) {
753 case Intrinsic::amdgcn_interp_p1:
754 case Intrinsic::amdgcn_interp_p2:
755 case Intrinsic::amdgcn_interp_mov:
756 case Intrinsic::amdgcn_interp_p1_f16:
757 case Intrinsic::amdgcn_interp_p2_f16:
758 return false;
759 default:
760 return true;
761 }
762 }
763 case ISD::SELECT:
764 return selectSupportsSourceMods(N);
765 default:
766 return true;
767 }
768}
769
770bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N,
771 unsigned CostThreshold) {
772 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
773 // it is truly free to use a source modifier in all cases. If there are
774 // multiple users but for each one will necessitate using VOP3, there will be
775 // a code size increase. Try to avoid increasing code size unless we know it
776 // will save on the instruction count.
777 unsigned NumMayIncreaseSize = 0;
778 MVT VT = N->getValueType(ResNo: 0).getScalarType().getSimpleVT();
779
780 assert(!N->use_empty());
781
782 // XXX - Should this limit number of uses to check?
783 for (const SDNode *U : N->users()) {
784 if (!hasSourceMods(N: U))
785 return false;
786
787 if (!opMustUseVOP3Encoding(N: U, VT)) {
788 if (++NumMayIncreaseSize > CostThreshold)
789 return false;
790 }
791 }
792
793 return true;
794}
795
796EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
797 ISD::NodeType ExtendKind) const {
798 assert(!VT.isVector() && "only scalar expected");
799
800 // Round to the next multiple of 32-bits.
801 unsigned Size = VT.getSizeInBits();
802 if (Size <= 32)
803 return MVT::i32;
804 return EVT::getIntegerVT(Context, BitWidth: 32 * ((Size + 31) / 32));
805}
806
807unsigned AMDGPUTargetLowering::getVectorIdxWidth(const DataLayout &) const {
808 return 32;
809}
810
811bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
812 return true;
813}
814
815// The backend supports 32 and 64 bit floating point immediates.
816// FIXME: Why are we reporting vectors of FP immediates as legal?
817bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
818 bool ForCodeSize) const {
819 return isTypeLegal(VT: VT.getScalarType());
820}
821
822// We don't want to shrink f64 / f32 constants.
823bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
824 EVT ScalarVT = VT.getScalarType();
825 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
826}
827
828bool AMDGPUTargetLowering::shouldReduceLoadWidth(
829 SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
830 std::optional<unsigned> ByteOffset) const {
831 // TODO: This may be worth removing. Check regression tests for diffs.
832 if (!TargetLoweringBase::shouldReduceLoadWidth(Load: N, ExtTy, NewVT, ByteOffset))
833 return false;
834
835 unsigned NewSize = NewVT.getStoreSizeInBits();
836
837 // If we are reducing to a 32-bit load or a smaller multi-dword load,
838 // this is always better.
839 if (NewSize >= 32)
840 return true;
841
842 EVT OldVT = N->getValueType(ResNo: 0);
843 unsigned OldSize = OldVT.getStoreSizeInBits();
844
845 MemSDNode *MN = cast<MemSDNode>(Val: N);
846 unsigned AS = MN->getAddressSpace();
847 // Do not shrink an aligned scalar load to sub-dword.
848 // Scalar engine cannot do sub-dword loads.
849 // TODO: Update this for GFX12 which does have scalar sub-dword loads.
850 if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) &&
851 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
852 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
853 (isa<LoadSDNode>(Val: N) && AS == AMDGPUAS::GLOBAL_ADDRESS &&
854 MN->isInvariant())) &&
855 AMDGPU::isUniformMMO(MMO: MN->getMemOperand()))
856 return false;
857
858 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
859 // extloads, so doing one requires using a buffer_load. In cases where we
860 // still couldn't use a scalar load, using the wider load shouldn't really
861 // hurt anything.
862
863 // If the old size already had to be an extload, there's no harm in continuing
864 // to reduce the width.
865 return (OldSize < 32);
866}
867
868bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
869 const SelectionDAG &DAG,
870 const MachineMemOperand &MMO) const {
871
872 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
873
874 if (LoadTy.getScalarType() == MVT::i32)
875 return false;
876
877 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
878 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
879
880 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
881 return false;
882
883 unsigned Fast = 0;
884 return allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
885 VT: CastTy, MMO, Fast: &Fast) &&
886 Fast;
887}
888
889// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
890// profitable with the expansion for 64-bit since it's generally good to
891// speculate things.
892bool AMDGPUTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
893 return true;
894}
895
896bool AMDGPUTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
897 return true;
898}
899
900bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const {
901 switch (N->getOpcode()) {
902 case ISD::EntryToken:
903 case ISD::TokenFactor:
904 return true;
905 case ISD::INTRINSIC_WO_CHAIN: {
906 unsigned IntrID = N->getConstantOperandVal(Num: 0);
907 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
908 }
909 case ISD::INTRINSIC_W_CHAIN: {
910 unsigned IntrID = N->getConstantOperandVal(Num: 1);
911 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
912 }
913 case ISD::LOAD:
914 if (cast<LoadSDNode>(Val: N)->getMemOperand()->getAddrSpace() ==
915 AMDGPUAS::CONSTANT_ADDRESS_32BIT)
916 return true;
917 return false;
918 case AMDGPUISD::SETCC: // ballot-style instruction
919 return true;
920 }
921 return false;
922}
923
924SDValue AMDGPUTargetLowering::getNegatedExpression(
925 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
926 NegatibleCost &Cost, unsigned Depth) const {
927
928 switch (Op.getOpcode()) {
929 case ISD::FMA:
930 case ISD::FMAD: {
931 // Negating a fma is not free if it has users without source mods.
932 if (!allUsesHaveSourceMods(N: Op.getNode()))
933 return SDValue();
934 break;
935 }
936 case AMDGPUISD::RCP: {
937 SDValue Src = Op.getOperand(i: 0);
938 EVT VT = Op.getValueType();
939 SDLoc SL(Op);
940
941 SDValue NegSrc = getNegatedExpression(Op: Src, DAG, LegalOperations,
942 ForCodeSize, Cost, Depth: Depth + 1);
943 if (NegSrc)
944 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: NegSrc, Flags: Op->getFlags());
945 return SDValue();
946 }
947 default:
948 break;
949 }
950
951 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps: LegalOperations,
952 OptForSize: ForCodeSize, Cost, Depth);
953}
954
955//===---------------------------------------------------------------------===//
956// Target Properties
957//===---------------------------------------------------------------------===//
958
959bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
960 assert(VT.isFloatingPoint());
961
962 // Packed operations do not have a fabs modifier.
963 // Report this based on the end legalized type.
964 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
965}
966
967bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
968 assert(VT.isFloatingPoint());
969 // Report this based on the end legalized type.
970 VT = VT.getScalarType();
971 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
972}
973
974bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
975 unsigned NumElem,
976 unsigned AS) const {
977 return true;
978}
979
980bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
981 // There are few operations which truly have vector input operands. Any vector
982 // operation is going to involve operations on each component, and a
983 // build_vector will be a copy per element, so it always makes sense to use a
984 // build_vector input in place of the extracted element to avoid a copy into a
985 // super register.
986 //
987 // We should probably only do this if all users are extracts only, but this
988 // should be the common case.
989 return true;
990}
991
992bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
993 // Truncate is just accessing a subregister.
994
995 unsigned SrcSize = Source.getSizeInBits();
996 unsigned DestSize = Dest.getSizeInBits();
997
998 return DestSize < SrcSize && DestSize % 32 == 0 ;
999}
1000
1001bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
1002 // Truncate is just accessing a subregister.
1003
1004 unsigned SrcSize = Source->getScalarSizeInBits();
1005 unsigned DestSize = Dest->getScalarSizeInBits();
1006
1007 if (DestSize== 16 && Subtarget->has16BitInsts())
1008 return SrcSize >= 32;
1009
1010 return DestSize < SrcSize && DestSize % 32 == 0;
1011}
1012
1013bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
1014 unsigned SrcSize = Src->getScalarSizeInBits();
1015 unsigned DestSize = Dest->getScalarSizeInBits();
1016
1017 if (SrcSize == 16 && Subtarget->has16BitInsts())
1018 return DestSize >= 32;
1019
1020 return SrcSize == 32 && DestSize == 64;
1021}
1022
1023bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
1024 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
1025 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
1026 // this will enable reducing 64-bit operations the 32-bit, which is always
1027 // good.
1028
1029 if (Src == MVT::i16)
1030 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1031
1032 return Src == MVT::i32 && Dest == MVT::i64;
1033}
1034
1035bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
1036 EVT DestVT) const {
1037 switch (N->getOpcode()) {
1038 case ISD::ABS:
1039 case ISD::ADD:
1040 case ISD::SUB:
1041 case ISD::SHL:
1042 case ISD::SRL:
1043 case ISD::SRA:
1044 case ISD::AND:
1045 case ISD::OR:
1046 case ISD::XOR:
1047 case ISD::MUL:
1048 case ISD::SETCC:
1049 case ISD::SELECT:
1050 case ISD::SMIN:
1051 case ISD::SMAX:
1052 case ISD::UMIN:
1053 case ISD::UMAX:
1054 case ISD::USUBSAT:
1055 case ISD::UADDSAT:
1056 if (isTypeLegal(VT: MVT::i16) &&
1057 (!DestVT.isVector() ||
1058 !isOperationLegal(Op: ISD::ADD, VT: MVT::v2i16))) { // Check if VOP3P
1059 // Don't narrow back down to i16 if promoted to i32 already.
1060 if (!N->isDivergent() && DestVT.isInteger() &&
1061 DestVT.getScalarSizeInBits() > 1 &&
1062 DestVT.getScalarSizeInBits() <= 16 &&
1063 SrcVT.getScalarSizeInBits() > 16) {
1064 return false;
1065 }
1066 }
1067 return true;
1068 default:
1069 break;
1070 }
1071
1072 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
1073 // limited number of native 64-bit operations. Shrinking an operation to fit
1074 // in a single 32-bit register should always be helpful. As currently used,
1075 // this is much less general than the name suggests, and is only used in
1076 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
1077 // not profitable, and may actually be harmful.
1078 if (isa<LoadSDNode>(Val: N))
1079 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
1080
1081 return true;
1082}
1083
1084bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
1085 const SDNode* N, CombineLevel Level) const {
1086 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
1087 N->getOpcode() == ISD::SRL) &&
1088 "Expected shift op");
1089
1090 SDValue ShiftLHS = N->getOperand(Num: 0);
1091 if (!ShiftLHS->hasOneUse())
1092 return false;
1093
1094 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
1095 !ShiftLHS.getOperand(i: 0)->hasOneUse())
1096 return false;
1097
1098 // Always commute pre-type legalization and right shifts.
1099 // We're looking for shl(or(x,y),z) patterns.
1100 if (Level < CombineLevel::AfterLegalizeTypes ||
1101 N->getOpcode() != ISD::SHL || N->getOperand(Num: 0).getOpcode() != ISD::OR)
1102 return true;
1103
1104 // If only user is a i32 right-shift, then don't destroy a BFE pattern.
1105 if (N->getValueType(ResNo: 0) == MVT::i32 && N->hasOneUse() &&
1106 (N->user_begin()->getOpcode() == ISD::SRA ||
1107 N->user_begin()->getOpcode() == ISD::SRL))
1108 return false;
1109
1110 // Don't destroy or(shl(load_zext(),c), load_zext()) patterns.
1111 auto IsShiftAndLoad = [](SDValue LHS, SDValue RHS) {
1112 if (LHS.getOpcode() != ISD::SHL)
1113 return false;
1114 auto *RHSLd = dyn_cast<LoadSDNode>(Val&: RHS);
1115 auto *LHS0 = dyn_cast<LoadSDNode>(Val: LHS.getOperand(i: 0));
1116 auto *LHS1 = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
1117 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD &&
1118 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1119 RHSLd->getExtensionType() == ISD::ZEXTLOAD;
1120 };
1121 SDValue LHS = N->getOperand(Num: 0).getOperand(i: 0);
1122 SDValue RHS = N->getOperand(Num: 0).getOperand(i: 1);
1123 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS));
1124}
1125
1126//===---------------------------------------------------------------------===//
1127// TargetLowering Callbacks
1128//===---------------------------------------------------------------------===//
1129
1130CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
1131 bool IsVarArg) {
1132 switch (CC) {
1133 case CallingConv::AMDGPU_VS:
1134 case CallingConv::AMDGPU_GS:
1135 case CallingConv::AMDGPU_PS:
1136 case CallingConv::AMDGPU_CS:
1137 case CallingConv::AMDGPU_HS:
1138 case CallingConv::AMDGPU_ES:
1139 case CallingConv::AMDGPU_LS:
1140 return CC_AMDGPU;
1141 case CallingConv::AMDGPU_CS_Chain:
1142 case CallingConv::AMDGPU_CS_ChainPreserve:
1143 return CC_AMDGPU_CS_CHAIN;
1144 case CallingConv::C:
1145 case CallingConv::Fast:
1146 case CallingConv::Cold:
1147 return CC_AMDGPU_Func;
1148 case CallingConv::AMDGPU_Gfx:
1149 case CallingConv::AMDGPU_Gfx_WholeWave:
1150 return CC_SI_Gfx;
1151 case CallingConv::AMDGPU_KERNEL:
1152 case CallingConv::SPIR_KERNEL:
1153 default:
1154 reportFatalUsageError(reason: "unsupported calling convention for call");
1155 }
1156}
1157
1158CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
1159 bool IsVarArg) {
1160 switch (CC) {
1161 case CallingConv::AMDGPU_KERNEL:
1162 case CallingConv::SPIR_KERNEL:
1163 llvm_unreachable("kernels should not be handled here");
1164 case CallingConv::AMDGPU_VS:
1165 case CallingConv::AMDGPU_GS:
1166 case CallingConv::AMDGPU_PS:
1167 case CallingConv::AMDGPU_CS:
1168 case CallingConv::AMDGPU_CS_Chain:
1169 case CallingConv::AMDGPU_CS_ChainPreserve:
1170 case CallingConv::AMDGPU_HS:
1171 case CallingConv::AMDGPU_ES:
1172 case CallingConv::AMDGPU_LS:
1173 return RetCC_SI_Shader;
1174 case CallingConv::AMDGPU_Gfx:
1175 case CallingConv::AMDGPU_Gfx_WholeWave:
1176 return RetCC_SI_Gfx;
1177 case CallingConv::C:
1178 case CallingConv::Fast:
1179 case CallingConv::Cold:
1180 return RetCC_AMDGPU_Func;
1181 default:
1182 reportFatalUsageError(reason: "unsupported calling convention");
1183 }
1184}
1185
1186/// The SelectionDAGBuilder will automatically promote function arguments
1187/// with illegal types. However, this does not work for the AMDGPU targets
1188/// since the function arguments are stored in memory as these illegal types.
1189/// In order to handle this properly we need to get the original types sizes
1190/// from the LLVM IR Function and fixup the ISD:InputArg values before
1191/// passing them to AnalyzeFormalArguments()
1192
1193/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
1194/// input values across multiple registers. Each item in the Ins array
1195/// represents a single value that will be stored in registers. Ins[x].VT is
1196/// the value type of the value that will be stored in the register, so
1197/// whatever SDNode we lower the argument to needs to be this type.
1198///
1199/// In order to correctly lower the arguments we need to know the size of each
1200/// argument. Since Ins[x].VT gives us the size of the register that will
1201/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
1202/// for the original function argument so that we can deduce the correct memory
1203/// type to use for Ins[x]. In most cases the correct memory type will be
1204/// Ins[x].ArgVT. However, this will not always be the case. If, for example,
1205/// we have a kernel argument of type v8i8, this argument will be split into
1206/// 8 parts and each part will be represented by its own item in the Ins array.
1207/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
1208/// the argument before it was split. From this, we deduce that the memory type
1209/// for each individual part is i8. We pass the memory type as LocVT to the
1210/// calling convention analysis function and the register type (Ins[x].VT) as
1211/// the ValVT.
1212void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
1213 CCState &State,
1214 const SmallVectorImpl<ISD::InputArg> &Ins) const {
1215 const MachineFunction &MF = State.getMachineFunction();
1216 const Function &Fn = MF.getFunction();
1217 LLVMContext &Ctx = Fn.getContext();
1218 const unsigned ExplicitOffset = Subtarget->getExplicitKernelArgOffset();
1219 CallingConv::ID CC = Fn.getCallingConv();
1220
1221 Align MaxAlign = Align(1);
1222 uint64_t ExplicitArgOffset = 0;
1223 const DataLayout &DL = Fn.getDataLayout();
1224
1225 unsigned InIndex = 0;
1226
1227 for (const Argument &Arg : Fn.args()) {
1228 const bool IsByRef = Arg.hasByRefAttr();
1229 Type *BaseArgTy = Arg.getType();
1230 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1231 Align Alignment = DL.getValueOrABITypeAlignment(
1232 Alignment: IsByRef ? Arg.getParamAlign() : std::nullopt, Ty: MemArgTy);
1233 MaxAlign = std::max(a: Alignment, b: MaxAlign);
1234 uint64_t AllocSize = DL.getTypeAllocSize(Ty: MemArgTy);
1235
1236 uint64_t ArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + ExplicitOffset;
1237 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + AllocSize;
1238
1239 // We're basically throwing away everything passed into us and starting over
1240 // to get accurate in-memory offsets. The "PartOffset" is completely useless
1241 // to us as computed in Ins.
1242 //
1243 // We also need to figure out what type legalization is trying to do to get
1244 // the correct memory offsets.
1245
1246 SmallVector<EVT, 16> ValueVTs;
1247 SmallVector<uint64_t, 16> Offsets;
1248 ComputeValueVTs(TLI: *this, DL, Ty: BaseArgTy, ValueVTs, /*MemVTs=*/nullptr,
1249 FixedOffsets: &Offsets, StartingOffset: ArgOffset);
1250
1251 for (unsigned Value = 0, NumValues = ValueVTs.size();
1252 Value != NumValues; ++Value) {
1253 uint64_t BasePartOffset = Offsets[Value];
1254
1255 EVT ArgVT = ValueVTs[Value];
1256 EVT MemVT = ArgVT;
1257 MVT RegisterVT = getRegisterTypeForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1258 unsigned NumRegs = getNumRegistersForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1259
1260 if (NumRegs == 1) {
1261 // This argument is not split, so the IR type is the memory type.
1262 if (ArgVT.isExtended()) {
1263 // We have an extended type, like i24, so we should just use the
1264 // register type.
1265 MemVT = RegisterVT;
1266 } else {
1267 MemVT = ArgVT;
1268 }
1269 } else if (ArgVT.isVector() && RegisterVT.isVector() &&
1270 ArgVT.getScalarType() == RegisterVT.getScalarType()) {
1271 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
1272 // We have a vector value which has been split into a vector with
1273 // the same scalar type, but fewer elements. This should handle
1274 // all the floating-point vector types.
1275 MemVT = RegisterVT;
1276 } else if (ArgVT.isVector() &&
1277 ArgVT.getVectorNumElements() == NumRegs) {
1278 // This arg has been split so that each element is stored in a separate
1279 // register.
1280 MemVT = ArgVT.getScalarType();
1281 } else if (ArgVT.isExtended()) {
1282 // We have an extended type, like i65.
1283 MemVT = RegisterVT;
1284 } else {
1285 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
1286 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0);
1287 if (RegisterVT.isInteger()) {
1288 MemVT = EVT::getIntegerVT(Context&: State.getContext(), BitWidth: MemoryBits);
1289 } else if (RegisterVT.isVector()) {
1290 assert(!RegisterVT.getScalarType().isFloatingPoint());
1291 unsigned NumElements = RegisterVT.getVectorNumElements();
1292 assert(MemoryBits % NumElements == 0);
1293 // This vector type has been split into another vector type with
1294 // a different elements size.
1295 EVT ScalarVT = EVT::getIntegerVT(Context&: State.getContext(),
1296 BitWidth: MemoryBits / NumElements);
1297 MemVT = EVT::getVectorVT(Context&: State.getContext(), VT: ScalarVT, NumElements);
1298 } else {
1299 llvm_unreachable("cannot deduce memory type.");
1300 }
1301 }
1302
1303 // Convert one element vectors to scalar.
1304 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
1305 MemVT = MemVT.getScalarType();
1306
1307 // Round up vec3/vec5 argument.
1308 if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
1309 MemVT = MemVT.getPow2VectorType(Context&: State.getContext());
1310 } else if (!MemVT.isSimple() && !MemVT.isVector()) {
1311 MemVT = MemVT.getRoundIntegerType(Context&: State.getContext());
1312 }
1313
1314 unsigned PartOffset = 0;
1315 for (unsigned i = 0; i != NumRegs; ++i) {
1316 State.addLoc(V: CCValAssign::getCustomMem(ValNo: InIndex++, ValVT: RegisterVT,
1317 Offset: BasePartOffset + PartOffset,
1318 LocVT: MemVT.getSimpleVT(),
1319 HTP: CCValAssign::Full));
1320 PartOffset += MemVT.getStoreSize();
1321 }
1322 }
1323 }
1324}
1325
1326SDValue AMDGPUTargetLowering::LowerReturn(
1327 SDValue Chain, CallingConv::ID CallConv,
1328 bool isVarArg,
1329 const SmallVectorImpl<ISD::OutputArg> &Outs,
1330 const SmallVectorImpl<SDValue> &OutVals,
1331 const SDLoc &DL, SelectionDAG &DAG) const {
1332 // FIXME: Fails for r600 tests
1333 //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
1334 // "wave terminate should not have return values");
1335 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM, DL, VT: MVT::Other, Operand: Chain);
1336}
1337
1338//===---------------------------------------------------------------------===//
1339// Target specific lowering
1340//===---------------------------------------------------------------------===//
1341
1342/// Selects the correct CCAssignFn for a given CallingConvention value.
1343CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1344 bool IsVarArg) {
1345 return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1346}
1347
1348CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1349 bool IsVarArg) {
1350 return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1351}
1352
1353SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1354 SelectionDAG &DAG,
1355 MachineFrameInfo &MFI,
1356 int ClobberedFI) const {
1357 SmallVector<SDValue, 8> ArgChains;
1358 int64_t FirstByte = MFI.getObjectOffset(ObjectIdx: ClobberedFI);
1359 int64_t LastByte = FirstByte + MFI.getObjectSize(ObjectIdx: ClobberedFI) - 1;
1360
1361 // Include the original chain at the beginning of the list. When this is
1362 // used by target LowerCall hooks, this helps legalize find the
1363 // CALLSEQ_BEGIN node.
1364 ArgChains.push_back(Elt: Chain);
1365
1366 // Add a chain value for each stack argument corresponding
1367 for (SDNode *U : DAG.getEntryNode().getNode()->users()) {
1368 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U)) {
1369 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr())) {
1370 if (FI->getIndex() < 0) {
1371 int64_t InFirstByte = MFI.getObjectOffset(ObjectIdx: FI->getIndex());
1372 int64_t InLastByte = InFirstByte;
1373 InLastByte += MFI.getObjectSize(ObjectIdx: FI->getIndex()) - 1;
1374
1375 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1376 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1377 ArgChains.push_back(Elt: SDValue(L, 1));
1378 }
1379 }
1380 }
1381 }
1382
1383 // Build a tokenfactor for all the chains.
1384 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
1385}
1386
1387SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1388 SmallVectorImpl<SDValue> &InVals,
1389 StringRef Reason) const {
1390 SDValue Callee = CLI.Callee;
1391 SelectionDAG &DAG = CLI.DAG;
1392
1393 const Function &Fn = DAG.getMachineFunction().getFunction();
1394
1395 StringRef FuncName("<unknown>");
1396
1397 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Val&: Callee))
1398 FuncName = G->getSymbol();
1399 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
1400 FuncName = G->getGlobal()->getName();
1401
1402 DAG.getContext()->diagnose(
1403 DI: DiagnosticInfoUnsupported(Fn, Reason + FuncName, CLI.DL.getDebugLoc()));
1404
1405 if (!CLI.IsTailCall) {
1406 for (ISD::InputArg &Arg : CLI.Ins)
1407 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
1408 }
1409
1410 // FIXME: Hack because R600 doesn't handle callseq pseudos yet.
1411 if (getTargetMachine().getTargetTriple().getArch() == Triple::r600)
1412 return CLI.Chain;
1413
1414 SDValue Chain = DAG.getCALLSEQ_START(Chain: CLI.Chain, InSize: 0, OutSize: 0, DL: CLI.DL);
1415 return DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, /*InGlue=*/Glue: SDValue(), DL: CLI.DL);
1416}
1417
1418SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1419 SmallVectorImpl<SDValue> &InVals) const {
1420 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported call to function ");
1421}
1422
1423SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1424 SelectionDAG &DAG) const {
1425 const Function &Fn = DAG.getMachineFunction().getFunction();
1426
1427 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1428 Fn, "unsupported dynamic alloca", SDLoc(Op).getDebugLoc()));
1429 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()), Op.getOperand(i: 0)};
1430 return DAG.getMergeValues(Ops, dl: SDLoc());
1431}
1432
1433SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1434 SelectionDAG &DAG) const {
1435 switch (Op.getOpcode()) {
1436 default:
1437 Op->print(OS&: errs(), G: &DAG);
1438 llvm_unreachable("Custom lowering code for this "
1439 "instruction is not implemented yet!");
1440 break;
1441 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1442 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1443 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1444 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1445 case ISD::SDIVREM:
1446 return LowerSDIVREM(Op, DAG);
1447 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1448 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1449 case ISD::FRINT: return LowerFRINT(Op, DAG);
1450 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1451 case ISD::FROUNDEVEN:
1452 return LowerFROUNDEVEN(Op, DAG);
1453 case ISD::FROUND: return LowerFROUND(Op, DAG);
1454 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1455 case ISD::FLOG2:
1456 return LowerFLOG2(Op, DAG);
1457 case ISD::FLOG:
1458 case ISD::FLOG10:
1459 return LowerFLOGCommon(Op, DAG);
1460 case ISD::FEXP:
1461 case ISD::FEXP10:
1462 return lowerFEXP(Op, DAG);
1463 case ISD::FEXP2:
1464 return lowerFEXP2(Op, DAG);
1465 case ISD::FPOW:
1466 return lowerFPOW(Op, DAG);
1467 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1468 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1469 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1470 case ISD::FP_TO_SINT:
1471 case ISD::FP_TO_UINT:
1472 return LowerFP_TO_INT(Op, DAG);
1473 case ISD::FP_TO_SINT_SAT:
1474 case ISD::FP_TO_UINT_SAT:
1475 return LowerFP_TO_INT_SAT(Op, DAG);
1476 case ISD::CTTZ:
1477 case ISD::CTTZ_ZERO_POISON:
1478 case ISD::CTLZ:
1479 case ISD::CTLZ_ZERO_POISON:
1480 return LowerCTLZ_CTTZ(Op, DAG);
1481 case ISD::CTLS:
1482 return LowerCTLS(Op, DAG);
1483 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1484 }
1485 return Op;
1486}
1487
1488void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
1489 SmallVectorImpl<SDValue> &Results,
1490 SelectionDAG &DAG) const {
1491 switch (N->getOpcode()) {
1492 case ISD::SIGN_EXTEND_INREG:
1493 // Different parts of legalization seem to interpret which type of
1494 // sign_extend_inreg is the one to check for custom lowering. The extended
1495 // from type is what really matters, but some places check for custom
1496 // lowering of the result type. This results in trying to use
1497 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1498 // nothing here and let the illegal result integer be handled normally.
1499 return;
1500 case ISD::FLOG2:
1501 if (SDValue Lowered = LowerFLOG2(Op: SDValue(N, 0), DAG))
1502 Results.push_back(Elt: Lowered);
1503 return;
1504 case ISD::FLOG:
1505 case ISD::FLOG10:
1506 if (SDValue Lowered = LowerFLOGCommon(Op: SDValue(N, 0), DAG))
1507 Results.push_back(Elt: Lowered);
1508 return;
1509 case ISD::FEXP2:
1510 if (SDValue Lowered = lowerFEXP2(Op: SDValue(N, 0), DAG))
1511 Results.push_back(Elt: Lowered);
1512 return;
1513 case ISD::FEXP:
1514 case ISD::FEXP10:
1515 if (SDValue Lowered = lowerFEXP(Op: SDValue(N, 0), DAG))
1516 Results.push_back(Elt: Lowered);
1517 return;
1518 case ISD::CTLZ:
1519 case ISD::CTLZ_ZERO_POISON:
1520 if (auto Lowered = lowerCTLZResults(Op: SDValue(N, 0u), DAG))
1521 Results.push_back(Elt: Lowered);
1522 return;
1523 default:
1524 return;
1525 }
1526}
1527
1528SDValue AMDGPUTargetLowering::LowerBlockAddress(SDValue Op,
1529 SelectionDAG &DAG) const {
1530 BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val&: Op);
1531 SDLoc SL(Op);
1532 EVT VT = Op.getValueType();
1533 return DAG.getTargetBlockAddress(BA: BA->getBlockAddress(), VT, Offset: BA->getOffset(),
1534 TargetFlags: BA->getTargetFlags());
1535}
1536
1537SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
1538 SDValue Op,
1539 SelectionDAG &DAG) const {
1540
1541 const DataLayout &DL = DAG.getDataLayout();
1542 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Val&: Op);
1543 const GlobalValue *GV = G->getGlobal();
1544
1545 if (G->getAddressSpace() == AMDGPUAS::BARRIER) {
1546 const GlobalVariable *GVar = cast<GlobalVariable>(Val: GV);
1547
1548 if (!AMDGPU::isNamedBarrier(GV: *GVar)) {
1549 const Function &Fn = DAG.getMachineFunction().getFunction();
1550 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1551 Fn, "unsupported use of BARRIER address space",
1552 SDLoc(Op).getDebugLoc(), DS_Error));
1553 return DAG.getPOISON(VT: Op.getValueType());
1554 }
1555
1556 unsigned Offset = MFI->allocateBarrierGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1557 return DAG.getConstant(Val: Offset, DL: SDLoc(Op), VT: Op.getValueType());
1558 }
1559
1560 if (!MFI->isModuleEntryFunction()) {
1561 if (std::optional<uint32_t> Address =
1562 AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(
1563 GV: *GV, AS: AMDGPUAS::LOCAL_ADDRESS)) {
1564 return DAG.getConstant(Val: *Address, DL: SDLoc(Op), VT: Op.getValueType());
1565 }
1566 }
1567
1568 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1569 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
1570 if (!MFI->isModuleEntryFunction() &&
1571 GV->getName() != "llvm.amdgcn.module.lds") {
1572 SDLoc DL(Op);
1573 const Function &Fn = DAG.getMachineFunction().getFunction();
1574 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1575 Fn, "local memory global used by non-kernel function",
1576 DL.getDebugLoc(), DS_Warning));
1577
1578 // We currently don't have a way to correctly allocate LDS objects that
1579 // aren't directly associated with a kernel. We do force inlining of
1580 // functions that use local objects. However, if these dead functions are
1581 // not eliminated, we don't want a compile time error. Just emit a warning
1582 // and a trap, since there should be no callable path here.
1583 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL, VT: MVT::Other, Operand: DAG.getEntryNode());
1584 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
1585 N1: Trap, N2: DAG.getRoot());
1586 DAG.setRoot(OutputChain);
1587 return DAG.getPOISON(VT: Op.getValueType());
1588 }
1589
1590 // TODO: We could emit code to handle the initialization somewhere.
1591 // We ignore the initializer for now and legalize it to allow selection.
1592 // The initializer will anyway get errored out during assembly emission.
1593 unsigned Offset = MFI->allocateLDSGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1594 // A constant byte offset (e.g. from a GEP into an array of named barriers)
1595 // folds directly into the allocated LDS address.
1596 return DAG.getConstant(Val: Offset + G->getOffset(), DL: SDLoc(Op),
1597 VT: Op.getValueType());
1598 }
1599 return SDValue();
1600}
1601
1602SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1603 SelectionDAG &DAG) const {
1604 SmallVector<SDValue, 8> Args;
1605 SDLoc SL(Op);
1606
1607 EVT VT = Op.getValueType();
1608 if (VT.getVectorElementType().getSizeInBits() < 32) {
1609 unsigned OpBitSize = Op.getOperand(i: 0).getValueType().getSizeInBits();
1610 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1611 unsigned NewNumElt = OpBitSize / 32;
1612 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1613 : EVT::getVectorVT(Context&: *DAG.getContext(),
1614 VT: MVT::i32, NumElements: NewNumElt);
1615 for (const SDUse &U : Op->ops()) {
1616 SDValue In = U.get();
1617 SDValue NewIn = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewEltVT, Operand: In);
1618 if (NewNumElt > 1)
1619 DAG.ExtractVectorElements(Op: NewIn, Args);
1620 else
1621 Args.push_back(Elt: NewIn);
1622 }
1623
1624 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
1625 NumElements: NewNumElt * Op.getNumOperands());
1626 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1627 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: BV);
1628 }
1629 }
1630
1631 for (const SDUse &U : Op->ops())
1632 DAG.ExtractVectorElements(Op: U.get(), Args);
1633
1634 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1635}
1636
1637SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1638 SelectionDAG &DAG) const {
1639 SDLoc SL(Op);
1640 SmallVector<SDValue, 8> Args;
1641 unsigned Start = Op.getConstantOperandVal(i: 1);
1642 EVT VT = Op.getValueType();
1643 EVT SrcVT = Op.getOperand(i: 0).getValueType();
1644
1645 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
1646 unsigned NumElt = VT.getVectorNumElements();
1647 unsigned NumSrcElt = SrcVT.getVectorNumElements();
1648 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types");
1649
1650 // Extract 32-bit registers at a time.
1651 EVT NewSrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumSrcElt / 2);
1652 EVT NewVT = NumElt == 2
1653 ? MVT::i32
1654 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElt / 2);
1655 SDValue Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewSrcVT, Operand: Op.getOperand(i: 0));
1656
1657 DAG.ExtractVectorElements(Op: Tmp, Args, Start: Start / 2, Count: NumElt / 2);
1658 if (NumElt == 2)
1659 Tmp = Args[0];
1660 else
1661 Tmp = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1662
1663 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Tmp);
1664 }
1665
1666 DAG.ExtractVectorElements(Op: Op.getOperand(i: 0), Args, Start,
1667 Count: VT.getVectorNumElements());
1668
1669 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1670}
1671
1672// TODO: Handle fabs too
1673static SDValue peekFNeg(SDValue Val) {
1674 if (Val.getOpcode() == ISD::FNEG)
1675 return Val.getOperand(i: 0);
1676
1677 return Val;
1678}
1679
1680// SelectionDAG twin of AMDGPUCombinerHelper::canIgnoreLegacyMinMaxTies.
1681static bool canIgnoreLegacyMinMaxTies(const SelectionDAG &DAG,
1682 SDNodeFlags Flags, SDValue LHS,
1683 SDValue RHS) {
1684 return Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
1685 DAG.isKnownNeverLogicalZero(Op: RHS);
1686}
1687
1688SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl(
1689 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1690 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1691 SelectionDAG &DAG = DCI.DAG;
1692 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
1693 assert(CCOpcode != ISD::SETCC_INVALID && "Invalid setcc condcode!");
1694
1695 switch (CCOpcode) {
1696 case ISD::SETOLE:
1697 case ISD::SETOLT:
1698 case ISD::SETLE:
1699 case ISD::SETLT:
1700 case ISD::SETOGE:
1701 case ISD::SETOGT:
1702 case ISD::SETGE:
1703 case ISD::SETGT:
1704 // Only do this after legalization to avoid interfering with other combines
1705 // which might occur.
1706 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1707 !DCI.isCalledByLegalizer())
1708 return SDValue();
1709 break;
1710 default:
1711 break;
1712 }
1713
1714 // Canonicalize so the select returns the compare's LHS on a true predicate.
1715 if (LHS != True)
1716 CCOpcode = ISD::getSetCCInverse(Operation: CCOpcode, Type: VT);
1717
1718 unsigned Opc;
1719 bool Swap; // Emit (rhs, lhs) instead of (lhs, rhs).
1720 switch (CCOpcode) {
1721 case ISD::SETOLT:
1722 case ISD::SETLT:
1723 case ISD::SETOLE:
1724 Opc = AMDGPUISD::FMIN_LEGACY;
1725 Swap = false;
1726 break;
1727 case ISD::SETULE:
1728 case ISD::SETLE:
1729 case ISD::SETULT:
1730 Opc = AMDGPUISD::FMIN_LEGACY;
1731 Swap = true;
1732 break;
1733 case ISD::SETOGE:
1734 case ISD::SETGE:
1735 case ISD::SETOGT:
1736 Opc = AMDGPUISD::FMAX_LEGACY;
1737 Swap = false;
1738 break;
1739 case ISD::SETUGT:
1740 case ISD::SETGT:
1741 case ISD::SETUGE:
1742 Opc = AMDGPUISD::FMAX_LEGACY;
1743 Swap = true;
1744 break;
1745 default:
1746 return SDValue();
1747 }
1748
1749 // For these predicates the NaN-correct operand order is the signed zero
1750 // tie-incorrect one, so the fold needs the tie to be unobservable.
1751 if ((CCOpcode == ISD::SETOLE || CCOpcode == ISD::SETULT ||
1752 CCOpcode == ISD::SETOGT || CCOpcode == ISD::SETUGE) &&
1753 !canIgnoreLegacyMinMaxTies(DAG, Flags, LHS, RHS))
1754 return SDValue();
1755
1756 if (Swap)
1757 std::swap(a&: LHS, b&: RHS);
1758 return DAG.getNode(Opcode: Opc, DL, VT, N1: LHS, N2: RHS, Flags);
1759}
1760
1761/// Generate Min/Max node
1762SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(
1763 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1764 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1765 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
1766 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, Flags,
1767 DCI);
1768
1769 SelectionDAG &DAG = DCI.DAG;
1770
1771 // If we can't directly match this, try to see if we can fold an fneg to
1772 // match.
1773
1774 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
1775 ConstantFPSDNode *CFalse = dyn_cast<ConstantFPSDNode>(Val&: False);
1776 SDValue NegTrue = peekFNeg(Val: True);
1777
1778 // Undo the combine foldFreeOpFromSelect does if it helps us match the
1779 // fmin/fmax.
1780 //
1781 // select (fcmp olt (lhs, K)), (fneg lhs), -K
1782 // -> fneg (fmin_legacy lhs, K)
1783 //
1784 // TODO: Use getNegatedExpression
1785 if (LHS == NegTrue && CFalse && CRHS) {
1786 APFloat NegRHS = neg(X: CRHS->getValueAPF());
1787 if (NegRHS == CFalse->getValueAPF()) {
1788 SDValue Combined = combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True: NegTrue,
1789 False, CC, Flags, DCI);
1790 if (Combined)
1791 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Combined);
1792 return SDValue();
1793 }
1794 }
1795
1796 return SDValue();
1797}
1798
1799std::pair<SDValue, SDValue>
1800AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1801 SDLoc SL(Op);
1802
1803 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1804
1805 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1806 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1807
1808 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1809 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1810
1811 return std::pair(Lo, Hi);
1812}
1813
1814SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1815 SDLoc SL(Op);
1816
1817 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1818 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1819 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1820}
1821
1822SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1823 SDLoc SL(Op);
1824
1825 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1826 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1827 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1828}
1829
1830// Split a vector type into two parts. The first part is a power of two vector.
1831// The second part is whatever is left over, and is a scalar if it would
1832// otherwise be a 1-vector.
1833std::pair<EVT, EVT>
1834AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1835 EVT LoVT, HiVT;
1836 EVT EltVT = VT.getVectorElementType();
1837 unsigned NumElts = VT.getVectorNumElements();
1838 unsigned LoNumElts = PowerOf2Ceil(A: (NumElts + 1) / 2);
1839 LoVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: LoNumElts);
1840 HiVT = NumElts - LoNumElts == 1
1841 ? EltVT
1842 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts - LoNumElts);
1843 return std::pair(LoVT, HiVT);
1844}
1845
1846// Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1847// scalar.
1848std::pair<SDValue, SDValue>
1849AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1850 const EVT &LoVT, const EVT &HiVT,
1851 SelectionDAG &DAG) const {
1852 EVT VT = N.getValueType();
1853 assert(LoVT.getVectorNumElements() +
1854 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
1855 VT.getVectorNumElements() &&
1856 "More vector elements requested than available!");
1857 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N,
1858 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1859
1860 unsigned LoNumElts = LoVT.getVectorNumElements();
1861
1862 if (HiVT.isVector()) {
1863 unsigned HiNumElts = HiVT.getVectorNumElements();
1864 if ((VT.getVectorNumElements() % HiNumElts) == 0) {
1865 // Avoid creating an extract_subvector with an index that isn't a multiple
1866 // of the result type.
1867 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
1868 N2: DAG.getConstant(Val: LoNumElts, DL, VT: MVT::i32));
1869 return {Lo, Hi};
1870 }
1871
1872 SmallVector<SDValue, 8> Elts;
1873 DAG.ExtractVectorElements(Op: N, Args&: Elts, /*Start=*/LoNumElts,
1874 /*Count=*/HiNumElts);
1875 SDValue Hi = DAG.getBuildVector(VT: HiVT, DL, Ops: Elts);
1876 return {Lo, Hi};
1877 }
1878
1879 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: HiVT, N1: N,
1880 N2: DAG.getVectorIdxConstant(Val: LoNumElts, DL));
1881 return {Lo, Hi};
1882}
1883
1884SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1885 SelectionDAG &DAG) const {
1886 LoadSDNode *Load = cast<LoadSDNode>(Val: Op);
1887 EVT VT = Op.getValueType();
1888 SDLoc SL(Op);
1889
1890
1891 // If this is a 2 element vector, we really want to scalarize and not create
1892 // weird 1 element vectors.
1893 if (VT.getVectorNumElements() == 2) {
1894 SDValue Ops[2];
1895 std::tie(args&: Ops[0], args&: Ops[1]) = scalarizeVectorLoad(LD: Load, DAG);
1896 return DAG.getMergeValues(Ops, dl: SL);
1897 }
1898
1899 SDValue BasePtr = Load->getBasePtr();
1900 EVT MemVT = Load->getMemoryVT();
1901
1902 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1903
1904 EVT LoVT, HiVT;
1905 EVT LoMemVT, HiMemVT;
1906 SDValue Lo, Hi;
1907
1908 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1909 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1910 std::tie(args&: Lo, args&: Hi) = splitVector(N: Op, DL: SL, LoVT, HiVT, DAG);
1911
1912 unsigned Size = LoMemVT.getStoreSize();
1913 Align BaseAlign = Load->getAlign();
1914 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1915
1916 SDValue LoLoad = DAG.getExtLoad(
1917 ExtType: Load->getExtensionType(), dl: SL, VT: LoVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1918 MemVT: LoMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1919 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Size));
1920 SDValue HiLoad = DAG.getExtLoad(
1921 ExtType: Load->getExtensionType(), dl: SL, VT: HiVT, Chain: Load->getChain(), Ptr: HiPtr,
1922 PtrInfo: SrcValue.getWithOffset(O: LoMemVT.getStoreSize()), MemVT: HiMemVT, Alignment: HiAlign,
1923 MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1924
1925 SDValue Join;
1926 if (LoVT == HiVT) {
1927 // This is the case that the vector is power of two so was evenly split.
1928 Join = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, N1: LoLoad, N2: HiLoad);
1929 } else {
1930 Join = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SL, VT, N1: DAG.getPOISON(VT), N2: LoLoad,
1931 N3: DAG.getVectorIdxConstant(Val: 0, DL: SL));
1932 Join = DAG.getNode(
1933 Opcode: HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, DL: SL,
1934 VT, N1: Join, N2: HiLoad,
1935 N3: DAG.getVectorIdxConstant(Val: LoVT.getVectorNumElements(), DL: SL));
1936 }
1937
1938 SDValue Ops[] = {Join, DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
1939 N1: LoLoad.getValue(R: 1), N2: HiLoad.getValue(R: 1))};
1940
1941 return DAG.getMergeValues(Ops, dl: SL);
1942}
1943
1944SDValue AMDGPUTargetLowering::WidenOrSplitVectorLoad(SDValue Op,
1945 SelectionDAG &DAG) const {
1946 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
1947 EVT VT = Op.getValueType();
1948 SDValue BasePtr = Load->getBasePtr();
1949 EVT MemVT = Load->getMemoryVT();
1950 SDLoc SL(Op);
1951 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1952 Align BaseAlign = Load->getAlign();
1953 unsigned NumElements = MemVT.getVectorNumElements();
1954
1955 // Widen from vec3 to vec4 when the load is at least 8-byte aligned
1956 // or 16-byte fully dereferenceable. Otherwise, split the vector load.
1957 if (NumElements != 3 ||
1958 (BaseAlign < Align(8) &&
1959 !SrcValue.isDereferenceable(Size: 16, C&: *DAG.getContext(), DL: DAG.getDataLayout())))
1960 return SplitVectorLoad(Op, DAG);
1961
1962 assert(NumElements == 3);
1963
1964 EVT WideVT =
1965 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
1966 EVT WideMemVT =
1967 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(), NumElements: 4);
1968 SDValue WideLoad = DAG.getExtLoad(
1969 ExtType: Load->getExtensionType(), dl: SL, VT: WideVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1970 MemVT: WideMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags());
1971 return DAG.getMergeValues(
1972 Ops: {DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT, N1: WideLoad,
1973 N2: DAG.getVectorIdxConstant(Val: 0, DL: SL)),
1974 WideLoad.getValue(R: 1)},
1975 dl: SL);
1976}
1977
1978SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1979 SelectionDAG &DAG) const {
1980 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
1981 SDValue Val = Store->getValue();
1982 EVT VT = Val.getValueType();
1983
1984 // If this is a 2 element vector, we really want to scalarize and not create
1985 // weird 1 element vectors.
1986 if (VT.getVectorNumElements() == 2)
1987 return scalarizeVectorStore(ST: Store, DAG);
1988
1989 EVT MemVT = Store->getMemoryVT();
1990 SDValue Chain = Store->getChain();
1991 SDValue BasePtr = Store->getBasePtr();
1992 SDLoc SL(Op);
1993
1994 EVT LoVT, HiVT;
1995 EVT LoMemVT, HiMemVT;
1996 SDValue Lo, Hi;
1997
1998 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1999 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
2000 std::tie(args&: Lo, args&: Hi) = splitVector(N: Val, DL: SL, LoVT, HiVT, DAG);
2001
2002 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: LoMemVT.getStoreSize());
2003
2004 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
2005 Align BaseAlign = Store->getAlign();
2006 unsigned Size = LoMemVT.getStoreSize();
2007 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
2008
2009 SDValue LoStore =
2010 DAG.getTruncStore(Chain, dl: SL, Val: Lo, Ptr: BasePtr, PtrInfo: SrcValue, SVT: LoMemVT, Alignment: BaseAlign,
2011 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
2012 SDValue HiStore = DAG.getTruncStore(
2013 Chain, dl: SL, Val: Hi, Ptr: HiPtr, PtrInfo: SrcValue.getWithOffset(O: Size), SVT: HiMemVT, Alignment: HiAlign,
2014 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
2015
2016 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: LoStore, N2: HiStore);
2017}
2018
2019// This is a shortcut for integer division because we have fast i32<->f32
2020// conversions, and fast f32 reciprocal instructions.
2021SDValue AMDGPUTargetLowering::LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG,
2022 bool Sign) const {
2023 SDLoc DL(Op);
2024 EVT VT = Op.getValueType();
2025 assert(VT == MVT::i32 && "LowerDIVREMToFloat expects an i32");
2026
2027 SDValue LHS = Op.getOperand(i: 0);
2028 SDValue RHS = Op.getOperand(i: 1);
2029 MVT IntVT = MVT::i32;
2030 MVT FltVT = MVT::f32;
2031
2032 unsigned LHSSignBits;
2033 unsigned RHSSignBits;
2034 if (Sign) {
2035 LHSSignBits = DAG.ComputeNumSignBits(Op: LHS);
2036 RHSSignBits = DAG.ComputeNumSignBits(Op: RHS);
2037 if (LHSSignBits < 9 || RHSSignBits < 9)
2038 return SDValue();
2039 } else {
2040 KnownBits LHSKnown = DAG.computeKnownBits(Op: LHS);
2041 KnownBits RHSKnown = DAG.computeKnownBits(Op: RHS);
2042
2043 LHSSignBits = LHSKnown.countMinLeadingZeros();
2044 RHSSignBits = RHSKnown.countMinLeadingZeros();
2045 }
2046
2047 unsigned BitSize = VT.getSizeInBits();
2048 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
2049 unsigned DivBits = BitSize - SignBits;
2050 if (Sign)
2051 ++DivBits;
2052
2053 // In order to avoid problems due to 1 ulp accuracy issues with v_rcp_f32,
2054 // limit LowerDIVREMToFloat to:
2055 // [-0x400000,0x3FFFFF] for Sign
2056 // [ 0x000000,0x3FFFFF] for !Sign
2057 // This matches what is done in expandDivRemToFloatImpl.
2058 if (DivBits > (Sign ? 23 : 22))
2059 return SDValue();
2060
2061 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
2062 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
2063
2064 // int ia = (int)LHS;
2065 SDValue ia = LHS;
2066
2067 // int ib, (int)RHS;
2068 SDValue ib = RHS;
2069
2070 // The calculation:
2071 // fq = fa*recip(fb)
2072 // may be too small due to the 1ulp accuracy in the recip
2073 // operation and rounding issues. Since fq is truncated to produce
2074 // an integer value it may be too small by one. This is
2075 // dealt with by incrementing fa by 1ulp:
2076 // fq = (fa+1ulp)*recip(fb)
2077 // This will increase fa's magnitude by at most 0.5
2078 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
2079 // Thus, this method is safe since fa must be incremented by at least 1.0
2080 // for the quotient to increase by one.
2081 SDValue fa = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ia);
2082 SDValue faAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: fa);
2083 SDValue faIncremented = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: faAsInt,
2084 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
2085 fa = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FltVT, Operand: faIncremented);
2086
2087 // float fb = (float)ib;
2088 SDValue fb = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ib);
2089
2090 SDValue fq = DAG.getNode(Opcode: ISD::FMUL, DL, VT: FltVT,
2091 N1: fa, N2: DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: FltVT, Operand: fb));
2092
2093 // fq = trunc(fq);
2094 fq = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: FltVT, Operand: fq);
2095
2096 // int iq = (int)fq;
2097 SDValue Div = DAG.getNode(Opcode: ToInt, DL, VT: IntVT, Operand: fq);
2098
2099 // Rem needs compensation, it's easier to recompute it
2100 SDValue Rem = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Div, N2: RHS);
2101 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Rem);
2102
2103 return DAG.getMergeValues(Ops: { Div, Rem }, dl: DL);
2104}
2105
2106void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
2107 SelectionDAG &DAG,
2108 SmallVectorImpl<SDValue> &Results) const {
2109 SDLoc DL(Op);
2110 EVT VT = Op.getValueType();
2111
2112 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
2113
2114 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2115
2116 SDValue One = DAG.getConstant(Val: 1, DL, VT: HalfVT);
2117 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HalfVT);
2118
2119 //HiLo split
2120 SDValue LHS_Lo, LHS_Hi;
2121 SDValue LHS = Op.getOperand(i: 0);
2122 std::tie(args&: LHS_Lo, args&: LHS_Hi) = DAG.SplitScalar(N: LHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2123
2124 SDValue RHS_Lo, RHS_Hi;
2125 SDValue RHS = Op.getOperand(i: 1);
2126 std::tie(args&: RHS_Lo, args&: RHS_Hi) = DAG.SplitScalar(N: RHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2127
2128 if (DAG.MaskedValueIsZero(Op: RHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32)) &&
2129 DAG.MaskedValueIsZero(Op: LHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32))) {
2130
2131 SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2132 N1: LHS_Lo, N2: RHS_Lo);
2133
2134 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 0), Zero});
2135 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 1), Zero});
2136
2137 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV));
2138 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM));
2139 return;
2140 }
2141
2142 if (isTypeLegal(VT: MVT::i64)) {
2143 // The algorithm here is based on ideas from "Software Integer Division",
2144 // Tom Rodeheffer, August 2008.
2145
2146 MachineFunction &MF = DAG.getMachineFunction();
2147 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2148
2149 // Compute denominator reciprocal.
2150 unsigned FMAD =
2151 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA
2152 : MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign()
2153 ? (unsigned)ISD::FMAD
2154 : (unsigned)AMDGPUISD::FMAD_FTZ;
2155
2156 SDValue Cvt_Lo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Lo);
2157 SDValue Cvt_Hi = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Hi);
2158 SDValue Mad1 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Cvt_Hi,
2159 N2: DAG.getConstantFP(Val: APInt(32, 0x4f800000).bitsToFloat(), DL, VT: MVT::f32),
2160 N3: Cvt_Lo);
2161 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: MVT::f32, Operand: Mad1);
2162 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Rcp,
2163 N2: DAG.getConstantFP(Val: APInt(32, 0x5f7ffffc).bitsToFloat(), DL, VT: MVT::f32));
2164 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Mul1,
2165 N2: DAG.getConstantFP(Val: APInt(32, 0x2f800000).bitsToFloat(), DL, VT: MVT::f32));
2166 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: MVT::f32, Operand: Mul2);
2167 SDValue Mad2 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Trunc,
2168 N2: DAG.getConstantFP(Val: APInt(32, 0xcf800000).bitsToFloat(), DL, VT: MVT::f32),
2169 N3: Mul1);
2170 SDValue Rcp_Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Mad2);
2171 SDValue Rcp_Hi = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Trunc);
2172 SDValue Rcp64 = DAG.getBitcast(VT,
2173 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Rcp_Lo, Rcp_Hi}));
2174
2175 SDValue Zero64 = DAG.getConstant(Val: 0, DL, VT);
2176 SDValue One64 = DAG.getConstant(Val: 1, DL, VT);
2177 SDValue Zero1 = DAG.getConstant(Val: 0, DL, VT: MVT::i1);
2178 SDVTList HalfCarryVT = DAG.getVTList(VT1: HalfVT, VT2: MVT::i1);
2179
2180 // First round of UNR (Unsigned integer Newton-Raphson).
2181 SDValue Neg_RHS = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero64, N2: RHS);
2182 SDValue Mullo1 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Rcp64);
2183 SDValue Mulhi1 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Rcp64, N2: Mullo1);
2184 SDValue Mulhi1_Lo, Mulhi1_Hi;
2185 std::tie(args&: Mulhi1_Lo, args&: Mulhi1_Hi) =
2186 DAG.SplitScalar(N: Mulhi1, DL, LoVT: HalfVT, HiVT: HalfVT);
2187 SDValue Add1_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Lo,
2188 N2: Mulhi1_Lo, N3: Zero1);
2189 SDValue Add1_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Hi,
2190 N2: Mulhi1_Hi, N3: Add1_Lo.getValue(R: 1));
2191 SDValue Add1 = DAG.getBitcast(VT,
2192 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add1_Lo, Add1_Hi}));
2193
2194 // Second round of UNR.
2195 SDValue Mullo2 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Add1);
2196 SDValue Mulhi2 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Add1, N2: Mullo2);
2197 SDValue Mulhi2_Lo, Mulhi2_Hi;
2198 std::tie(args&: Mulhi2_Lo, args&: Mulhi2_Hi) =
2199 DAG.SplitScalar(N: Mulhi2, DL, LoVT: HalfVT, HiVT: HalfVT);
2200 SDValue Add2_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Lo,
2201 N2: Mulhi2_Lo, N3: Zero1);
2202 SDValue Add2_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Hi,
2203 N2: Mulhi2_Hi, N3: Add2_Lo.getValue(R: 1));
2204 SDValue Add2 = DAG.getBitcast(VT,
2205 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add2_Lo, Add2_Hi}));
2206
2207 SDValue Mulhi3 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: LHS, N2: Add2);
2208
2209 SDValue Mul3 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: RHS, N2: Mulhi3);
2210
2211 SDValue Mul3_Lo, Mul3_Hi;
2212 std::tie(args&: Mul3_Lo, args&: Mul3_Hi) = DAG.SplitScalar(N: Mul3, DL, LoVT: HalfVT, HiVT: HalfVT);
2213 SDValue Sub1_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Lo,
2214 N2: Mul3_Lo, N3: Zero1);
2215 SDValue Sub1_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Hi,
2216 N2: Mul3_Hi, N3: Sub1_Lo.getValue(R: 1));
2217 SDValue Sub1_Mi = DAG.getNode(Opcode: ISD::SUB, DL, VT: HalfVT, N1: LHS_Hi, N2: Mul3_Hi);
2218 SDValue Sub1 = DAG.getBitcast(VT,
2219 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub1_Lo, Sub1_Hi}));
2220
2221 SDValue MinusOne = DAG.getConstant(Val: 0xffffffffu, DL, VT: HalfVT);
2222 SDValue C1 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2223 Cond: ISD::SETUGE);
2224 SDValue C2 = DAG.getSelectCC(DL, LHS: Sub1_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2225 Cond: ISD::SETUGE);
2226 SDValue C3 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: C2, False: C1, Cond: ISD::SETEQ);
2227
2228 // TODO: Here and below portions of the code can be enclosed into if/endif.
2229 // Currently control flow is unconditional and we have 4 selects after
2230 // potential endif to substitute PHIs.
2231
2232 // if C3 != 0 ...
2233 SDValue Sub2_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Lo,
2234 N2: RHS_Lo, N3: Zero1);
2235 SDValue Sub2_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Mi,
2236 N2: RHS_Hi, N3: Sub1_Lo.getValue(R: 1));
2237 SDValue Sub2_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2238 N2: Zero, N3: Sub2_Lo.getValue(R: 1));
2239 SDValue Sub2 = DAG.getBitcast(VT,
2240 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub2_Lo, Sub2_Hi}));
2241
2242 SDValue Add3 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mulhi3, N2: One64);
2243
2244 SDValue C4 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2245 Cond: ISD::SETUGE);
2246 SDValue C5 = DAG.getSelectCC(DL, LHS: Sub2_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2247 Cond: ISD::SETUGE);
2248 SDValue C6 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: C5, False: C4, Cond: ISD::SETEQ);
2249
2250 // if (C6 != 0)
2251 SDValue Add4 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Add3, N2: One64);
2252
2253 SDValue Sub3_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Lo,
2254 N2: RHS_Lo, N3: Zero1);
2255 SDValue Sub3_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2256 N2: RHS_Hi, N3: Sub2_Lo.getValue(R: 1));
2257 SDValue Sub3_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub3_Mi,
2258 N2: Zero, N3: Sub3_Lo.getValue(R: 1));
2259 SDValue Sub3 = DAG.getBitcast(VT,
2260 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub3_Lo, Sub3_Hi}));
2261
2262 // endif C6
2263 // endif C3
2264
2265 SDValue Sel1 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Add4, False: Add3, Cond: ISD::SETNE);
2266 SDValue Div = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel1, False: Mulhi3, Cond: ISD::SETNE);
2267
2268 SDValue Sel2 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Sub3, False: Sub2, Cond: ISD::SETNE);
2269 SDValue Rem = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel2, False: Sub1, Cond: ISD::SETNE);
2270
2271 Results.push_back(Elt: Div);
2272 Results.push_back(Elt: Rem);
2273
2274 return;
2275 }
2276
2277 // r600 expandion.
2278 // Get Speculative values
2279 SDValue DIV_Part = DAG.getNode(Opcode: ISD::UDIV, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2280 SDValue REM_Part = DAG.getNode(Opcode: ISD::UREM, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2281
2282 SDValue REM_Lo = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: REM_Part, False: LHS_Hi, Cond: ISD::SETEQ);
2283 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {REM_Lo, Zero});
2284 REM = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM);
2285
2286 SDValue DIV_Hi = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: DIV_Part, False: Zero, Cond: ISD::SETEQ);
2287 SDValue DIV_Lo = Zero;
2288
2289 const unsigned halfBitWidth = HalfVT.getSizeInBits();
2290
2291 for (unsigned i = 0; i < halfBitWidth; ++i) {
2292 const unsigned bitPos = halfBitWidth - i - 1;
2293 SDValue POS = DAG.getConstant(Val: bitPos, DL, VT: HalfVT);
2294 // Get value of high bit
2295 SDValue HBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: HalfVT, N1: LHS_Lo, N2: POS);
2296 HBit = DAG.getNode(Opcode: ISD::AND, DL, VT: HalfVT, N1: HBit, N2: One);
2297 HBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: HBit);
2298
2299 // Shift
2300 REM = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: REM, N2: DAG.getConstant(Val: 1, DL, VT));
2301 // Add LHS high bit
2302 REM = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: REM, N2: HBit);
2303
2304 SDValue BIT = DAG.getConstant(Val: 1ULL << bitPos, DL, VT: HalfVT);
2305 SDValue realBIT = DAG.getSelectCC(DL, LHS: REM, RHS, True: BIT, False: Zero, Cond: ISD::SETUGE);
2306
2307 DIV_Lo = DAG.getNode(Opcode: ISD::OR, DL, VT: HalfVT, N1: DIV_Lo, N2: realBIT);
2308
2309 // Update REM
2310 SDValue REM_sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: REM, N2: RHS);
2311 REM = DAG.getSelectCC(DL, LHS: REM, RHS, True: REM_sub, False: REM, Cond: ISD::SETUGE);
2312 }
2313
2314 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {DIV_Lo, DIV_Hi});
2315 DIV = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV);
2316 Results.push_back(Elt: DIV);
2317 Results.push_back(Elt: REM);
2318}
2319
2320SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
2321 SelectionDAG &DAG) const {
2322 SDLoc DL(Op);
2323 EVT VT = Op.getValueType();
2324
2325 if (VT == MVT::i64) {
2326 SmallVector<SDValue, 2> Results;
2327 LowerUDIVREM64(Op, DAG, Results);
2328 return DAG.getMergeValues(Ops: Results, dl: DL);
2329 }
2330
2331 if (VT == MVT::i32) {
2332 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: false))
2333 return Res;
2334 }
2335
2336 SDValue X = Op.getOperand(i: 0);
2337 SDValue Y = Op.getOperand(i: 1);
2338
2339 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
2340 // algorithm used here.
2341
2342 // Initial estimate of inv(y).
2343 SDValue Z = DAG.getNode(Opcode: AMDGPUISD::URECIP, DL, VT, Operand: Y);
2344
2345 // One round of UNR.
2346 SDValue NegY = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Y);
2347 SDValue NegYZ = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: NegY, N2: Z);
2348 Z = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Z,
2349 N2: DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Z, N2: NegYZ));
2350
2351 // Quotient/remainder estimate.
2352 SDValue Q = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: X, N2: Z);
2353 SDValue R =
2354 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: X, N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Q, N2: Y));
2355
2356 // First quotient/remainder refinement.
2357 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2358 SDValue One = DAG.getConstant(Val: 1, DL, VT);
2359 SDValue Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2360 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2361 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2362 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2363 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2364
2365 // Second quotient/remainder refinement.
2366 Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2367 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2368 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2369 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2370 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2371
2372 return DAG.getMergeValues(Ops: {Q, R}, dl: DL);
2373}
2374
2375SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
2376 SelectionDAG &DAG) const {
2377 SDLoc DL(Op);
2378 EVT VT = Op.getValueType();
2379
2380 SDValue LHS = Op.getOperand(i: 0);
2381 SDValue RHS = Op.getOperand(i: 1);
2382
2383 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
2384 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2385
2386 if (VT == MVT::i32) {
2387 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: true))
2388 return Res;
2389 }
2390
2391 // LHS must have > 33 sign-bits to ensure that LHS != -2147483648
2392 // Otherwise 32-bit division cannot be used safely.
2393 // -2147483648/1 and -2147483648/-1 are not equal,
2394 // but they produce the same lower 32-bit result.
2395 if (VT == MVT::i64 && DAG.ComputeNumSignBits(Op: LHS) > 33 &&
2396 DAG.ComputeNumSignBits(Op: RHS) > 32) {
2397 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2398
2399 //HiLo split
2400 SDValue LHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: LHS, N2: Zero);
2401 SDValue RHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: RHS, N2: Zero);
2402 SDValue DIVREM = DAG.getNode(Opcode: ISD::SDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2403 N1: LHS_Lo, N2: RHS_Lo);
2404 SDValue Res[2] = {
2405 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 0)),
2406 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 1))
2407 };
2408 return DAG.getMergeValues(Ops: Res, dl: DL);
2409 }
2410
2411 SDValue LHSign = DAG.getSelectCC(DL, LHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2412 SDValue RHSign = DAG.getSelectCC(DL, LHS: RHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2413 SDValue DSign = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHSign, N2: RHSign);
2414 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2415
2416 LHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: LHSign);
2417 RHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: RHS, N2: RHSign);
2418
2419 LHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHS, N2: LHSign);
2420 RHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: RHS, N2: RHSign);
2421
2422 SDValue Div = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
2423 SDValue Rem = Div.getValue(R: 1);
2424
2425 Div = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Div, N2: DSign);
2426 Rem = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Rem, N2: RSign);
2427
2428 Div = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Div, N2: DSign);
2429 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Rem, N2: RSign);
2430
2431 SDValue Res[2] = {
2432 Div,
2433 Rem
2434 };
2435 return DAG.getMergeValues(Ops: Res, dl: DL);
2436}
2437
2438SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2439 SDLoc SL(Op);
2440 SDValue Src = Op.getOperand(i: 0);
2441
2442 // result = trunc(src)
2443 // if (src > 0.0 && src != result)
2444 // result += 1.0
2445
2446 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2447
2448 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2449 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
2450
2451 EVT SetCCVT =
2452 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2453
2454 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOGT);
2455 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2456 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2457
2458 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: One, N3: Zero);
2459 // TODO: Should this propagate fast-math-flags?
2460 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2461}
2462
2463static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2464 SelectionDAG &DAG) {
2465 const unsigned FractBits = 52;
2466 const unsigned ExpBits = 11;
2467
2468 SDValue ExpPart = DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32,
2469 N1: Hi,
2470 N2: DAG.getConstant(Val: FractBits - 32, DL: SL, VT: MVT::i32),
2471 N3: DAG.getConstant(Val: ExpBits, DL: SL, VT: MVT::i32));
2472 SDValue Exp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ExpPart,
2473 N2: DAG.getConstant(Val: 1023, DL: SL, VT: MVT::i32));
2474
2475 return Exp;
2476}
2477
2478SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2479 SDLoc SL(Op);
2480 SDValue Src = Op.getOperand(i: 0);
2481
2482 assert(Op.getValueType() == MVT::f64);
2483
2484 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
2485
2486 // Extract the upper half, since this is where we will find the sign and
2487 // exponent.
2488 SDValue Hi = getHiHalf64(Op: Src, DAG);
2489
2490 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2491
2492 const unsigned FractBits = 52;
2493
2494 // Extract the sign bit.
2495 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, DL: SL, VT: MVT::i32);
2496 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Hi, N2: SignBitMask);
2497
2498 // Extend back to 64-bits.
2499 SDValue SignBit64 = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Zero, SignBit});
2500 SignBit64 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: SignBit64);
2501
2502 SDValue BcInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Src);
2503 const SDValue FractMask
2504 = DAG.getConstant(Val: (UINT64_C(1) << FractBits) - 1, DL: SL, VT: MVT::i64);
2505
2506 SDValue Shr = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: FractMask, N2: Exp);
2507 SDValue Not = DAG.getNOT(DL: SL, Val: Shr, VT: MVT::i64);
2508 SDValue Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i64, N1: BcInt, N2: Not);
2509
2510 EVT SetCCVT =
2511 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i32);
2512
2513 const SDValue FiftyOne = DAG.getConstant(Val: FractBits - 1, DL: SL, VT: MVT::i32);
2514
2515 SDValue ExpLt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: Zero, Cond: ISD::SETLT);
2516 SDValue ExpGt51 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: FiftyOne, Cond: ISD::SETGT);
2517
2518 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpLt0, N2: SignBit64, N3: Tmp0);
2519 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpGt51, N2: BcInt, N3: Tmp1);
2520
2521 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f64, Operand: Tmp2);
2522}
2523
2524SDValue AMDGPUTargetLowering::LowerFROUNDEVEN(SDValue Op,
2525 SelectionDAG &DAG) const {
2526 SDLoc SL(Op);
2527 SDValue Src = Op.getOperand(i: 0);
2528
2529 assert(Op.getValueType() == MVT::f64);
2530
2531 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2532 SDValue C1 = DAG.getConstantFP(Val: C1Val, DL: SL, VT: MVT::f64);
2533 SDValue CopySign = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MVT::f64, N1: C1, N2: Src);
2534
2535 // TODO: Should this propagate fast-math-flags?
2536
2537 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Src, N2: CopySign);
2538 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT: MVT::f64, N1: Tmp1, N2: CopySign);
2539
2540 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f64, Operand: Src);
2541
2542 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2543 SDValue C2 = DAG.getConstantFP(Val: C2Val, DL: SL, VT: MVT::f64);
2544
2545 EVT SetCCVT =
2546 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2547 SDValue Cond = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Fabs, RHS: C2, Cond: ISD::SETOGT);
2548
2549 return DAG.getSelect(DL: SL, VT: MVT::f64, Cond, LHS: Src, RHS: Tmp2);
2550}
2551
2552SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op,
2553 SelectionDAG &DAG) const {
2554 // FNEARBYINT and FRINT are the same, except in their handling of FP
2555 // exceptions. Those aren't really meaningful for us, and OpenCL only has
2556 // rint, so just treat them as equivalent.
2557 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT: Op.getValueType(),
2558 Operand: Op.getOperand(i: 0));
2559}
2560
2561SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2562 auto VT = Op.getValueType();
2563 auto Arg = Op.getOperand(i: 0u);
2564 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT, Operand: Arg);
2565}
2566
2567// XXX - May require not supporting f32 denormals?
2568
2569// Don't handle v2f16. The extra instructions to scalarize and repack around the
2570// compare and vselect end up producing worse code than scalarizing the whole
2571// operation.
2572SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2573 SDLoc SL(Op);
2574 SDValue X = Op.getOperand(i: 0);
2575 EVT VT = Op.getValueType();
2576
2577 SDValue T = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: X);
2578
2579 // TODO: Should this propagate fast-math-flags?
2580
2581 SDValue Diff = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: T);
2582
2583 SDValue AbsDiff = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Diff);
2584
2585 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2586 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2587
2588 EVT SetCCVT =
2589 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2590
2591 const SDValue Half = DAG.getConstantFP(Val: 0.5, DL: SL, VT);
2592 SDValue Cmp = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsDiff, RHS: Half, Cond: ISD::SETOGE);
2593 SDValue OneOrZeroFP = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cmp, N2: One, N3: Zero);
2594
2595 SDValue SignedOffset = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: OneOrZeroFP, N2: X);
2596 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: T, N2: SignedOffset);
2597}
2598
2599SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2600 SDLoc SL(Op);
2601 SDValue Src = Op.getOperand(i: 0);
2602
2603 // result = trunc(src);
2604 // if (src < 0.0 && src != result)
2605 // result += -1.0.
2606
2607 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2608
2609 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2610 const SDValue NegOne = DAG.getConstantFP(Val: -1.0, DL: SL, VT: MVT::f64);
2611
2612 EVT SetCCVT =
2613 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2614
2615 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOLT);
2616 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2617 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2618
2619 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: NegOne, N3: Zero);
2620 // TODO: Should this propagate fast-math-flags?
2621 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2622}
2623
2624/// Return true if it's known that \p Src can never be an f32 denormal value.
2625static bool valueIsKnownNeverF32Denorm(SDValue Src) {
2626 switch (Src.getOpcode()) {
2627 case ISD::FP_EXTEND:
2628 return Src.getOperand(i: 0).getValueType() == MVT::f16;
2629 case ISD::FP16_TO_FP:
2630 case ISD::FFREXP:
2631 case ISD::FSQRT:
2632 case AMDGPUISD::LOG:
2633 case AMDGPUISD::EXP:
2634 return true;
2635 case ISD::INTRINSIC_WO_CHAIN: {
2636 unsigned IntrinsicID = Src.getConstantOperandVal(i: 0);
2637 switch (IntrinsicID) {
2638 case Intrinsic::amdgcn_frexp_mant:
2639 case Intrinsic::amdgcn_log:
2640 case Intrinsic::amdgcn_log_clamp:
2641 case Intrinsic::amdgcn_exp2:
2642 case Intrinsic::amdgcn_sqrt:
2643 return true;
2644 default:
2645 return false;
2646 }
2647 }
2648 default:
2649 return false;
2650 }
2651
2652 llvm_unreachable("covered opcode switch");
2653}
2654
2655bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG,
2656 SDNodeFlags Flags) {
2657 return Flags.hasApproximateFuncs();
2658}
2659
2660bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG,
2661 SDValue Src,
2662 SDNodeFlags Flags) {
2663 return !valueIsKnownNeverF32Denorm(Src) &&
2664 DAG.getMachineFunction()
2665 .getDenormalMode(FPType: APFloat::IEEEsingle())
2666 .Input != DenormalMode::PreserveSign;
2667}
2668
2669SDValue AMDGPUTargetLowering::getIsLtSmallestNormal(SelectionDAG &DAG,
2670 SDValue Src,
2671 SDNodeFlags Flags) const {
2672 SDLoc SL(Src);
2673 EVT VT = Src.getValueType();
2674 const fltSemantics &Semantics = VT.getFltSemantics();
2675 SDValue SmallestNormal =
2676 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2677
2678 // Want to scale denormals up, but negatives and 0 work just as well on the
2679 // scaled path.
2680 SDValue IsLtSmallestNormal = DAG.getSetCC(
2681 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2682 RHS: SmallestNormal, Cond: ISD::SETOLT);
2683
2684 return IsLtSmallestNormal;
2685}
2686
2687SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src,
2688 SDNodeFlags Flags) const {
2689 SDLoc SL(Src);
2690 EVT VT = Src.getValueType();
2691 const fltSemantics &Semantics = VT.getFltSemantics();
2692 SDValue Inf = DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL: SL, VT);
2693
2694 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Src, Flags);
2695 SDValue IsFinite = DAG.getSetCC(
2696 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Fabs,
2697 RHS: Inf, Cond: ISD::SETOLT);
2698 return IsFinite;
2699}
2700
2701/// If denormal handling is required return the scaled input to FLOG2, and the
2702/// check for denormal range. Otherwise, return null values.
2703std::pair<SDValue, SDValue>
2704AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL,
2705 SDValue Src, SDNodeFlags Flags) const {
2706 if (!needsDenormHandlingF32(DAG, Src, Flags))
2707 return {};
2708
2709 MVT VT = MVT::f32;
2710 const fltSemantics &Semantics = APFloat::IEEEsingle();
2711 SDValue SmallestNormal =
2712 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2713
2714 SDValue IsLtSmallestNormal = DAG.getSetCC(
2715 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2716 RHS: SmallestNormal, Cond: ISD::SETOLT);
2717
2718 SDValue Scale32 = DAG.getConstantFP(Val: 0x1.0p+32, DL: SL, VT);
2719 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2720 SDValue ScaleFactor =
2721 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: Scale32, N3: One, Flags);
2722
2723 SDValue ScaledInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Src, N2: ScaleFactor, Flags);
2724 return {ScaledInput, IsLtSmallestNormal};
2725}
2726
2727SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const {
2728 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
2729 // If we have to handle denormals, scale up the input and adjust the result.
2730
2731 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
2732 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
2733
2734 SDLoc SL(Op);
2735 EVT VT = Op.getValueType();
2736 SDValue Src = Op.getOperand(i: 0);
2737 SDNodeFlags Flags = Op->getFlags();
2738
2739 if (VT == MVT::f16) {
2740 // Nothing in half is a denormal when promoted to f32.
2741 assert(!isTypeLegal(VT));
2742 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
2743 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
2744 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
2745 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
2746 }
2747
2748 auto [ScaledInput, IsLtSmallestNormal] =
2749 getScaledLogInput(DAG, SL, Src, Flags);
2750 if (!ScaledInput)
2751 return DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: Src, Flags);
2752
2753 SDValue Log2 = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2754
2755 SDValue ThirtyTwo = DAG.getConstantFP(Val: 32.0, DL: SL, VT);
2756 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2757 SDValue ResultOffset =
2758 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: ThirtyTwo, N3: Zero);
2759 return DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: Log2, N2: ResultOffset, Flags);
2760}
2761
2762static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2763 SDValue Y, SDValue C, SDNodeFlags Flags = SDNodeFlags()) {
2764 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Y, Flags);
2765 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: C, Flags);
2766}
2767
2768SDValue AMDGPUTargetLowering::LowerFLOGCommon(SDValue Op,
2769 SelectionDAG &DAG) const {
2770 SDValue X = Op.getOperand(i: 0);
2771 EVT VT = Op.getValueType();
2772 SDNodeFlags Flags = Op->getFlags();
2773 SDLoc DL(Op);
2774 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10;
2775 assert(IsLog10 || Op.getOpcode() == ISD::FLOG);
2776
2777 if (VT == MVT::f16 || Flags.hasApproximateFuncs()) {
2778 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
2779 // depending on !fpmath metadata.
2780
2781 bool PromoteToF32 = VT == MVT::f16 && (!Flags.hasApproximateFuncs() ||
2782 !isTypeLegal(VT: MVT::f16));
2783
2784 if (PromoteToF32) {
2785 // Log and multiply in f32 is always good enough for f16.
2786 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2787 }
2788
2789 SDValue Lowered = LowerFLOGUnsafe(Op: X, SL: DL, DAG, IsLog10, Flags);
2790 if (PromoteToF32) {
2791 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Lowered,
2792 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32), Flags);
2793 }
2794
2795 return Lowered;
2796 }
2797
2798 SDValue ScaledInput, IsScaled;
2799 if (VT == MVT::f16)
2800 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2801 else {
2802 std::tie(args&: ScaledInput, args&: IsScaled) = getScaledLogInput(DAG, SL: DL, Src: X, Flags);
2803 if (ScaledInput)
2804 X = ScaledInput;
2805 }
2806
2807 SDValue Y = DAG.getNode(Opcode: AMDGPUISD::LOG, DL, VT, Operand: X, Flags);
2808
2809 SDValue R;
2810 if (Subtarget->hasFastFMAF32()) {
2811 // c+cc are ln(2)/ln(10) to more than 49 bits
2812 const float c_log10 = 0x1.344134p-2f;
2813 const float cc_log10 = 0x1.09f79ep-26f;
2814
2815 // c + cc is ln(2) to more than 49 bits
2816 const float c_log = 0x1.62e42ep-1f;
2817 const float cc_log = 0x1.efa39ep-25f;
2818
2819 SDValue C = DAG.getConstantFP(Val: IsLog10 ? c_log10 : c_log, DL, VT);
2820 SDValue CC = DAG.getConstantFP(Val: IsLog10 ? cc_log10 : cc_log, DL, VT);
2821 // This adds correction terms for which contraction may lead to an increase
2822 // in the error of the approximation, so disable it.
2823 Flags.setAllowContract(false);
2824 R = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Y, N2: C, Flags);
2825 SDValue NegR = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: R, Flags);
2826 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: C, N3: NegR, Flags);
2827 SDValue FMA1 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: CC, N3: FMA0, Flags);
2828 R = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: R, N2: FMA1, Flags);
2829 } else {
2830 // ch+ct is ln(2)/ln(10) to more than 36 bits
2831 const float ch_log10 = 0x1.344000p-2f;
2832 const float ct_log10 = 0x1.3509f6p-18f;
2833
2834 // ch + ct is ln(2) to more than 36 bits
2835 const float ch_log = 0x1.62e000p-1f;
2836 const float ct_log = 0x1.0bfbe8p-15f;
2837
2838 SDValue CH = DAG.getConstantFP(Val: IsLog10 ? ch_log10 : ch_log, DL, VT);
2839 SDValue CT = DAG.getConstantFP(Val: IsLog10 ? ct_log10 : ct_log, DL, VT);
2840
2841 SDValue YAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Y);
2842 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL, VT: MVT::i32);
2843 SDValue YHInt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: YAsInt, N2: MaskConst);
2844 SDValue YH = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: YHInt);
2845 SDValue YT = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: YH, Flags);
2846 // This adds correction terms for which contraction may lead to an increase
2847 // in the error of the approximation, so disable it.
2848 Flags.setAllowContract(false);
2849 SDValue YTCT = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: YT, N2: CT, Flags);
2850 SDValue Mad0 = getMad(DAG, SL: DL, VT, X: YH, Y: CT, C: YTCT, Flags);
2851 SDValue Mad1 = getMad(DAG, SL: DL, VT, X: YT, Y: CH, C: Mad0, Flags);
2852 R = getMad(DAG, SL: DL, VT, X: YH, Y: CH, C: Mad1);
2853 }
2854
2855 const bool IsFiniteOnly = Flags.hasNoNaNs() && Flags.hasNoInfs();
2856
2857 // TODO: Check if known finite from source value.
2858 if (!IsFiniteOnly) {
2859 SDValue IsFinite = getIsFinite(DAG, Src: Y, Flags);
2860 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsFinite, N2: R, N3: Y, Flags);
2861 }
2862
2863 if (IsScaled) {
2864 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
2865 SDValue ShiftK =
2866 DAG.getConstantFP(Val: IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2867 SDValue Shift =
2868 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsScaled, N2: ShiftK, N3: Zero, Flags);
2869 R = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: R, N2: Shift, Flags);
2870 }
2871
2872 return R;
2873}
2874
2875SDValue AMDGPUTargetLowering::LowerFLOG10(SDValue Op, SelectionDAG &DAG) const {
2876 return LowerFLOGCommon(Op, DAG);
2877}
2878
2879// Do f32 fast math expansion for flog2 or flog10. This is accurate enough for a
2880// promote f16 operation.
2881SDValue AMDGPUTargetLowering::LowerFLOGUnsafe(SDValue Src, const SDLoc &SL,
2882 SelectionDAG &DAG, bool IsLog10,
2883 SDNodeFlags Flags) const {
2884 EVT VT = Src.getValueType();
2885 unsigned LogOp =
2886 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2;
2887
2888 double Log2BaseInverted =
2889 IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2890
2891 if (VT == MVT::f32) {
2892 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags);
2893 if (ScaledInput) {
2894 SDValue LogSrc = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2895 SDValue ScaledResultOffset =
2896 DAG.getConstantFP(Val: -32.0 * Log2BaseInverted, DL: SL, VT);
2897
2898 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL: SL, VT);
2899
2900 SDValue ResultOffset = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsScaled,
2901 N2: ScaledResultOffset, N3: Zero, Flags);
2902
2903 SDValue Log2Inv = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2904
2905 if (Subtarget->hasFastFMAF32())
2906 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: LogSrc, N2: Log2Inv, N3: ResultOffset,
2907 Flags);
2908 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LogSrc, N2: Log2Inv, Flags);
2909 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: ResultOffset);
2910 }
2911 }
2912
2913 SDValue Log2Operand = DAG.getNode(Opcode: LogOp, DL: SL, VT, Operand: Src, Flags);
2914 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2915
2916 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Log2Operand, N2: Log2BaseInvertedOperand,
2917 Flags);
2918}
2919
2920// This expansion gives a result slightly better than 1ulp.
2921SDValue AMDGPUTargetLowering::lowerFEXPF64(SDValue Op,
2922 SelectionDAG &DAG) const {
2923 SDLoc DL(Op);
2924 SDValue X = Op.getOperand(i: 0);
2925
2926 // TODO: Check if reassoc is safe. There is an output change in exp2 and
2927 // exp10, which slightly increases ulp.
2928 SDNodeFlags Flags = Op->getFlags() & ~SDNodeFlags::AllowReassociation;
2929
2930 SDValue DN, F, T;
2931
2932 if (Op.getOpcode() == ISD::FEXP2) {
2933 // dn = rint(x)
2934 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: X, Flags);
2935 // f = x - dn
2936 F = DAG.getNode(Opcode: ISD::FSUB, DL, VT: MVT::f64, N1: X, N2: DN, Flags);
2937 // t = f*C1 + f*C2
2938 SDValue C1 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2939 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2940 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C2, Flags);
2941 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C1, N3: Mul2, Flags);
2942 } else if (Op.getOpcode() == ISD::FEXP10) {
2943 // dn = rint(x * C1)
2944 SDValue C1 = DAG.getConstantFP(Val: 0x1.a934f0979a371p+1, DL, VT: MVT::f64);
2945 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2946 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2947
2948 // f = FMA(-dn, C2, FMA(-dn, C3, x))
2949 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2950 SDValue C2 = DAG.getConstantFP(Val: -0x1.9dc1da994fd21p-59, DL, VT: MVT::f64);
2951 SDValue C3 = DAG.getConstantFP(Val: 0x1.34413509f79ffp-2, DL, VT: MVT::f64);
2952 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2953 F = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2954
2955 // t = FMA(f, C4, f*C5)
2956 SDValue C4 = DAG.getConstantFP(Val: 0x1.26bb1bbb55516p+1, DL, VT: MVT::f64);
2957 SDValue C5 = DAG.getConstantFP(Val: -0x1.f48ad494ea3e9p-53, DL, VT: MVT::f64);
2958 SDValue MulF = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C5, Flags);
2959 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C4, N3: MulF, Flags);
2960 } else { // ISD::FEXP
2961 // dn = rint(x * C1)
2962 SDValue C1 = DAG.getConstantFP(Val: 0x1.71547652b82fep+0, DL, VT: MVT::f64);
2963 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2964 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2965
2966 // t = FMA(-dn, C2, FMA(-dn, C3, x))
2967 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2968 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2969 SDValue C3 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2970 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2971 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2972 }
2973
2974 // Polynomial expansion for p
2975 SDValue P = DAG.getConstantFP(Val: 0x1.ade156a5dcb37p-26, DL, VT: MVT::f64);
2976 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2977 N3: DAG.getConstantFP(Val: 0x1.28af3fca7ab0cp-22, DL, VT: MVT::f64),
2978 Flags);
2979 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2980 N3: DAG.getConstantFP(Val: 0x1.71dee623fde64p-19, DL, VT: MVT::f64),
2981 Flags);
2982 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2983 N3: DAG.getConstantFP(Val: 0x1.a01997c89e6b0p-16, DL, VT: MVT::f64),
2984 Flags);
2985 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2986 N3: DAG.getConstantFP(Val: 0x1.a01a014761f6ep-13, DL, VT: MVT::f64),
2987 Flags);
2988 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2989 N3: DAG.getConstantFP(Val: 0x1.6c16c1852b7b0p-10, DL, VT: MVT::f64),
2990 Flags);
2991 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2992 N3: DAG.getConstantFP(Val: 0x1.1111111122322p-7, DL, VT: MVT::f64), Flags);
2993 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2994 N3: DAG.getConstantFP(Val: 0x1.55555555502a1p-5, DL, VT: MVT::f64), Flags);
2995 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2996 N3: DAG.getConstantFP(Val: 0x1.5555555555511p-3, DL, VT: MVT::f64), Flags);
2997 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2998 N3: DAG.getConstantFP(Val: 0x1.000000000000bp-1, DL, VT: MVT::f64), Flags);
2999
3000 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT: MVT::f64);
3001
3002 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
3003 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
3004
3005 // z = ldexp(p, (int)dn)
3006 SDValue DNInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: MVT::i32, Operand: DN);
3007 SDValue Z = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: P, N2: DNInt, Flags);
3008
3009 // Overflow/underflow guards
3010 SDValue CondHi = DAG.getSetCC(
3011 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: 1024.0, DL, VT: MVT::f64), Cond: ISD::SETULE);
3012
3013 if (!Flags.hasNoInfs()) {
3014 SDValue PInf = DAG.getConstantFP(Val: std::numeric_limits<double>::infinity(),
3015 DL, VT: MVT::f64);
3016 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondHi, LHS: Z, RHS: PInf, Flags);
3017 }
3018
3019 SDValue CondLo = DAG.getSetCC(
3020 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: -1075.0, DL, VT: MVT::f64), Cond: ISD::SETUGE);
3021 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
3022 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondLo, LHS: Z, RHS: Zero, Flags);
3023
3024 return Z;
3025}
3026
3027SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const {
3028 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
3029 // If we have to handle denormals, scale up the input and adjust the result.
3030
3031 EVT VT = Op.getValueType();
3032 if (VT == MVT::f64)
3033 return lowerFEXPF64(Op, DAG);
3034
3035 SDLoc SL(Op);
3036 SDValue Src = Op.getOperand(i: 0);
3037 SDNodeFlags Flags = Op->getFlags();
3038
3039 if (VT == MVT::f16) {
3040 // Nothing in half is a denormal when promoted to f32.
3041 assert(!isTypeLegal(MVT::f16));
3042 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3043 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
3044 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
3045 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3046 }
3047
3048 assert(VT == MVT::f32);
3049
3050 if (!needsDenormHandlingF32(DAG, Src, Flags))
3051 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3052
3053 // bool needs_scaling = x < -0x1.f80000p+6f;
3054 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3055
3056 // -nextafter(128.0, -1)
3057 SDValue RangeCheckConst = DAG.getConstantFP(Val: -0x1.f80000p+6f, DL: SL, VT);
3058
3059 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3060
3061 SDValue NeedsScaling =
3062 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: RangeCheckConst, Cond: ISD::SETOLT);
3063
3064 SDValue SixtyFour = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3065 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3066
3067 SDValue AddOffset =
3068 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: SixtyFour, N3: Zero);
3069
3070 SDValue AddInput = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Src, N2: AddOffset, Flags);
3071 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: AddInput, Flags);
3072
3073 SDValue TwoExpNeg64 = DAG.getConstantFP(Val: 0x1.0p-64f, DL: SL, VT);
3074 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
3075 SDValue ResultScale =
3076 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: TwoExpNeg64, N3: One);
3077
3078 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScale, Flags);
3079}
3080
3081SDValue AMDGPUTargetLowering::lowerFEXPUnsafeImpl(SDValue X, const SDLoc &SL,
3082 SelectionDAG &DAG,
3083 SDNodeFlags Flags,
3084 bool IsExp10) const {
3085 // exp(x) -> exp2(M_LOG2E_F * x);
3086 // exp10(x) -> exp2(log2(10) * x);
3087 EVT VT = X.getValueType();
3088 SDValue Const =
3089 DAG.getConstantFP(Val: IsExp10 ? 0x1.a934f0p+1f : numbers::log2e, DL: SL, VT);
3090
3091 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Const, Flags);
3092 return DAG.getNode(Opcode: VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
3093 : (unsigned)ISD::FEXP2,
3094 DL: SL, VT, Operand: Mul, Flags);
3095}
3096
3097SDValue AMDGPUTargetLowering::lowerFEXPUnsafe(SDValue X, const SDLoc &SL,
3098 SelectionDAG &DAG,
3099 SDNodeFlags Flags) const {
3100 EVT VT = X.getValueType();
3101 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags))
3102 return lowerFEXPUnsafeImpl(X, SL, DAG, Flags, /*IsExp10=*/false);
3103
3104 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3105
3106 SDValue Threshold = DAG.getConstantFP(Val: -0x1.5d58a0p+6f, DL: SL, VT);
3107 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3108
3109 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3110
3111 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3112
3113 SDValue AdjustedX =
3114 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3115
3116 const SDValue Log2E = DAG.getConstantFP(Val: numbers::log2e, DL: SL, VT);
3117 SDValue ExpInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: Log2E, Flags);
3118
3119 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: ExpInput, Flags);
3120
3121 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.969d48p-93f, DL: SL, VT);
3122 SDValue AdjustedResult =
3123 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScaleFactor, Flags);
3124
3125 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: Exp2,
3126 Flags);
3127}
3128
3129/// Emit approx-funcs appropriate lowering for exp10. inf/nan should still be
3130/// handled correctly.
3131SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL,
3132 SelectionDAG &DAG,
3133 SDNodeFlags Flags) const {
3134 const EVT VT = X.getValueType();
3135
3136 const unsigned Exp2Op = VT == MVT::f32 ? static_cast<unsigned>(AMDGPUISD::EXP)
3137 : static_cast<unsigned>(ISD::FEXP2);
3138
3139 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags)) {
3140 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
3141 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3142 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3143
3144 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K0, Flags);
3145 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3146 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K1, Flags);
3147 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3148 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1);
3149 }
3150
3151 // bool s = x < -0x1.2f7030p+5f;
3152 // x += s ? 0x1.0p+5f : 0.0f;
3153 // exp10 = exp2(x * 0x1.a92000p+1f) *
3154 // exp2(x * 0x1.4f0978p-11f) *
3155 // (s ? 0x1.9f623ep-107f : 1.0f);
3156
3157 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3158
3159 SDValue Threshold = DAG.getConstantFP(Val: -0x1.2f7030p+5f, DL: SL, VT);
3160 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3161
3162 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+5f, DL: SL, VT);
3163 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3164 SDValue AdjustedX =
3165 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3166
3167 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3168 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3169
3170 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K0, Flags);
3171 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3172 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K1, Flags);
3173 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3174
3175 SDValue MulExps = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1, Flags);
3176
3177 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.9f623ep-107f, DL: SL, VT);
3178 SDValue AdjustedResult =
3179 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: MulExps, N2: ResultScaleFactor, Flags);
3180
3181 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: MulExps,
3182 Flags);
3183}
3184
3185SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
3186 EVT VT = Op.getValueType();
3187
3188 if (VT == MVT::f64)
3189 return lowerFEXPF64(Op, DAG);
3190
3191 SDLoc SL(Op);
3192 SDValue X = Op.getOperand(i: 0);
3193 SDNodeFlags Flags = Op->getFlags();
3194 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10;
3195
3196 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
3197 // library behavior. Also, is known-not-daz source sufficient?
3198 if (allowApproxFunc(DAG, Flags)) { // TODO: Does this really require fast?
3199 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags)
3200 : lowerFEXPUnsafe(X, SL, DAG, Flags);
3201 }
3202
3203 if (VT.getScalarType() == MVT::f16) {
3204 if (VT.isVector())
3205 return SDValue();
3206
3207 // Nothing in half is a denormal when promoted to f32.
3208 //
3209 // exp(f16 x) ->
3210 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
3211 //
3212 // exp10(f16 x) ->
3213 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
3214 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: X, Flags);
3215 SDValue Lowered = lowerFEXPUnsafeImpl(X: Ext, SL, DAG, Flags, IsExp10);
3216 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Lowered,
3217 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3218 }
3219
3220 assert(VT == MVT::f32);
3221
3222 // Algorithm:
3223 //
3224 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
3225 //
3226 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
3227 // n = 64*m + j, 0 <= j < 64
3228 //
3229 // e^x = 2^((64*m + j + f)/64)
3230 // = (2^m) * (2^(j/64)) * 2^(f/64)
3231 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
3232 //
3233 // f = x*(64/ln(2)) - n
3234 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
3235 //
3236 // e^x = (2^m) * (2^(j/64)) * e^r
3237 //
3238 // (2^(j/64)) is precomputed
3239 //
3240 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3241 // e^r = 1 + q
3242 //
3243 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3244 //
3245 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
3246 SDNodeFlags FlagsNoContract = Flags;
3247 FlagsNoContract.setAllowContract(false);
3248
3249 SDValue PH, PL;
3250 if (Subtarget->hasFastFMAF32()) {
3251 const float c_exp = numbers::log2ef;
3252 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
3253 const float c_exp10 = 0x1.a934f0p+1f;
3254 const float cc_exp10 = 0x1.2f346ep-24f;
3255
3256 SDValue C = DAG.getConstantFP(Val: IsExp10 ? c_exp10 : c_exp, DL: SL, VT);
3257 SDValue CC = DAG.getConstantFP(Val: IsExp10 ? cc_exp10 : cc_exp, DL: SL, VT);
3258
3259 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: C, Flags);
3260 SDValue NegPH = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: PH, Flags);
3261 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: C, N3: NegPH, Flags);
3262 PL = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: CC, N3: FMA0, Flags);
3263 } else {
3264 const float ch_exp = 0x1.714000p+0f;
3265 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
3266
3267 const float ch_exp10 = 0x1.a92000p+1f;
3268 const float cl_exp10 = 0x1.4f0978p-11f;
3269
3270 SDValue CH = DAG.getConstantFP(Val: IsExp10 ? ch_exp10 : ch_exp, DL: SL, VT);
3271 SDValue CL = DAG.getConstantFP(Val: IsExp10 ? cl_exp10 : cl_exp, DL: SL, VT);
3272
3273 SDValue XAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: X);
3274 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL: SL, VT: MVT::i32);
3275 SDValue XHAsInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: XAsInt, N2: MaskConst);
3276 SDValue XH = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: XHAsInt);
3277 SDValue XL = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: XH, Flags);
3278
3279 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XH, N2: CH, Flags);
3280
3281 SDValue XLCL = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XL, N2: CL, Flags);
3282 SDValue Mad0 = getMad(DAG, SL, VT, X: XL, Y: CH, C: XLCL, Flags);
3283 PL = getMad(DAG, SL, VT, X: XH, Y: CL, C: Mad0, Flags);
3284 }
3285
3286 SDValue E = DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SL, VT, Operand: PH, Flags);
3287
3288 // It is unsafe to contract this fsub into the PH multiply.
3289 SDValue PHSubE = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: PH, N2: E, Flags: FlagsNoContract);
3290
3291 SDValue A = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: PHSubE, N2: PL, Flags);
3292 SDValue IntE = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: SL, VT: MVT::i32, Operand: E);
3293 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: A, Flags);
3294
3295 SDValue R = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: Exp2, N2: IntE, Flags);
3296
3297 SDValue UnderflowCheckConst =
3298 DAG.getConstantFP(Val: IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, DL: SL, VT);
3299
3300 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3301 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3302 SDValue Underflow =
3303 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: UnderflowCheckConst, Cond: ISD::SETOLT);
3304
3305 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Underflow, N2: Zero, N3: R);
3306
3307 if (!Flags.hasNoInfs()) {
3308 SDValue OverflowCheckConst =
3309 DAG.getConstantFP(Val: IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, DL: SL, VT);
3310 SDValue Overflow =
3311 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: OverflowCheckConst, Cond: ISD::SETOGT);
3312 SDValue Inf =
3313 DAG.getConstantFP(Val: APFloat::getInf(Sem: APFloat::IEEEsingle()), DL: SL, VT);
3314 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Overflow, N2: Inf, N3: R);
3315 }
3316
3317 return R;
3318}
3319
3320// No pow instruction or libcall to fall back on. fmul_legacy returns 0 for a
3321// zero operand even against an infinity or a NaN, so pow(x, 0) and pow(1, y)
3322// fall out as exp2(0) = 1.
3323SDValue AMDGPUTargetLowering::lowerFPOW(SDValue Op, SelectionDAG &DAG) const {
3324 EVT VT = Op.getValueType();
3325 assert(VT == MVT::f32);
3326
3327 SDLoc SL(Op);
3328 SDValue X = Op.getOperand(i: 0);
3329 SDValue Y = Op.getOperand(i: 1);
3330 SDNodeFlags Flags = Op->getFlags();
3331
3332 // log2(0) is -inf, which exp2 turns back into a finite result, so the core
3333 // goes infinite for inputs a ninf fpow still asserts about, like pow(0, 2).
3334 SDNodeFlags CoreFlags = Flags;
3335 CoreFlags.setNoInfs(false);
3336
3337 // Fast expansion: ignores denormals, NaN for a negative base.
3338 if (allowApproxFunc(DAG, Flags)) {
3339 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: X, Flags: CoreFlags);
3340 SDValue Mul =
3341 DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL: SL, VT, N1: Y, N2: Log, Flags: CoreFlags);
3342 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: Mul, Flags: CoreFlags);
3343 }
3344
3345 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: X, Flags);
3346 SDValue Log = DAG.getNode(Opcode: ISD::FLOG2, DL: SL, VT, Operand: Abs, Flags: CoreFlags);
3347 SDValue Mul = DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL: SL, VT, N1: Y, N2: Log, Flags: CoreFlags);
3348 SDValue R = DAG.getNode(Opcode: ISD::FEXP2, DL: SL, VT, Operand: Mul, Flags: CoreFlags);
3349
3350 // A base that is never negative needs neither the sign fixup nor the NaN.
3351 if (DAG.computeKnownFPClass(Op: X, InterestedClasses: fcNegative).signBitIsZeroOrNaN())
3352 return R;
3353
3354 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3355
3356 // Infinities count as integers, and every f32 >= 2^24 in magnitude is even.
3357 SDValue YTrunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: Y);
3358 SDValue YIsInt = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: YTrunc, RHS: Y, Cond: ISD::SETOEQ);
3359 SDValue YHalf =
3360 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Y, N2: DAG.getConstantFP(Val: 0.5, DL: SL, VT));
3361 SDValue YHalfTrunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: YHalf);
3362 SDValue YIsOdd =
3363 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: YIsInt,
3364 N2: DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: YHalfTrunc, RHS: YHalf, Cond: ISD::SETONE));
3365
3366 // pow(-x, odd y) = -pow(x, y). Selecting the copysign lets even y fold it.
3367 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: YIsOdd,
3368 N2: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: R, N2: X), N3: R);
3369
3370 if (Flags.hasNoNaNs())
3371 return R;
3372
3373 // A negative finite base to a non-integral power is NaN. -inf is excluded:
3374 // the core already gives pow(+inf, y). So are subnormals when flushed.
3375 FPClassTest NegFiniteMask = fcNegNormal;
3376 if (!DAG.getMachineFunction()
3377 .getDenormalMode(FPType: APFloat::IEEEsingle())
3378 .inputsAreZero())
3379 NegFiniteMask |= fcNegSubnormal;
3380 SDValue XNegFinite =
3381 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: SL, VT: SetCCVT, N1: X,
3382 N2: DAG.getTargetConstant(Val: NegFiniteMask, DL: SL, VT: MVT::i32));
3383 // Not a SETONE compare: pow(-1, NaN) needs the NaN-true behavior of !SETOEQ.
3384 SDValue NegNonInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: XNegFinite,
3385 N2: DAG.getNOT(DL: SL, Val: YIsInt, VT: SetCCVT));
3386 SDValue NaN =
3387 DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()), DL: SL, VT);
3388 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NegNonInt, N2: NaN, N3: R);
3389}
3390
3391static bool isCtlzOpc(unsigned Opc) {
3392 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
3393}
3394
3395static bool isCttzOpc(unsigned Opc) {
3396 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_POISON;
3397}
3398
3399SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op,
3400 SelectionDAG &DAG) const {
3401 auto SL = SDLoc(Op);
3402 auto Opc = Op.getOpcode();
3403 auto Arg = Op.getOperand(i: 0u);
3404 auto ResultVT = Op.getValueType();
3405
3406 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3407 return {};
3408
3409 assert(isCtlzOpc(Opc));
3410 assert(ResultVT == Arg.getValueType());
3411
3412 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3413 SDValue NumExtBits = DAG.getConstant(Val: 32u - NumBits, DL: SL, VT: MVT::i32);
3414 SDValue NewOp;
3415
3416 if (Opc == ISD::CTLZ_ZERO_POISON) {
3417 NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3418 NewOp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3419 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3420 } else {
3421 NewOp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3422 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3423 NewOp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3424 }
3425
3426 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ResultVT, Operand: NewOp);
3427}
3428
3429SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
3430 SDLoc SL(Op);
3431 SDValue Src = Op.getOperand(i: 0);
3432
3433 assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode()));
3434 bool Ctlz = isCtlzOpc(Opc: Op.getOpcode());
3435 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;
3436
3437 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_POISON ||
3438 Op.getOpcode() == ISD::CTTZ_ZERO_POISON;
3439 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3440
3441 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3442 // (ctlz hi:lo) -> (umin (ffbh src), 32)
3443 // (cttz hi:lo) -> (umin (ffbl src), 32)
3444 // (ctlz_zero_poison src) -> (ffbh src)
3445 // (cttz_zero_poison src) -> (ffbl src)
3446
3447 // 64-bit scalar version produce 32-bit result
3448 // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64)
3449 // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64)
3450 // (ctlz_zero_poison src) -> (S_FLBIT_I32_B64 src)
3451 // (cttz_zero_poison src) -> (S_FF1_I32_B64 src)
3452 SDValue NewOpr = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Src);
3453 if (!ZeroUndef) {
3454 const SDValue ConstVal = DAG.getConstant(
3455 Val: Op.getValueType().getScalarSizeInBits(), DL: SL, VT: MVT::i32);
3456 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: ConstVal);
3457 }
3458 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: Src.getValueType(), Operand: NewOpr);
3459 }
3460
3461 SDValue Lo, Hi;
3462 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3463
3464 SDValue OprLo = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Lo);
3465 SDValue OprHi = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Hi);
3466
3467 // (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64)
3468 // (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64)
3469 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
3470 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
3471
3472 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT;
3473 const SDValue Const32 = DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32);
3474 if (Ctlz)
3475 OprLo = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprLo, N2: Const32);
3476 else
3477 OprHi = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprHi, N2: Const32);
3478
3479 SDValue NewOpr;
3480 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: OprLo, N2: OprHi);
3481 if (!ZeroUndef) {
3482 const SDValue Const64 = DAG.getConstant(Val: 64, DL: SL, VT: MVT::i32);
3483 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: Const64);
3484 }
3485
3486 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: NewOpr);
3487}
3488
3489SDValue AMDGPUTargetLowering::LowerCTLS(SDValue Op, SelectionDAG &DAG) const {
3490 SDLoc SL(Op);
3491 SDValue Src = Op.getOperand(i: 0);
3492 assert(Src.getValueType() == MVT::i32 && "LowerCTLS only supports i32");
3493 SDValue Ffbh = DAG.getNode(
3494 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3495 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Src);
3496 SDValue Clamped = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: Ffbh,
3497 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3498 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: Clamped,
3499 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i32));
3500}
3501
3502SDValue AMDGPUTargetLowering::LowerINT_TO_FP16(SDValue Op, SelectionDAG &DAG,
3503 EVT FP16Ty) const {
3504 assert(FP16Ty == MVT::f16 || FP16Ty == MVT::bf16);
3505 SDLoc SL(Op);
3506 SDValue Src = Op.getOperand(i: 0);
3507 SDValue ToF32 = DAG.getNode(Opcode: Op.getOpcode(), DL: SL, VT: MVT::f32, Operand: Src);
3508 SDValue FPRoundFlag = DAG.getIntPtrConstant(Val: 0, DL: SL, /*isTarget=*/true);
3509 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: FP16Ty, N1: ToF32, N2: FPRoundFlag);
3510}
3511
3512SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
3513 bool Signed) const {
3514 // The regular method converting a 64-bit integer to float roughly consists of
3515 // 2 steps: normalization and rounding. In fact, after normalization, the
3516 // conversion from a 64-bit integer to a float is essentially the same as the
3517 // one from a 32-bit integer. The only difference is that it has more
3518 // trailing bits to be rounded. To leverage the native 32-bit conversion, a
3519 // 64-bit integer could be preprocessed and fit into a 32-bit integer then
3520 // converted into the correct float number. The basic steps for the unsigned
3521 // conversion are illustrated in the following pseudo code:
3522 //
3523 // f32 uitofp(i64 u) {
3524 // i32 hi, lo = split(u);
3525 // // Only count the leading zeros in hi as we have native support of the
3526 // // conversion from i32 to f32. If hi is all 0s, the conversion is
3527 // // reduced to a 32-bit one automatically.
3528 // i32 shamt = clz(hi); // Return 32 if hi is all 0s.
3529 // u <<= shamt;
3530 // hi, lo = split(u);
3531 // hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo.
3532 // // convert it as a 32-bit integer and scale the result back.
3533 // return uitofp(hi) * 2^(32 - shamt);
3534 // }
3535 //
3536 // The signed one follows the same principle but uses 'ffbh_i32' to count its
3537 // sign bits instead. If 'ffbh_i32' is not available, its absolute value is
3538 // converted instead followed by negation based its sign bit.
3539
3540 SDLoc SL(Op);
3541 SDValue Src = Op.getOperand(i: 0);
3542
3543 SDValue Lo, Hi;
3544 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3545 SDValue Sign;
3546 SDValue ShAmt;
3547 if (Signed && Subtarget->isGCN()) {
3548 // We also need to consider the sign bit in Lo if Hi has just sign bits,
3549 // i.e. Hi is 0 or -1. However, that only needs to take the MSB into
3550 // account. That is, the maximal shift is
3551 // - 32 if Lo and Hi have opposite signs;
3552 // - 33 if Lo and Hi have the same sign.
3553 //
3554 // Or, MaxShAmt = 33 + OppositeSign, where
3555 //
3556 // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is
3557 // - -1 if Lo and Hi have opposite signs; and
3558 // - 0 otherwise.
3559 //
3560 // All in all, ShAmt is calculated as
3561 //
3562 // umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1.
3563 //
3564 // or
3565 //
3566 // umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31).
3567 //
3568 // to reduce the critical path.
3569 SDValue OppositeSign = DAG.getNode(
3570 Opcode: ISD::SRA, DL: SL, VT: MVT::i32, N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: Lo, N2: Hi),
3571 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3572 SDValue MaxShAmt =
3573 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3574 N2: OppositeSign);
3575 // Count the leading sign bits.
3576 ShAmt = DAG.getNode(
3577 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3578 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Hi);
3579 // Different from unsigned conversion, the shift should be one bit less to
3580 // preserve the sign bit.
3581 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ShAmt,
3582 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
3583 ShAmt = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: ShAmt, N2: MaxShAmt);
3584 } else {
3585 if (Signed) {
3586 // Without 'ffbh_i32', only leading zeros could be counted. Take the
3587 // absolute value first.
3588 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: Src,
3589 N2: DAG.getConstant(Val: 63, DL: SL, VT: MVT::i64));
3590 SDValue Abs =
3591 DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64,
3592 N1: DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: Src, N2: Sign), N2: Sign);
3593 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Abs, DAG);
3594 }
3595 // Count the leading zeros.
3596 ShAmt = DAG.getNode(Opcode: ISD::CTLZ, DL: SL, VT: MVT::i32, Operand: Hi);
3597 // The shift amount for signed integers is [0, 32].
3598 }
3599 // Normalize the given 64-bit integer.
3600 SDValue Norm = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i64, N1: Src, N2: ShAmt);
3601 // Split it again.
3602 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Norm, DAG);
3603 // Calculate the adjust bit for rounding.
3604 // (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo)
3605 SDValue Adjust = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32,
3606 N1: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32), N2: Lo);
3607 // Get the 32-bit normalized integer.
3608 Norm = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Hi, N2: Adjust);
3609 // Convert the normalized 32-bit integer into f32.
3610
3611 bool UseLDEXP = isOperationLegal(Op: ISD::FLDEXP, VT: MVT::f32);
3612 unsigned Opc = Signed && UseLDEXP ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3613 SDValue FVal = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::f32, Operand: Norm);
3614
3615 // Finally, need to scale back the converted floating number as the original
3616 // 64-bit integer is converted as a 32-bit one.
3617 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3618 N2: ShAmt);
3619 // On GCN, use LDEXP directly.
3620 if (UseLDEXP)
3621 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f32, N1: FVal, N2: ShAmt);
3622
3623 // Otherwise, align 'ShAmt' to the exponent part and add it into the exponent
3624 // part directly to emulate the multiplication of 2^ShAmt. That 8-bit
3625 // exponent is enough to avoid overflowing into the sign bit.
3626 SDValue Exp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ShAmt,
3627 N2: DAG.getConstant(Val: 23, DL: SL, VT: MVT::i32));
3628 SDValue IVal =
3629 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32,
3630 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: FVal), N2: Exp);
3631 if (Signed) {
3632 // Set the sign bit.
3633 Sign = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32,
3634 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Sign),
3635 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3636 IVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: IVal, N2: Sign);
3637 }
3638 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: IVal);
3639}
3640
3641SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
3642 bool Signed) const {
3643 SDLoc SL(Op);
3644 SDValue Src = Op.getOperand(i: 0);
3645
3646 SDValue Lo, Hi;
3647 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3648
3649 SDValue CvtHi = DAG.getNode(Opcode: Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
3650 DL: SL, VT: MVT::f64, Operand: Hi);
3651
3652 SDValue CvtLo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: SL, VT: MVT::f64, Operand: Lo);
3653
3654 SDValue LdExp = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f64, N1: CvtHi,
3655 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3656 // TODO: Should this propagate fast-math-flags?
3657 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: LdExp, N2: CvtLo);
3658}
3659
3660SDValue AMDGPUTargetLowering::lowerINT_TO_FPImpl(SDValue Op, SelectionDAG &DAG,
3661 bool Signed) const {
3662 EVT DestVT = Op.getValueType();
3663 SDValue Src = Op.getOperand(i: 0);
3664 EVT SrcVT = Src.getValueType();
3665 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3666 unsigned CvtOpc = Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3667
3668 if (SrcVT == MVT::i16) {
3669 if (DestVT == MVT::f16)
3670 return Op;
3671
3672 SDLoc DL(Op);
3673 SDValue Ext = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i32, Operand: Src);
3674 return DAG.getNode(Opcode: CvtOpc, DL, VT: DestVT, Operand: Ext);
3675 }
3676
3677 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3678 return LowerINT_TO_FP16(Op, DAG, FP16Ty: DestVT);
3679
3680 if (SrcVT != MVT::i64)
3681 return Op;
3682
3683 if (DestVT == MVT::f32)
3684 return LowerINT_TO_FP32(Op, DAG, Signed);
3685
3686 assert(DestVT == MVT::f64);
3687 return LowerINT_TO_FP64(Op, DAG, Signed);
3688}
3689
3690SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
3691 SelectionDAG &DAG) const {
3692 return lowerINT_TO_FPImpl(Op, DAG, Signed: false);
3693}
3694
3695SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
3696 SelectionDAG &DAG) const {
3697 return lowerINT_TO_FPImpl(Op, DAG, Signed: true);
3698}
3699
3700SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
3701 bool Signed) const {
3702 SDLoc SL(Op);
3703
3704 SDValue Src = Op.getOperand(i: 0);
3705 EVT SrcVT = Src.getValueType();
3706
3707 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3708
3709 // The basic idea of converting a floating point number into a pair of 32-bit
3710 // integers is illustrated as follows:
3711 //
3712 // tf := trunc(val);
3713 // hif := floor(tf * 2^-32);
3714 // lof := tf - hif * 2^32; // lof is always positive due to floor.
3715 // hi := fptoi(hif);
3716 // lo := fptoi(lof);
3717 //
3718 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: SrcVT, Operand: Src);
3719 SDValue Sign;
3720 if (Signed && SrcVT == MVT::f32) {
3721 // However, a 32-bit floating point number has only 23 bits mantissa and
3722 // it's not enough to hold all the significant bits of `lof` if val is
3723 // negative. To avoid the loss of precision, We need to take the absolute
3724 // value after truncating and flip the result back based on the original
3725 // signedness.
3726 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i32,
3727 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Trunc),
3728 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3729 Trunc = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: SrcVT, Operand: Trunc);
3730 }
3731
3732 SDValue K0, K1;
3733 if (SrcVT == MVT::f64) {
3734 K0 = DAG.getConstantFP(
3735 Val: llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), DL: SL,
3736 VT: SrcVT);
3737 K1 = DAG.getConstantFP(
3738 Val: llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), DL: SL,
3739 VT: SrcVT);
3740 } else {
3741 K0 = DAG.getConstantFP(
3742 Val: llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), DL: SL, VT: SrcVT);
3743 K1 = DAG.getConstantFP(
3744 Val: llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), DL: SL, VT: SrcVT);
3745 }
3746 // TODO: Should this propagate fast-math-flags?
3747 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: SrcVT, N1: Trunc, N2: K0);
3748
3749 SDValue FloorMul = DAG.getNode(Opcode: ISD::FFLOOR, DL: SL, VT: SrcVT, Operand: Mul);
3750
3751 SDValue Fma = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: SrcVT, N1: FloorMul, N2: K1, N3: Trunc);
3752
3753 SDValue Hi = DAG.getNode(Opcode: (Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
3754 : ISD::FP_TO_UINT,
3755 DL: SL, VT: MVT::i32, Operand: FloorMul);
3756 SDValue Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SL, VT: MVT::i32, Operand: Fma);
3757
3758 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3759 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Lo, Hi}));
3760
3761 if (Signed && SrcVT == MVT::f32) {
3762 assert(Sign);
3763 // Flip the result based on the signedness, which is either all 0s or 1s.
3764 Sign = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3765 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Sign, Sign}));
3766 // r := xor(r, sign) - sign;
3767 Result =
3768 DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i64,
3769 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64, N1: Result, N2: Sign), N2: Sign);
3770 }
3771
3772 return Result;
3773}
3774
3775SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
3776 SDLoc DL(Op);
3777 SDValue N0 = Op.getOperand(i: 0);
3778
3779 // Convert to target node to get known bits
3780 if (N0.getValueType() == MVT::f32)
3781 return DAG.getNode(Opcode: AMDGPUISD::FP_TO_FP16, DL, VT: Op.getValueType(), Operand: N0);
3782
3783 if (Op->getFlags().hasApproximateFuncs()) {
3784 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
3785 return SDValue();
3786 }
3787
3788 return LowerF64ToF16Safe(Src: N0, DL, DAG);
3789}
3790
3791// return node in i32
3792SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
3793 SelectionDAG &DAG) const {
3794 assert(Src.getSimpleValueType() == MVT::f64);
3795
3796 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
3797 // TODO: We can generate better code for True16.
3798 const unsigned ExpMask = 0x7ff;
3799 const unsigned ExpBiasf64 = 1023;
3800 const unsigned ExpBiasf16 = 15;
3801 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3802 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
3803 SDValue U = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Src);
3804 SDValue UH = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: U,
3805 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3806 UH = DAG.getZExtOrTrunc(Op: UH, DL, VT: MVT::i32);
3807 U = DAG.getZExtOrTrunc(Op: U, DL, VT: MVT::i32);
3808 SDValue E = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3809 N2: DAG.getConstant(Val: 20, DL, VT: MVT::i64));
3810 E = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: E,
3811 N2: DAG.getConstant(Val: ExpMask, DL, VT: MVT::i32));
3812 // Subtract the fp64 exponent bias (1023) to get the real exponent and
3813 // add the f16 bias (15) to get the biased exponent for the f16 format.
3814 E = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: E,
3815 N2: DAG.getConstant(Val: -ExpBiasf64 + ExpBiasf16, DL, VT: MVT::i32));
3816
3817 SDValue M = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3818 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32));
3819 M = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M,
3820 N2: DAG.getConstant(Val: 0xffe, DL, VT: MVT::i32));
3821
3822 SDValue MaskedSig = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: UH,
3823 N2: DAG.getConstant(Val: 0x1ff, DL, VT: MVT::i32));
3824 MaskedSig = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: MaskedSig, N2: U);
3825
3826 SDValue Lo40Set = DAG.getSelectCC(DL, LHS: MaskedSig, RHS: Zero, True: Zero, False: One, Cond: ISD::SETEQ);
3827 M = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M, N2: Lo40Set);
3828
3829 // (M != 0 ? 0x0200 : 0) | 0x7c00;
3830 SDValue I = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
3831 N1: DAG.getSelectCC(DL, LHS: M, RHS: Zero, True: DAG.getConstant(Val: 0x0200, DL, VT: MVT::i32),
3832 False: Zero, Cond: ISD::SETNE), N2: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32));
3833
3834 // N = M | (E << 12);
3835 SDValue N = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3836 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: E,
3837 N2: DAG.getConstant(Val: 12, DL, VT: MVT::i32)));
3838
3839 // B = clamp(1-E, 0, 13);
3840 SDValue OneSubExp = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
3841 N1: One, N2: E);
3842 SDValue B = DAG.getNode(Opcode: ISD::SMAX, DL, VT: MVT::i32, N1: OneSubExp, N2: Zero);
3843 B = DAG.getNode(Opcode: ISD::SMIN, DL, VT: MVT::i32, N1: B,
3844 N2: DAG.getConstant(Val: 13, DL, VT: MVT::i32));
3845
3846 SDValue SigSetHigh = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3847 N2: DAG.getConstant(Val: 0x1000, DL, VT: MVT::i32));
3848
3849 SDValue D = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SigSetHigh, N2: B);
3850 SDValue D0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: D, N2: B);
3851 SDValue D1 = DAG.getSelectCC(DL, LHS: D0, RHS: SigSetHigh, True: One, False: Zero, Cond: ISD::SETNE);
3852 D = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: D, N2: D1);
3853
3854 SDValue V = DAG.getSelectCC(DL, LHS: E, RHS: One, True: D, False: N, Cond: ISD::SETLT);
3855 SDValue VLow3 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: V,
3856 N2: DAG.getConstant(Val: 0x7, DL, VT: MVT::i32));
3857 V = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: V,
3858 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3859 SDValue V0 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 3, DL, VT: MVT::i32),
3860 True: One, False: Zero, Cond: ISD::SETEQ);
3861 SDValue V1 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 5, DL, VT: MVT::i32),
3862 True: One, False: Zero, Cond: ISD::SETGT);
3863 V1 = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: V0, N2: V1);
3864 V = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: V, N2: V1);
3865
3866 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 30, DL, VT: MVT::i32),
3867 True: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32), False: V, Cond: ISD::SETGT);
3868 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 1039, DL, VT: MVT::i32),
3869 True: I, False: V, Cond: ISD::SETEQ);
3870
3871 // Extract the sign bit.
3872 SDValue Sign = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3873 N2: DAG.getConstant(Val: 16, DL, VT: MVT::i32));
3874 Sign = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Sign,
3875 N2: DAG.getConstant(Val: 0x8000, DL, VT: MVT::i32));
3876
3877 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Sign, N2: V);
3878}
3879
3880SDValue AMDGPUTargetLowering::LowerFP_TO_INT(const SDValue Op,
3881 SelectionDAG &DAG) const {
3882 SDValue Src = Op.getOperand(i: 0);
3883 unsigned OpOpcode = Op.getOpcode();
3884 EVT SrcVT = Src.getValueType();
3885 EVT DestVT = Op.getValueType();
3886
3887 // Will be selected natively
3888 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3889 return Op;
3890
3891 if (SrcVT == MVT::bf16 || (SrcVT == MVT::f16 && DestVT == MVT::i32)) {
3892 SDLoc DL(Op);
3893 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3894 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DestVT, Operand: PromotedSrc);
3895 }
3896
3897 // Promote i16 to i32
3898 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3899 SDLoc DL(Op);
3900
3901 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3902 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToInt32);
3903 }
3904
3905 if (DestVT != MVT::i64)
3906 return Op;
3907
3908 if (SrcVT == MVT::f16 ||
3909 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
3910 SDLoc DL(Op);
3911
3912 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3913 unsigned Ext =
3914 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3915 return DAG.getNode(Opcode: Ext, DL, VT: MVT::i64, Operand: FpToInt32);
3916 }
3917
3918 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3919 return LowerFP_TO_INT64(Op, DAG, Signed: OpOpcode == ISD::FP_TO_SINT);
3920
3921 return SDValue();
3922}
3923
3924SDValue AMDGPUTargetLowering::LowerFP_TO_INT_SAT(const SDValue Op,
3925 SelectionDAG &DAG) const {
3926 SDValue Src = Op.getOperand(i: 0);
3927 unsigned OpOpcode = Op.getOpcode();
3928 EVT SrcVT = Src.getValueType();
3929 EVT DstVT = Op.getValueType();
3930 SDValue SatVTOp = Op.getNode()->getOperand(Num: 1);
3931 EVT SatVT = cast<VTSDNode>(Val&: SatVTOp)->getVT();
3932 SDLoc DL(Op);
3933
3934 uint64_t DstWidth = DstVT.getScalarSizeInBits();
3935 uint64_t SatWidth = SatVT.getScalarSizeInBits();
3936 assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
3937
3938 // Scalar cases will be selected natively to v_cvt_/s_cvt_ instructions.
3939 // v2f32 -> v2i16 will be selected natively to v_cvt_pk_[iu]16_f32.
3940 if (SatWidth == DstWidth) {
3941 if ((DstVT == MVT::i32 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
3942 (DstVT == MVT::i16 && (SrcVT == MVT::f16 || SrcVT == MVT::f32)) ||
3943 (DstVT == MVT::v2i16 && SrcVT == MVT::v2f32))
3944 return Op;
3945 }
3946
3947 // Vectors can only be selected natively.
3948 if (DstVT.isVector())
3949 return SDValue();
3950
3951 // Perform all saturation at selected width (i16 or i32) and truncate
3952 if (SatWidth < DstWidth && SatWidth <= 32) {
3953 // For f16 conversion with sub-i16 saturation perform saturation
3954 // at i16, if available in the target. This removes the need for extra f16
3955 // to f32 conversion. For all the others use i32.
3956 MVT ResultVT =
3957 Subtarget->has16BitInsts() && SrcVT == MVT::f16 && SatWidth < 16
3958 ? MVT::i16
3959 : MVT::i32;
3960
3961 const SDValue ResultVTOp = DAG.getValueType(ResultVT);
3962 const uint64_t ResultWidth = ResultVT.getScalarSizeInBits();
3963
3964 // First, convert input float into selected integer (i16 or i32)
3965 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: ResultVT, N1: Src, N2: ResultVTOp);
3966 SDValue IntSatVal;
3967
3968 // Then, clamp at the saturation width using either i16 or i32 instructions
3969 if (OpOpcode == ISD::FP_TO_SINT_SAT) {
3970 SDValue MinConst = DAG.getConstant(
3971 Val: APInt::getSignedMaxValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3972 SDValue MaxConst = DAG.getConstant(
3973 Val: APInt::getSignedMinValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3974 SDValue MinVal = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3975 IntSatVal = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ResultVT, N1: MinVal, N2: MaxConst);
3976 } else {
3977 SDValue MinConst = DAG.getConstant(
3978 Val: APInt::getMaxValue(numBits: SatWidth).zext(width: ResultWidth), DL, VT: ResultVT);
3979 IntSatVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3980 }
3981
3982 // Finally, after saturating at i16 or i32 fit into the destination type
3983 return DAG.getExtOrTrunc(IsSigned: OpOpcode == ISD::FP_TO_SINT_SAT, Op: IntSatVal, DL,
3984 VT: DstVT);
3985 }
3986
3987 // SatWidth == DstWidth or SatWidth > 32
3988
3989 // Saturate at i32 for i64 dst and f16/bf16 src (will invoke f16 promotion
3990 // below)
3991 if (DstVT == MVT::i64 &&
3992 (SrcVT == MVT::f16 || SrcVT == MVT::bf16 ||
3993 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP))) {
3994 const SDValue Int32VTOp = DAG.getValueType(MVT::i32);
3995 return DAG.getNode(Opcode: OpOpcode, DL, VT: DstVT, N1: Src, N2: Int32VTOp);
3996 }
3997
3998 // Promote f16/bf16 src to f32 for i32 conversion
3999 if (DstVT == MVT::i32 && (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
4000 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
4001 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DstVT, N1: PromotedSrc, N2: SatVTOp);
4002 }
4003
4004 // For DstWidth < 16, promote i1 and i8 dst to i16 (if legal) with sub-i16
4005 // saturation. For DstWidth == 16, promote i16 dst to i32 with sub-i32
4006 // saturation; this covers i16.f32 and i16.f64
4007 if (DstWidth < 32) {
4008 // Note: this triggers SatWidth < DstWidth above to generate saturated
4009 // truncate by requesting MVT::i16/i32 destination with SatWidth < 16/32.
4010 MVT PromoteVT =
4011 (DstWidth < 16 && Subtarget->has16BitInsts()) ? MVT::i16 : MVT::i32;
4012 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: PromoteVT, N1: Src, N2: SatVTOp);
4013 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: FpToInt);
4014 }
4015
4016 // TODO: can we implement i64 dst for f32/f64?
4017
4018 return SDValue();
4019}
4020
4021SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
4022 SelectionDAG &DAG) const {
4023 EVT ExtraVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4024 MVT VT = Op.getSimpleValueType();
4025 MVT ScalarVT = VT.getScalarType();
4026
4027 assert(VT.isVector());
4028
4029 SDValue Src = Op.getOperand(i: 0);
4030 SDLoc DL(Op);
4031
4032 // TODO: Don't scalarize on Evergreen?
4033 unsigned NElts = VT.getVectorNumElements();
4034 SmallVector<SDValue, 8> Args;
4035 DAG.ExtractVectorElements(Op: Src, Args, Start: 0, Count: NElts);
4036
4037 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
4038 for (unsigned I = 0; I < NElts; ++I)
4039 Args[I] = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: ScalarVT, N1: Args[I], N2: VTOp);
4040
4041 return DAG.getBuildVector(VT, DL, Ops: Args);
4042}
4043
4044//===----------------------------------------------------------------------===//
4045// Custom DAG optimizations
4046//===----------------------------------------------------------------------===//
4047
4048static bool isU24(SDValue Op, SelectionDAG &DAG) {
4049 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
4050}
4051
4052static bool isI24(SDValue Op, SelectionDAG &DAG) {
4053 EVT VT = Op.getValueType();
4054 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
4055 // as unsigned 24-bit values.
4056 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24;
4057}
4058
4059static SDValue simplifyMul24(SDNode *Node24,
4060 TargetLowering::DAGCombinerInfo &DCI) {
4061 SelectionDAG &DAG = DCI.DAG;
4062 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4063 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
4064
4065 SDValue LHS = IsIntrin ? Node24->getOperand(Num: 1) : Node24->getOperand(Num: 0);
4066 SDValue RHS = IsIntrin ? Node24->getOperand(Num: 2) : Node24->getOperand(Num: 1);
4067 unsigned NewOpcode = Node24->getOpcode();
4068 if (IsIntrin) {
4069 unsigned IID = Node24->getConstantOperandVal(Num: 0);
4070 switch (IID) {
4071 case Intrinsic::amdgcn_mul_i24:
4072 NewOpcode = AMDGPUISD::MUL_I24;
4073 break;
4074 case Intrinsic::amdgcn_mul_u24:
4075 NewOpcode = AMDGPUISD::MUL_U24;
4076 break;
4077 case Intrinsic::amdgcn_mulhi_i24:
4078 NewOpcode = AMDGPUISD::MULHI_I24;
4079 break;
4080 case Intrinsic::amdgcn_mulhi_u24:
4081 NewOpcode = AMDGPUISD::MULHI_U24;
4082 break;
4083 default:
4084 llvm_unreachable("Expected 24-bit mul intrinsic");
4085 }
4086 }
4087
4088 APInt Demanded = APInt::getLowBitsSet(numBits: LHS.getValueSizeInBits(), loBitsSet: 24);
4089
4090 // First try to simplify using SimplifyMultipleUseDemandedBits which allows
4091 // the operands to have other uses, but will only perform simplifications that
4092 // involve bypassing some nodes for this user.
4093 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(Op: LHS, DemandedBits: Demanded, DAG);
4094 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(Op: RHS, DemandedBits: Demanded, DAG);
4095 if (DemandedLHS || DemandedRHS)
4096 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(Node24), VTList: Node24->getVTList(),
4097 N1: DemandedLHS ? DemandedLHS : LHS,
4098 N2: DemandedRHS ? DemandedRHS : RHS);
4099
4100 // Now try SimplifyDemandedBits which can simplify the nodes used by our
4101 // operands if this node is the only user.
4102 if (TLI.SimplifyDemandedBits(Op: LHS, DemandedBits: Demanded, DCI))
4103 return SDValue(Node24, 0);
4104 if (TLI.SimplifyDemandedBits(Op: RHS, DemandedBits: Demanded, DCI))
4105 return SDValue(Node24, 0);
4106
4107 return SDValue();
4108}
4109
4110template <typename IntTy>
4111static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
4112 uint32_t Width, const SDLoc &DL) {
4113 if (Width + Offset < 32) {
4114 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
4115 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
4116 if constexpr (std::is_signed_v<IntTy>) {
4117 return DAG.getSignedConstant(Val: Result, DL, VT: MVT::i32);
4118 } else {
4119 return DAG.getConstant(Result, DL, MVT::i32);
4120 }
4121 }
4122
4123 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
4124}
4125
4126static bool hasVolatileUser(SDNode *Val) {
4127 for (SDNode *U : Val->users()) {
4128 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: U)) {
4129 if (M->isVolatile())
4130 return true;
4131 }
4132 }
4133
4134 return false;
4135}
4136
4137bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
4138 // i32 vectors are the canonical memory type.
4139 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
4140 return false;
4141
4142 if (!VT.isByteSized())
4143 return false;
4144
4145 unsigned Size = VT.getStoreSize();
4146
4147 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
4148 return false;
4149
4150 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
4151 return false;
4152
4153 return true;
4154}
4155
4156// Replace load of an illegal type with a bitcast from a load of a friendlier
4157// type.
4158SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
4159 DAGCombinerInfo &DCI) const {
4160 if (!DCI.isBeforeLegalize())
4161 return SDValue();
4162
4163 LoadSDNode *LN = cast<LoadSDNode>(Val: N);
4164 if (!LN->isSimple() || !ISD::isNormalLoad(N: LN) || hasVolatileUser(Val: LN))
4165 return SDValue();
4166
4167 SDLoc SL(N);
4168 SelectionDAG &DAG = DCI.DAG;
4169 EVT VT = LN->getMemoryVT();
4170
4171 unsigned Size = VT.getStoreSize();
4172 Align Alignment = LN->getAlign();
4173 if (Alignment < Size && isTypeLegal(VT)) {
4174 unsigned IsFast;
4175 unsigned AS = LN->getAddressSpace();
4176
4177 // Expand unaligned loads earlier than legalization. Due to visitation order
4178 // problems during legalization, the emitted instructions to pack and unpack
4179 // the bytes again are not eliminated in the case of an unaligned copy.
4180 if (!allowsMisalignedMemoryAccesses(
4181 VT, AddrSpace: AS, Alignment, Flags: LN->getMemOperand()->getFlags(), &IsFast)) {
4182 if (VT.isVector())
4183 return SplitVectorLoad(Op: SDValue(LN, 0), DAG);
4184
4185 SDValue Ops[2];
4186 std::tie(args&: Ops[0], args&: Ops[1]) = expandUnalignedLoad(LD: LN, DAG);
4187
4188 return DAG.getMergeValues(Ops, dl: SDLoc(N));
4189 }
4190
4191 if (!IsFast)
4192 return SDValue();
4193 }
4194
4195 if (!shouldCombineMemoryType(VT))
4196 return SDValue();
4197
4198 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4199
4200 SDValue NewLoad
4201 = DAG.getLoad(VT: NewVT, dl: SL, Chain: LN->getChain(),
4202 Ptr: LN->getBasePtr(), MMO: LN->getMemOperand());
4203
4204 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewLoad);
4205 DCI.CombineTo(N, Res0: BC, Res1: NewLoad.getValue(R: 1));
4206 return SDValue(N, 0);
4207}
4208
4209// Replace store of an illegal type with a store of a bitcast to a friendlier
4210// type.
4211SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
4212 DAGCombinerInfo &DCI) const {
4213 if (!DCI.isBeforeLegalize())
4214 return SDValue();
4215
4216 StoreSDNode *SN = cast<StoreSDNode>(Val: N);
4217 if (!SN->isSimple() || !ISD::isNormalStore(N: SN))
4218 return SDValue();
4219
4220 EVT VT = SN->getMemoryVT();
4221 unsigned Size = VT.getStoreSize();
4222
4223 SDLoc SL(N);
4224 SelectionDAG &DAG = DCI.DAG;
4225 Align Alignment = SN->getAlign();
4226 if (Alignment < Size && isTypeLegal(VT)) {
4227 unsigned IsFast;
4228 unsigned AS = SN->getAddressSpace();
4229
4230 // Expand unaligned stores earlier than legalization. Due to visitation
4231 // order problems during legalization, the emitted instructions to pack and
4232 // unpack the bytes again are not eliminated in the case of an unaligned
4233 // copy.
4234 if (!allowsMisalignedMemoryAccesses(
4235 VT, AddrSpace: AS, Alignment, Flags: SN->getMemOperand()->getFlags(), &IsFast)) {
4236 if (VT.isVector())
4237 return SplitVectorStore(Op: SDValue(SN, 0), DAG);
4238
4239 return expandUnalignedStore(ST: SN, DAG);
4240 }
4241
4242 if (!IsFast)
4243 return SDValue();
4244 }
4245
4246 if (!shouldCombineMemoryType(VT))
4247 return SDValue();
4248
4249 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4250 SDValue Val = SN->getValue();
4251
4252 // DCI.AddToWorklist(Val.getNode());
4253
4254 bool OtherUses = !Val.hasOneUse();
4255 SDValue CastVal = DAG.getBitcast(VT: NewVT, V: Val);
4256 if (OtherUses) {
4257 SDValue CastBack = DAG.getBitcast(VT, V: CastVal);
4258 DAG.ReplaceAllUsesOfValueWith(From: Val, To: CastBack);
4259 }
4260
4261 return DAG.getStore(Chain: SN->getChain(), dl: SL, Val: CastVal,
4262 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
4263}
4264
4265// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
4266// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
4267// issues.
4268SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
4269 DAGCombinerInfo &DCI) const {
4270 SelectionDAG &DAG = DCI.DAG;
4271 SDValue N0 = N->getOperand(Num: 0);
4272
4273 // (vt2 (assertzext (truncate vt0:x), vt1)) ->
4274 // (vt2 (truncate (assertzext vt0:x, vt1)))
4275 if (N0.getOpcode() == ISD::TRUNCATE) {
4276 SDValue N1 = N->getOperand(Num: 1);
4277 EVT ExtVT = cast<VTSDNode>(Val&: N1)->getVT();
4278 SDLoc SL(N);
4279
4280 SDValue Src = N0.getOperand(i: 0);
4281 EVT SrcVT = Src.getValueType();
4282 if (SrcVT.bitsGE(VT: ExtVT)) {
4283 SDValue NewInReg = DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: SrcVT, N1: Src, N2: N1);
4284 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: N->getValueType(ResNo: 0), Operand: NewInReg);
4285 }
4286 }
4287
4288 return SDValue();
4289}
4290
4291SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
4292 SDNode *N, DAGCombinerInfo &DCI) const {
4293 unsigned IID = N->getConstantOperandVal(Num: 0);
4294 switch (IID) {
4295 case Intrinsic::amdgcn_mul_i24:
4296 case Intrinsic::amdgcn_mul_u24:
4297 case Intrinsic::amdgcn_mulhi_i24:
4298 case Intrinsic::amdgcn_mulhi_u24:
4299 return simplifyMul24(Node24: N, DCI);
4300 case Intrinsic::amdgcn_fract:
4301 case Intrinsic::amdgcn_rsq:
4302 case Intrinsic::amdgcn_rcp_legacy:
4303 case Intrinsic::amdgcn_rsq_legacy:
4304 case Intrinsic::amdgcn_rsq_clamp:
4305 case Intrinsic::amdgcn_tanh:
4306 case Intrinsic::amdgcn_prng_b32: {
4307 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
4308 SDValue Src = N->getOperand(Num: 1);
4309 return Src.isUndef() ? Src : SDValue();
4310 }
4311 case Intrinsic::amdgcn_frexp_exp: {
4312 // frexp_exp (fneg x) -> frexp_exp x
4313 // frexp_exp (fabs x) -> frexp_exp x
4314 // frexp_exp (fneg (fabs x)) -> frexp_exp x
4315 SDValue Src = N->getOperand(Num: 1);
4316 SDValue PeekSign = peekFPSignOps(Val: Src);
4317 if (PeekSign == Src)
4318 return SDValue();
4319 return SDValue(DCI.DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: PeekSign),
4320 0);
4321 }
4322 default:
4323 return SDValue();
4324 }
4325}
4326
4327/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
4328/// binary operation \p Opc to it with the corresponding constant operands.
4329SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
4330 DAGCombinerInfo &DCI, const SDLoc &SL,
4331 unsigned Opc, SDValue LHS,
4332 uint32_t ValLo, uint32_t ValHi) const {
4333 SelectionDAG &DAG = DCI.DAG;
4334 SDValue Lo, Hi;
4335 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: LHS, DAG);
4336
4337 SDValue LoRHS = DAG.getConstant(Val: ValLo, DL: SL, VT: MVT::i32);
4338 SDValue HiRHS = DAG.getConstant(Val: ValHi, DL: SL, VT: MVT::i32);
4339
4340 SDValue LoAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Lo, N2: LoRHS);
4341 SDValue HiAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Hi, N2: HiRHS);
4342
4343 // Re-visit the ands. It's possible we eliminated one of them and it could
4344 // simplify the vector.
4345 DCI.AddToWorklist(N: Lo.getNode());
4346 DCI.AddToWorklist(N: Hi.getNode());
4347
4348 SDValue Vec = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoAnd, HiAnd});
4349 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
4350}
4351
4352SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
4353 DAGCombinerInfo &DCI) const {
4354 EVT VT = N->getValueType(ResNo: 0);
4355 SDValue LHS = N->getOperand(Num: 0);
4356 SDValue RHS = N->getOperand(Num: 1);
4357 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4358 SDLoc SL(N);
4359 SelectionDAG &DAG = DCI.DAG;
4360
4361 unsigned RHSVal;
4362 if (CRHS) {
4363 RHSVal = CRHS->getZExtValue();
4364 if (!RHSVal)
4365 return LHS;
4366
4367 switch (LHS->getOpcode()) {
4368 default:
4369 break;
4370 case ISD::ZERO_EXTEND:
4371 case ISD::SIGN_EXTEND:
4372 case ISD::ANY_EXTEND: {
4373 SDValue X = LHS->getOperand(Num: 0);
4374
4375 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
4376 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16)) {
4377 // Prefer build_vector as the canonical form if packed types are legal.
4378 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
4379 SDValue Vec = DAG.getBuildVector(
4380 VT: MVT::v2i16, DL: SL,
4381 Ops: {DAG.getConstant(Val: 0, DL: SL, VT: MVT::i16), LHS->getOperand(Num: 0)});
4382 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Vec);
4383 }
4384
4385 // shl (ext x) => zext (shl x), if shift does not overflow int
4386 if (VT != MVT::i64)
4387 break;
4388 KnownBits Known = DAG.computeKnownBits(Op: X);
4389 unsigned LZ = Known.countMinLeadingZeros();
4390 if (LZ < RHSVal)
4391 break;
4392 EVT XVT = X.getValueType();
4393 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: XVT, N1: X, N2: SDValue(CRHS, 0));
4394 return DAG.getZExtOrTrunc(Op: Shl, DL: SL, VT);
4395 }
4396 }
4397 }
4398
4399 if (VT.getScalarType() != MVT::i64)
4400 return SDValue();
4401
4402 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4403 // common case, splitting this into a move and a 32-bit shift is faster and
4404 // the same code size.
4405 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4406
4407 EVT ElementType = VT.getScalarType();
4408 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4409 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4410
4411 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4412 return SDValue();
4413 SDValue ShiftAmt;
4414
4415 if (CRHS) {
4416 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4417 VT: TargetType);
4418 } else {
4419 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4420 const SDValue ShiftMask =
4421 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4422 // This AND instruction will clamp out of bounds shift values.
4423 // It will also be removed during later instruction selection.
4424 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4425 }
4426
4427 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: LHS);
4428 SDValue NewShift =
4429 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: TargetType, N1: Lo, N2: ShiftAmt, Flags: N->getFlags());
4430
4431 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4432 SDValue Vec;
4433
4434 if (VT.isVector()) {
4435 EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4436 unsigned NElts = TargetType.getVectorNumElements();
4437 SmallVector<SDValue, 8> HiOps;
4438 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4439
4440 DAG.ExtractVectorElements(Op: NewShift, Args&: HiOps, Start: 0, Count: NElts);
4441 for (unsigned I = 0; I != NElts; ++I)
4442 HiAndLoOps[2 * I + 1] = HiOps[I];
4443 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4444 } else {
4445 EVT ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4446 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {Zero, NewShift});
4447 }
4448 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4449}
4450
4451SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
4452 DAGCombinerInfo &DCI) const {
4453 SDValue RHS = N->getOperand(Num: 1);
4454 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4455 EVT VT = N->getValueType(ResNo: 0);
4456 SDValue LHS = N->getOperand(Num: 0);
4457 SelectionDAG &DAG = DCI.DAG;
4458 SDLoc SL(N);
4459
4460 if (VT.getScalarType() != MVT::i64)
4461 return SDValue();
4462
4463 // For C >= 32
4464 // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
4465
4466 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4467 // common case, splitting this into a move and a 32-bit shift is faster and
4468 // the same code size.
4469 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4470
4471 EVT ElementType = VT.getScalarType();
4472 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4473 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4474
4475 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4476 return SDValue();
4477
4478 SDValue ShiftFullAmt =
4479 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4480 SDValue ShiftAmt;
4481 if (CRHS) {
4482 unsigned RHSVal = CRHS->getZExtValue();
4483 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4484 VT: TargetType);
4485 } else if (Known.getMinValue().getZExtValue() ==
4486 (ElementType.getSizeInBits() - 1)) {
4487 ShiftAmt = ShiftFullAmt;
4488 } else {
4489 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4490 const SDValue ShiftMask =
4491 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4492 // This AND instruction will clamp out of bounds shift values.
4493 // It will also be removed during later instruction selection.
4494 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4495 }
4496
4497 EVT ConcatType;
4498 SDValue Hi;
4499 SDLoc LHSSL(LHS);
4500 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4501 if (VT.isVector()) {
4502 unsigned NElts = TargetType.getVectorNumElements();
4503 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4504 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4505 SmallVector<SDValue, 8> HiOps(NElts);
4506 SmallVector<SDValue, 16> HiAndLoOps;
4507
4508 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, Start: 0, Count: NElts * 2);
4509 for (unsigned I = 0; I != NElts; ++I) {
4510 HiOps[I] = HiAndLoOps[2 * I + 1];
4511 }
4512 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4513 } else {
4514 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4515 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4516 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4517 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4518 }
4519
4520 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
4521 SDValue NewShift, HiShift;
4522 if (KnownLHS.isNegative()) {
4523 HiShift = DAG.getAllOnesConstant(DL: SL, VT: TargetType);
4524 NewShift =
4525 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4526 } else if (CRHS &&
4527 CRHS->getZExtValue() == (ElementType.getSizeInBits() - 1)) {
4528 NewShift = HiShift =
4529 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4530 } else {
4531 Hi = DAG.getFreeze(V: Hi);
4532 HiShift = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftFullAmt);
4533 NewShift =
4534 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4535 }
4536
4537 SDValue Vec;
4538 if (VT.isVector()) {
4539 unsigned NElts = TargetType.getVectorNumElements();
4540 SmallVector<SDValue, 8> HiOps;
4541 SmallVector<SDValue, 8> LoOps;
4542 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
4543
4544 DAG.ExtractVectorElements(Op: HiShift, Args&: HiOps, Start: 0, Count: NElts);
4545 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4546 for (unsigned I = 0; I != NElts; ++I) {
4547 HiAndLoOps[2 * I + 1] = HiOps[I];
4548 HiAndLoOps[2 * I] = LoOps[I];
4549 }
4550 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4551 } else {
4552 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, HiShift});
4553 }
4554 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4555}
4556
4557SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
4558 DAGCombinerInfo &DCI) const {
4559 SDValue RHS = N->getOperand(Num: 1);
4560 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4561 EVT VT = N->getValueType(ResNo: 0);
4562 SDValue LHS = N->getOperand(Num: 0);
4563 SelectionDAG &DAG = DCI.DAG;
4564 SDLoc SL(N);
4565 unsigned RHSVal;
4566
4567 if (CRHS) {
4568 RHSVal = CRHS->getZExtValue();
4569
4570 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
4571 // this improves the ability to match BFE patterns in isel.
4572 if (LHS.getOpcode() == ISD::AND) {
4573 if (auto *Mask = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
4574 unsigned MaskIdx, MaskLen;
4575 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4576 MaskIdx == RHSVal) {
4577 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT,
4578 N1: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 0),
4579 N2: N->getOperand(Num: 1)),
4580 N2: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 1),
4581 N2: N->getOperand(Num: 1)));
4582 }
4583 }
4584 }
4585 }
4586
4587 if (VT.getScalarType() != MVT::i64)
4588 return SDValue();
4589
4590 // for C >= 32
4591 // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
4592
4593 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4594 // common case, splitting this into a move and a 32-bit shift is faster and
4595 // the same code size.
4596 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4597
4598 EVT ElementType = VT.getScalarType();
4599 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4600 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4601
4602 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4603 return SDValue();
4604
4605 SDValue ShiftAmt;
4606 if (CRHS) {
4607 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4608 VT: TargetType);
4609 } else {
4610 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4611 const SDValue ShiftMask =
4612 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4613 // This AND instruction will clamp out of bounds shift values.
4614 // It will also be removed during later instruction selection.
4615 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4616 }
4617
4618 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4619 EVT ConcatType;
4620 SDValue Hi;
4621 SDLoc LHSSL(LHS);
4622 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4623 if (VT.isVector()) {
4624 unsigned NElts = TargetType.getVectorNumElements();
4625 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4626 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4627 SmallVector<SDValue, 8> HiOps(NElts);
4628 SmallVector<SDValue, 16> HiAndLoOps;
4629
4630 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, /*Start=*/0, Count: NElts * 2);
4631 for (unsigned I = 0; I != NElts; ++I)
4632 HiOps[I] = HiAndLoOps[2 * I + 1];
4633 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4634 } else {
4635 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4636 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4637 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4638 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4639 }
4640
4641 SDValue NewShift =
4642 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4643
4644 SDValue Vec;
4645 if (VT.isVector()) {
4646 unsigned NElts = TargetType.getVectorNumElements();
4647 SmallVector<SDValue, 8> LoOps;
4648 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4649
4650 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4651 for (unsigned I = 0; I != NElts; ++I)
4652 HiAndLoOps[2 * I] = LoOps[I];
4653 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4654 } else {
4655 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, Zero});
4656 }
4657 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4658}
4659
4660SDValue AMDGPUTargetLowering::performTruncateCombine(
4661 SDNode *N, DAGCombinerInfo &DCI) const {
4662 SDLoc SL(N);
4663 SelectionDAG &DAG = DCI.DAG;
4664 EVT VT = N->getValueType(ResNo: 0);
4665 SDValue Src = N->getOperand(Num: 0);
4666
4667 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
4668 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
4669 SDValue Vec = Src.getOperand(i: 0);
4670 if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
4671 SDValue Elt0 = Vec.getOperand(i: 0);
4672 EVT EltVT = Elt0.getValueType();
4673 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
4674 if (EltVT.isFloatingPoint()) {
4675 Elt0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL,
4676 VT: EltVT.changeTypeToInteger(), Operand: Elt0);
4677 }
4678
4679 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Elt0);
4680 }
4681 }
4682 }
4683
4684 // Equivalent of above for accessing the high element of a vector as an
4685 // integer operation.
4686 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
4687 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
4688 if (auto *K = isConstOrConstSplat(N: Src.getOperand(i: 1))) {
4689 SDValue BV = stripBitcast(Val: Src.getOperand(i: 0));
4690 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
4691 EVT SrcEltVT = BV.getOperand(i: 0).getValueType();
4692 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
4693 unsigned BitIndex = K->getZExtValue();
4694 unsigned PartIndex = BitIndex / SrcEltSize;
4695
4696 if (PartIndex * SrcEltSize == BitIndex &&
4697 PartIndex < BV.getNumOperands()) {
4698 if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
4699 SDValue SrcElt =
4700 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcEltVT.changeTypeToInteger(),
4701 Operand: BV.getOperand(i: PartIndex));
4702 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: SrcElt);
4703 }
4704 }
4705 }
4706 }
4707 }
4708
4709 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
4710 //
4711 // i16 (trunc (srl i64:x, K)), K <= 16 ->
4712 // i16 (trunc (srl (i32 (trunc x), K)))
4713 if (VT.getScalarSizeInBits() < 32) {
4714 EVT SrcVT = Src.getValueType();
4715 if (SrcVT.getScalarSizeInBits() > 32 &&
4716 (Src.getOpcode() == ISD::SRL ||
4717 Src.getOpcode() == ISD::SRA ||
4718 Src.getOpcode() == ISD::SHL)) {
4719 SDValue Amt = Src.getOperand(i: 1);
4720 KnownBits Known = DAG.computeKnownBits(Op: Amt);
4721
4722 // - For left shifts, do the transform as long as the shift
4723 // amount is still legal for i32, so when ShiftAmt < 32 (<= 31)
4724 // - For right shift, do it if ShiftAmt <= (32 - Size) to avoid
4725 // losing information stored in the high bits when truncating.
4726 const unsigned MaxCstSize =
4727 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
4728 if (Known.getMaxValue().ule(RHS: MaxCstSize)) {
4729 EVT MidVT = VT.isVector() ?
4730 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
4731 NumElements: VT.getVectorNumElements()) : MVT::i32;
4732
4733 EVT NewShiftVT = getShiftAmountTy(LHSTy: MidVT, DL: DAG.getDataLayout());
4734 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MidVT,
4735 Operand: Src.getOperand(i: 0));
4736 DCI.AddToWorklist(N: Trunc.getNode());
4737
4738 if (Amt.getValueType() != NewShiftVT) {
4739 Amt = DAG.getZExtOrTrunc(Op: Amt, DL: SL, VT: NewShiftVT);
4740 DCI.AddToWorklist(N: Amt.getNode());
4741 }
4742
4743 SDValue ShrunkShift = DAG.getNode(Opcode: Src.getOpcode(), DL: SL, VT: MidVT,
4744 N1: Trunc, N2: Amt);
4745 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: ShrunkShift);
4746 }
4747 }
4748 }
4749
4750 return SDValue();
4751}
4752
4753// We need to specifically handle i64 mul here to avoid unnecessary conversion
4754// instructions. If we only match on the legalized i64 mul expansion,
4755// SimplifyDemandedBits will be unable to remove them because there will be
4756// multiple uses due to the separate mul + mulh[su].
4757static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
4758 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
4759 if (Size <= 32) {
4760 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4761 return DAG.getNode(Opcode: MulOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4762 }
4763
4764 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4765 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
4766
4767 SDValue MulLo = DAG.getNode(Opcode: MulLoOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4768 SDValue MulHi = DAG.getNode(Opcode: MulHiOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4769
4770 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: MulLo, N2: MulHi);
4771}
4772
4773/// If \p V is an add of a constant 1, returns the other operand. Otherwise
4774/// return SDValue().
4775static SDValue getAddOneOp(const SDNode *V) {
4776 if (V->getOpcode() != ISD::ADD)
4777 return SDValue();
4778
4779 return isOneConstant(V: V->getOperand(Num: 1)) ? V->getOperand(Num: 0) : SDValue();
4780}
4781
4782SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
4783 DAGCombinerInfo &DCI) const {
4784 assert(N->getOpcode() == ISD::MUL);
4785 EVT VT = N->getValueType(ResNo: 0);
4786
4787 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4788 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4789 // unnecessarily). isDivergent() is used as an approximation of whether the
4790 // value is in an SGPR.
4791 if (!N->isDivergent())
4792 return SDValue();
4793
4794 unsigned Size = VT.getSizeInBits();
4795 if (VT.isVector() || Size > 64)
4796 return SDValue();
4797
4798 SelectionDAG &DAG = DCI.DAG;
4799 SDLoc DL(N);
4800
4801 SDValue N0 = N->getOperand(Num: 0);
4802 SDValue N1 = N->getOperand(Num: 1);
4803
4804 // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
4805 // matching.
4806
4807 // mul x, (add y, 1) -> add (mul x, y), x
4808 auto IsFoldableAdd = [](SDValue V) -> SDValue {
4809 SDValue AddOp = getAddOneOp(V: V.getNode());
4810 if (!AddOp)
4811 return SDValue();
4812
4813 if (V.hasOneUse() || all_of(Range: V->users(), P: [](const SDNode *U) -> bool {
4814 return U->getOpcode() == ISD::MUL;
4815 }))
4816 return AddOp;
4817
4818 return SDValue();
4819 };
4820
4821 // FIXME: The selection pattern is not properly checking for commuted
4822 // operands, so we have to place the mul in the LHS
4823 if (SDValue MulOper = IsFoldableAdd(N0)) {
4824 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1, N2: MulOper);
4825 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N1);
4826 }
4827
4828 if (SDValue MulOper = IsFoldableAdd(N1)) {
4829 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N0, N2: MulOper);
4830 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N0);
4831 }
4832
4833 // There are i16 integer mul/mad.
4834 if (isTypeLegal(VT: MVT::i16) && VT.getScalarType().bitsLE(VT: MVT::i16))
4835 return SDValue();
4836
4837 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4838 // in the source into any_extends if the result of the mul is truncated. Since
4839 // we can assume the high bits are whatever we want, use the underlying value
4840 // to avoid the unknown high bits from interfering.
4841 if (N0.getOpcode() == ISD::ANY_EXTEND)
4842 N0 = N0.getOperand(i: 0);
4843
4844 if (N1.getOpcode() == ISD::ANY_EXTEND)
4845 N1 = N1.getOperand(i: 0);
4846
4847 SDValue Mul;
4848
4849 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4850 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4851 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4852 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: false);
4853 } else if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4854 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4855 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4856 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: true);
4857 } else {
4858 return SDValue();
4859 }
4860
4861 // We need to use sext even for MUL_U24, because MUL_U24 is used
4862 // for signed multiply of 8 and 16-bit types.
4863 return DAG.getSExtOrTrunc(Op: Mul, DL, VT);
4864}
4865
4866SDValue
4867AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N,
4868 DAGCombinerInfo &DCI) const {
4869 if (N->getValueType(ResNo: 0) != MVT::i32)
4870 return SDValue();
4871
4872 SelectionDAG &DAG = DCI.DAG;
4873 SDLoc DL(N);
4874
4875 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
4876 SDValue N0 = N->getOperand(Num: 0);
4877 SDValue N1 = N->getOperand(Num: 1);
4878
4879 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4880 // in the source into any_extends if the result of the mul is truncated. Since
4881 // we can assume the high bits are whatever we want, use the underlying value
4882 // to avoid the unknown high bits from interfering.
4883 if (N0.getOpcode() == ISD::ANY_EXTEND)
4884 N0 = N0.getOperand(i: 0);
4885 if (N1.getOpcode() == ISD::ANY_EXTEND)
4886 N1 = N1.getOperand(i: 0);
4887
4888 // Try to use two fast 24-bit multiplies (one for each half of the result)
4889 // instead of one slow extending multiply.
4890 unsigned LoOpcode = 0;
4891 unsigned HiOpcode = 0;
4892 if (Signed) {
4893 if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4894 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4895 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4896 LoOpcode = AMDGPUISD::MUL_I24;
4897 HiOpcode = AMDGPUISD::MULHI_I24;
4898 }
4899 } else {
4900 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4901 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4902 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4903 LoOpcode = AMDGPUISD::MUL_U24;
4904 HiOpcode = AMDGPUISD::MULHI_U24;
4905 }
4906 }
4907 if (!LoOpcode)
4908 return SDValue();
4909
4910 SDValue Lo = DAG.getNode(Opcode: LoOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4911 SDValue Hi = DAG.getNode(Opcode: HiOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4912 DCI.CombineTo(N, Res0: Lo, Res1: Hi);
4913 return SDValue(N, 0);
4914}
4915
4916SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
4917 DAGCombinerInfo &DCI) const {
4918 EVT VT = N->getValueType(ResNo: 0);
4919
4920 if (!Subtarget->hasMulI24() || VT.isVector())
4921 return SDValue();
4922
4923 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4924 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4925 // unnecessarily). isDivergent() is used as an approximation of whether the
4926 // value is in an SGPR.
4927 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4928 // valu op anyway)
4929 if (Subtarget->hasSMulHi() && !N->isDivergent())
4930 return SDValue();
4931
4932 SelectionDAG &DAG = DCI.DAG;
4933 SDLoc DL(N);
4934
4935 SDValue N0 = N->getOperand(Num: 0);
4936 SDValue N1 = N->getOperand(Num: 1);
4937
4938 if (!isI24(Op: N0, DAG) || !isI24(Op: N1, DAG))
4939 return SDValue();
4940
4941 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4942 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4943
4944 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_I24, DL, VT: MVT::i32, N1: N0, N2: N1);
4945 DCI.AddToWorklist(N: Mulhi.getNode());
4946 return DAG.getSExtOrTrunc(Op: Mulhi, DL, VT);
4947}
4948
4949SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
4950 DAGCombinerInfo &DCI) const {
4951 EVT VT = N->getValueType(ResNo: 0);
4952
4953 if (VT.isVector() || VT.getSizeInBits() > 32 || !Subtarget->hasMulU24())
4954 return SDValue();
4955
4956 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4957 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4958 // unnecessarily). isDivergent() is used as an approximation of whether the
4959 // value is in an SGPR.
4960 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4961 // valu op anyway)
4962 if (!N->isDivergent() && Subtarget->hasSMulHi())
4963 return SDValue();
4964
4965 SelectionDAG &DAG = DCI.DAG;
4966 SDLoc DL(N);
4967
4968 SDValue N0 = N->getOperand(Num: 0);
4969 SDValue N1 = N->getOperand(Num: 1);
4970
4971 if (!isU24(Op: N0, DAG) || !isU24(Op: N1, DAG))
4972 return SDValue();
4973
4974 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4975 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4976
4977 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_U24, DL, VT: MVT::i32, N1: N0, N2: N1);
4978 DCI.AddToWorklist(N: Mulhi.getNode());
4979 return DAG.getZExtOrTrunc(Op: Mulhi, DL, VT);
4980}
4981
4982SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
4983 SDValue Op,
4984 const SDLoc &DL,
4985 unsigned Opc) const {
4986 EVT VT = Op.getValueType();
4987 if (VT.bitsGT(VT: MVT::i32))
4988 return SDValue();
4989
4990 if (VT != MVT::i32)
4991 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: Op);
4992
4993 SDValue FFBX = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, Operand: Op);
4994 if (VT != MVT::i32)
4995 FFBX = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FFBX);
4996
4997 return FFBX;
4998}
4999
5000// The native instructions return -1 on 0 input. Optimize out a select that
5001// produces -1 on 0.
5002//
5003// TODO: If zero is not undef, we could also do this if the output is compared
5004// against the bitwidth.
5005//
5006// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
5007SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
5008 SDValue LHS, SDValue RHS,
5009 DAGCombinerInfo &DCI) const {
5010 if (!isNullConstant(V: Cond.getOperand(i: 1)))
5011 return SDValue();
5012
5013 SelectionDAG &DAG = DCI.DAG;
5014 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
5015 SDValue CmpLHS = Cond.getOperand(i: 0);
5016
5017 // select (setcc x, 0, eq), -1, (ctlz_zero_poison x) -> ffbh_u32 x
5018 // select (setcc x, 0, eq), -1, (cttz_zero_poison x) -> ffbl_u32 x
5019 if (CCOpcode == ISD::SETEQ &&
5020 (isCtlzOpc(Opc: RHS.getOpcode()) || isCttzOpc(Opc: RHS.getOpcode())) &&
5021 RHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: LHS)) {
5022 unsigned Opc =
5023 isCttzOpc(Opc: RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
5024 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
5025 }
5026
5027 // select (setcc x, 0, ne), (ctlz_zero_poison x), -1 -> ffbh_u32 x
5028 // select (setcc x, 0, ne), (cttz_zero_poison x), -1 -> ffbl_u32 x
5029 if (CCOpcode == ISD::SETNE &&
5030 (isCtlzOpc(Opc: LHS.getOpcode()) || isCttzOpc(Opc: LHS.getOpcode())) &&
5031 LHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: RHS)) {
5032 unsigned Opc =
5033 isCttzOpc(Opc: LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
5034
5035 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
5036 }
5037
5038 return SDValue();
5039}
5040
5041static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
5042 unsigned Op,
5043 const SDLoc &SL,
5044 SDValue Cond,
5045 SDValue N1,
5046 SDValue N2) {
5047 SelectionDAG &DAG = DCI.DAG;
5048 EVT VT = N1.getValueType();
5049
5050 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cond,
5051 N2: N1.getOperand(i: 0), N3: N2.getOperand(i: 0));
5052 DCI.AddToWorklist(N: NewSelect.getNode());
5053 return DAG.getNode(Opcode: Op, DL: SL, VT, Operand: NewSelect);
5054}
5055
5056// Pull a free FP operation out of a select so it may fold into uses.
5057//
5058// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
5059// select c, (fneg x), k -> fneg (select c, x, (fneg k))
5060//
5061// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
5062// select c, (fabs x), +k -> fabs (select c, x, k)
5063SDValue
5064AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
5065 SDValue N) const {
5066 SelectionDAG &DAG = DCI.DAG;
5067 SDValue Cond = N.getOperand(i: 0);
5068 SDValue LHS = N.getOperand(i: 1);
5069 SDValue RHS = N.getOperand(i: 2);
5070
5071 EVT VT = N.getValueType();
5072 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
5073 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
5074 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5075 return SDValue();
5076
5077 return distributeOpThroughSelect(DCI, Op: LHS.getOpcode(),
5078 SL: SDLoc(N), Cond, N1: LHS, N2: RHS);
5079 }
5080
5081 bool Inv = false;
5082 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
5083 std::swap(a&: LHS, b&: RHS);
5084 Inv = true;
5085 }
5086
5087 // TODO: Support vector constants.
5088 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
5089 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS &&
5090 !selectSupportsSourceMods(N: N.getNode())) {
5091 SDLoc SL(N);
5092 // If one side is an fneg/fabs and the other is a constant, we can push the
5093 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
5094 SDValue NewLHS = LHS.getOperand(i: 0);
5095 SDValue NewRHS = RHS;
5096
5097 // Careful: if the neg can be folded up, don't try to pull it back down.
5098 bool ShouldFoldNeg = true;
5099
5100 if (NewLHS.hasOneUse()) {
5101 unsigned Opc = NewLHS.getOpcode();
5102 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(N: NewLHS.getNode()))
5103 ShouldFoldNeg = false;
5104 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
5105 ShouldFoldNeg = false;
5106 }
5107
5108 if (ShouldFoldNeg) {
5109 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative())
5110 return SDValue();
5111
5112 // We're going to be forced to use a source modifier anyway, there's no
5113 // point to pulling the negate out unless we can get a size reduction by
5114 // negating the constant.
5115 //
5116 // TODO: Generalize to use getCheaperNegatedExpression which doesn't know
5117 // about cheaper constants.
5118 if (NewLHS.getOpcode() == ISD::FABS &&
5119 getConstantNegateCost(C: CRHS) != NegatibleCost::Cheaper)
5120 return SDValue();
5121
5122 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5123 return SDValue();
5124
5125 if (LHS.getOpcode() == ISD::FNEG)
5126 NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5127
5128 if (Inv)
5129 std::swap(a&: NewLHS, b&: NewRHS);
5130
5131 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT,
5132 N1: Cond, N2: NewLHS, N3: NewRHS);
5133 DCI.AddToWorklist(N: NewSelect.getNode());
5134 return DAG.getNode(Opcode: LHS.getOpcode(), DL: SL, VT, Operand: NewSelect);
5135 }
5136 }
5137
5138 return SDValue();
5139}
5140
5141SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
5142 DAGCombinerInfo &DCI) const {
5143 if (SDValue Folded = foldFreeOpFromSelect(DCI, N: SDValue(N, 0)))
5144 return Folded;
5145
5146 SDValue Cond = N->getOperand(Num: 0);
5147 if (Cond.getOpcode() != ISD::SETCC)
5148 return SDValue();
5149
5150 EVT VT = N->getValueType(ResNo: 0);
5151 SDValue LHS = Cond.getOperand(i: 0);
5152 SDValue RHS = Cond.getOperand(i: 1);
5153 SDValue CC = Cond.getOperand(i: 2);
5154
5155 SDValue True = N->getOperand(Num: 1);
5156 SDValue False = N->getOperand(Num: 2);
5157
5158 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
5159 SelectionDAG &DAG = DCI.DAG;
5160 if (DAG.isConstantValueOfAnyType(N: True) &&
5161 !DAG.isConstantValueOfAnyType(N: False)) {
5162 // Swap cmp + select pair to move constant to false input.
5163 // This will allow using VOPC cndmasks more often.
5164 // select (setcc x, y), k, x -> select (setccinv x, y), x, k
5165
5166 SDLoc SL(N);
5167 ISD::CondCode NewCC =
5168 getSetCCInverse(Operation: cast<CondCodeSDNode>(Val&: CC)->get(), Type: LHS.getValueType());
5169
5170 SDValue NewCond = DAG.getSetCC(DL: SL, VT: Cond.getValueType(), LHS, RHS, Cond: NewCC);
5171 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NewCond, N2: False, N3: True);
5172 }
5173
5174 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
5175 SDValue MinMax = combineFMinMaxLegacy(DL: SDLoc(N), VT, LHS, RHS, True, False,
5176 CC, Flags: N->getFlags(), DCI);
5177 // Revisit this node so we can catch min3/max3/med3 patterns.
5178 //DCI.AddToWorklist(MinMax.getNode());
5179 return MinMax;
5180 }
5181 }
5182
5183 // There's no reason to not do this if the condition has other uses.
5184 return performCtlz_CttzCombine(SL: SDLoc(N), Cond, LHS: True, RHS: False, DCI);
5185}
5186
5187static bool isInv2Pi(const APFloat &APF) {
5188 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
5189 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
5190 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
5191
5192 return APF.bitwiseIsEqual(RHS: KF16) ||
5193 APF.bitwiseIsEqual(RHS: KF32) ||
5194 APF.bitwiseIsEqual(RHS: KF64);
5195}
5196
5197// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
5198// additional cost to negate them.
5199TargetLowering::NegatibleCost
5200AMDGPUTargetLowering::getConstantNegateCost(const ConstantFPSDNode *C) const {
5201 if (C->isZero())
5202 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5203
5204 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(APF: C->getValueAPF()))
5205 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5206
5207 return NegatibleCost::Neutral;
5208}
5209
5210bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
5211 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5212 return getConstantNegateCost(C) == NegatibleCost::Expensive;
5213 return false;
5214}
5215
5216bool AMDGPUTargetLowering::isConstantCheaperToNegate(SDValue N) const {
5217 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5218 return getConstantNegateCost(C) == NegatibleCost::Cheaper;
5219 return false;
5220}
5221
5222static unsigned inverseMinMax(unsigned Opc) {
5223 switch (Opc) {
5224 case ISD::FMAXNUM:
5225 return ISD::FMINNUM;
5226 case ISD::FMINNUM:
5227 return ISD::FMAXNUM;
5228 case ISD::FMAXNUM_IEEE:
5229 return ISD::FMINNUM_IEEE;
5230 case ISD::FMINNUM_IEEE:
5231 return ISD::FMAXNUM_IEEE;
5232 case ISD::FMAXIMUM:
5233 return ISD::FMINIMUM;
5234 case ISD::FMINIMUM:
5235 return ISD::FMAXIMUM;
5236 case ISD::FMAXIMUMNUM:
5237 return ISD::FMINIMUMNUM;
5238 case ISD::FMINIMUMNUM:
5239 return ISD::FMAXIMUMNUM;
5240 case AMDGPUISD::FMAX_LEGACY:
5241 return AMDGPUISD::FMIN_LEGACY;
5242 case AMDGPUISD::FMIN_LEGACY:
5243 return AMDGPUISD::FMAX_LEGACY;
5244 default:
5245 llvm_unreachable("invalid min/max opcode");
5246 }
5247}
5248
5249/// \return true if it's profitable to try to push an fneg into its source
5250/// instruction.
5251bool AMDGPUTargetLowering::shouldFoldFNegIntoSrc(SDNode *N, SDValue N0) {
5252 // If the input has multiple uses and we can either fold the negate down, or
5253 // the other uses cannot, give up. This both prevents unprofitable
5254 // transformations and infinite loops: we won't repeatedly try to fold around
5255 // a negate that has no 'good' form.
5256 if (N0.hasOneUse()) {
5257 // This may be able to fold into the source, but at a code size cost. Don't
5258 // fold if the fold into the user is free.
5259 if (allUsesHaveSourceMods(N, CostThreshold: 0))
5260 return false;
5261 } else {
5262 if (fnegFoldsIntoOp(N: N0.getNode()) &&
5263 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N: N0.getNode())))
5264 return false;
5265 }
5266
5267 return true;
5268}
5269
5270SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
5271 DAGCombinerInfo &DCI) const {
5272 SelectionDAG &DAG = DCI.DAG;
5273 SDValue N0 = N->getOperand(Num: 0);
5274 EVT VT = N->getValueType(ResNo: 0);
5275
5276 unsigned Opc = N0.getOpcode();
5277
5278 if (!shouldFoldFNegIntoSrc(N, N0))
5279 return SDValue();
5280
5281 bool MayIgnoreSignedZeroForAllUses =
5282 N0->getFlags().hasNoSignedZeros() ||
5283 (N0.hasOneUse() && N->getFlags().hasNoSignedZeros());
5284
5285 SDLoc SL(N);
5286 switch (Opc) {
5287 case ISD::FADD: {
5288 if (!MayIgnoreSignedZeroForAllUses)
5289 return SDValue();
5290
5291 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
5292 SDValue LHS = N0.getOperand(i: 0);
5293 SDValue RHS = N0.getOperand(i: 1);
5294
5295 if (LHS.getOpcode() != ISD::FNEG)
5296 LHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5297 else
5298 LHS = LHS.getOperand(i: 0);
5299
5300 if (RHS.getOpcode() != ISD::FNEG)
5301 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5302 else
5303 RHS = RHS.getOperand(i: 0);
5304
5305 SDValue Res = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5306 if (Res.getOpcode() != ISD::FADD)
5307 return SDValue(); // Op got folded away.
5308 if (!N0.hasOneUse())
5309 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5310 return Res;
5311 }
5312 case ISD::FMUL:
5313 case AMDGPUISD::FMUL_LEGACY: {
5314 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
5315 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
5316 SDValue LHS = N0.getOperand(i: 0);
5317 SDValue RHS = N0.getOperand(i: 1);
5318
5319 if (LHS.getOpcode() == ISD::FNEG)
5320 LHS = LHS.getOperand(i: 0);
5321 else if (RHS.getOpcode() == ISD::FNEG)
5322 RHS = RHS.getOperand(i: 0);
5323 else
5324 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5325
5326 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5327 if (Res.getOpcode() != Opc)
5328 return SDValue(); // Op got folded away.
5329 if (!N0.hasOneUse())
5330 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5331 return Res;
5332 }
5333 case ISD::FMA:
5334 case ISD::FMAD: {
5335 // TODO: handle llvm.amdgcn.fma.legacy
5336 if (!MayIgnoreSignedZeroForAllUses)
5337 return SDValue();
5338
5339 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
5340 SDValue LHS = N0.getOperand(i: 0);
5341 SDValue MHS = N0.getOperand(i: 1);
5342 SDValue RHS = N0.getOperand(i: 2);
5343
5344 if (LHS.getOpcode() == ISD::FNEG)
5345 LHS = LHS.getOperand(i: 0);
5346 else if (MHS.getOpcode() == ISD::FNEG)
5347 MHS = MHS.getOperand(i: 0);
5348 else
5349 MHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: MHS);
5350
5351 if (RHS.getOpcode() != ISD::FNEG)
5352 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5353 else
5354 RHS = RHS.getOperand(i: 0);
5355
5356 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: MHS, N3: RHS);
5357 if (Res.getOpcode() != Opc)
5358 return SDValue(); // Op got folded away.
5359 if (!N0.hasOneUse())
5360 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5361 return Res;
5362 }
5363 case ISD::FMAXNUM:
5364 case ISD::FMINNUM:
5365 case ISD::FMAXNUM_IEEE:
5366 case ISD::FMINNUM_IEEE:
5367 case ISD::FMINIMUM:
5368 case ISD::FMAXIMUM:
5369 case ISD::FMINIMUMNUM:
5370 case ISD::FMAXIMUMNUM:
5371 case AMDGPUISD::FMAX_LEGACY:
5372 case AMDGPUISD::FMIN_LEGACY: {
5373 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
5374 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
5375 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
5376 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
5377
5378 SDValue LHS = N0.getOperand(i: 0);
5379 SDValue RHS = N0.getOperand(i: 1);
5380
5381 // 0 doesn't have a negated inline immediate.
5382 // TODO: This constant check should be generalized to other operations.
5383 if (isConstantCostlierToNegate(N: RHS))
5384 return SDValue();
5385
5386 // Swapping min<->max flips which operand a signed zero tie selects.
5387 if ((Opc == AMDGPUISD::FMIN_LEGACY || Opc == AMDGPUISD::FMAX_LEGACY) &&
5388 !canIgnoreLegacyMinMaxTies(DAG, Flags: N0->getFlags(), LHS, RHS))
5389 return SDValue();
5390
5391 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5392 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5393 unsigned Opposite = inverseMinMax(Opc);
5394
5395 SDValue Res = DAG.getNode(Opcode: Opposite, DL: SL, VT, N1: NegLHS, N2: NegRHS, Flags: N0->getFlags());
5396 if (Res.getOpcode() != Opposite)
5397 return SDValue(); // Op got folded away.
5398 if (!N0.hasOneUse())
5399 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5400 return Res;
5401 }
5402 case AMDGPUISD::FMED3: {
5403 // med3 sorts a NaN input as smaller than everything regardless of its sign,
5404 // so negating all operands does not sign-flip the median when an input may
5405 // be NaN.
5406 if (!N0->getFlags().hasNoNaNs())
5407 return SDValue();
5408
5409 SDValue Ops[3];
5410 for (unsigned I = 0; I < 3; ++I)
5411 Ops[I] = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N0->getOperand(Num: I), Flags: N0->getFlags());
5412
5413 SDValue Res = DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT, Ops, Flags: N0->getFlags());
5414 if (Res.getOpcode() != AMDGPUISD::FMED3)
5415 return SDValue(); // Op got folded away.
5416
5417 if (!N0.hasOneUse()) {
5418 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res);
5419 DAG.ReplaceAllUsesWith(From: N0, To: Neg);
5420
5421 for (SDNode *U : Neg->users())
5422 DCI.AddToWorklist(N: U);
5423 }
5424
5425 return Res;
5426 }
5427 case ISD::FP_EXTEND:
5428 case ISD::FTRUNC:
5429 case ISD::FRINT:
5430 case ISD::FNEARBYINT: // XXX - Should fround be handled?
5431 case ISD::FROUNDEVEN:
5432 case ISD::FSIN:
5433 case ISD::FCANONICALIZE:
5434 case AMDGPUISD::RCP:
5435 case AMDGPUISD::RCP_LEGACY:
5436 case AMDGPUISD::RCP_IFLAG:
5437 case AMDGPUISD::SIN_HW: {
5438 SDValue CvtSrc = N0.getOperand(i: 0);
5439 if (CvtSrc.getOpcode() == ISD::FNEG) {
5440 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
5441 // (fneg (rcp (fneg x))) -> (rcp x)
5442 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: CvtSrc.getOperand(i: 0));
5443 }
5444
5445 if (!N0.hasOneUse())
5446 return SDValue();
5447
5448 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
5449 // (fneg (rcp x)) -> (rcp (fneg x))
5450 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5451 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: Neg, Flags: N0->getFlags());
5452 }
5453 case ISD::FP_ROUND: {
5454 SDValue CvtSrc = N0.getOperand(i: 0);
5455
5456 if (CvtSrc.getOpcode() == ISD::FNEG) {
5457 // (fneg (fp_round (fneg x))) -> (fp_round x)
5458 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT,
5459 N1: CvtSrc.getOperand(i: 0), N2: N0.getOperand(i: 1));
5460 }
5461
5462 if (!N0.hasOneUse())
5463 return SDValue();
5464
5465 // (fneg (fp_round x)) -> (fp_round (fneg x))
5466 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5467 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Neg, N2: N0.getOperand(i: 1));
5468 }
5469 case ISD::FP16_TO_FP: {
5470 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
5471 // f16, but legalization of f16 fneg ends up pulling it out of the source.
5472 // Put the fneg back as a legal source operation that can be matched later.
5473 SDLoc SL(N);
5474
5475 SDValue Src = N0.getOperand(i: 0);
5476 EVT SrcVT = Src.getValueType();
5477
5478 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
5479 SDValue IntFNeg = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: SrcVT, N1: Src,
5480 N2: DAG.getConstant(Val: 0x8000, DL: SL, VT: SrcVT));
5481 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFNeg);
5482 }
5483 case ISD::SELECT: {
5484 // fneg (select c, a, b) -> select c, (fneg a), (fneg b)
5485 // TODO: Invert conditions of foldFreeOpFromSelect
5486 return SDValue();
5487 }
5488 case ISD::BITCAST: {
5489 SDLoc SL(N);
5490 SDValue BCSrc = N0.getOperand(i: 0);
5491 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
5492 SDValue HighBits = BCSrc.getOperand(i: BCSrc.getNumOperands() - 1);
5493 if (VT != MVT::f64 || HighBits.getValueType().getSizeInBits() != 32 ||
5494 !fnegFoldsIntoOp(N: HighBits.getNode()))
5495 return SDValue();
5496
5497 // f64 fneg only really needs to operate on the high half of of the
5498 // register, so try to force it to an f32 operation to help make use of
5499 // source modifiers.
5500 //
5501 //
5502 // fneg (f64 (bitcast (build_vector x, y))) ->
5503 // f64 (bitcast (build_vector (bitcast i32:x to f32),
5504 // (fneg (bitcast i32:y to f32)))
5505
5506 SDValue CastHi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: HighBits);
5507 SDValue NegHi = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: CastHi);
5508 SDValue CastBack =
5509 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: HighBits.getValueType(), Operand: NegHi);
5510
5511 SmallVector<SDValue, 8> Ops(BCSrc->ops());
5512 Ops.back() = CastBack;
5513 DCI.AddToWorklist(N: NegHi.getNode());
5514 SDValue Build =
5515 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: BCSrc.getValueType(), Ops);
5516 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Build);
5517
5518 if (!N0.hasOneUse())
5519 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Result));
5520 return Result;
5521 }
5522
5523 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 &&
5524 BCSrc.hasOneUse()) {
5525 // fneg (bitcast (f32 (select cond, i32:lhs, i32:rhs))) ->
5526 // select cond, (bitcast i32:lhs to f32), (bitcast i32:rhs to f32)
5527
5528 // TODO: Cast back result for multiple uses is beneficial in some cases.
5529
5530 SDValue LHS =
5531 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 1));
5532 SDValue RHS =
5533 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 2));
5534
5535 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: LHS);
5536 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHS);
5537
5538 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: BCSrc.getOperand(i: 0), N2: NegLHS,
5539 N3: NegRHS);
5540 }
5541
5542 return SDValue();
5543 }
5544 default:
5545 return SDValue();
5546 }
5547}
5548
5549SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
5550 DAGCombinerInfo &DCI) const {
5551 SelectionDAG &DAG = DCI.DAG;
5552 SDValue N0 = N->getOperand(Num: 0);
5553
5554 if (!N0.hasOneUse())
5555 return SDValue();
5556
5557 switch (N0.getOpcode()) {
5558 case ISD::FP16_TO_FP: {
5559 assert(!isTypeLegal(MVT::f16) && "should only see if f16 is illegal");
5560 SDLoc SL(N);
5561 SDValue Src = N0.getOperand(i: 0);
5562 EVT SrcVT = Src.getValueType();
5563
5564 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
5565 SDValue IntFAbs = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SrcVT, N1: Src,
5566 N2: DAG.getConstant(Val: 0x7fff, DL: SL, VT: SrcVT));
5567 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFAbs);
5568 }
5569 case ISD::FP_ROUND: {
5570 SDLoc SL(N);
5571 SDValue CvtSrc = N0.getOperand(i: 0);
5572
5573 // fabs (fp_round x) -> fp_round (fabs x)
5574 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc,
5575 Flags: N->getFlags());
5576 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: N->getValueType(ResNo: 0), N1: Abs,
5577 N2: N0.getOperand(i: 1), Flags: N0->getFlags());
5578 }
5579 default:
5580 return SDValue();
5581 }
5582}
5583
5584SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
5585 DAGCombinerInfo &DCI) const {
5586 const auto *CFP = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
5587 if (!CFP)
5588 return SDValue();
5589
5590 std::optional<APFloat> Result = AMDGPU::evaluateRcp(Val: CFP->getValueAPF());
5591 if (!Result)
5592 return SDValue();
5593
5594 return DCI.DAG.getConstantFP(Val: *Result, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
5595}
5596
5597bool AMDGPUTargetLowering::isInt64ImmLegal(SDNode *N, SelectionDAG &DAG) const {
5598 if (!Subtarget->isGCN())
5599 return false;
5600
5601 ConstantSDNode *SDConstant = dyn_cast<ConstantSDNode>(Val: N);
5602 ConstantFPSDNode *SDFPConstant = dyn_cast<ConstantFPSDNode>(Val: N);
5603 auto &ST = DAG.getSubtarget<GCNSubtarget>();
5604 const auto *TII = ST.getInstrInfo();
5605
5606 if (!ST.hasVMovB64Inst() || (!SDConstant && !SDFPConstant))
5607 return false;
5608
5609 if (ST.has64BitLiterals())
5610 return true;
5611
5612 if (SDConstant) {
5613 const APInt &APVal = SDConstant->getAPIntValue();
5614 return isUInt<32>(x: APVal.getZExtValue()) || TII->isInlineConstant(Imm: APVal);
5615 }
5616
5617 APInt Val = SDFPConstant->getValueAPF().bitcastToAPInt();
5618 return isUInt<32>(x: Val.getZExtValue()) || TII->isInlineConstant(Imm: Val);
5619}
5620
5621SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
5622 DAGCombinerInfo &DCI) const {
5623 SelectionDAG &DAG = DCI.DAG;
5624 SDLoc DL(N);
5625
5626 switch(N->getOpcode()) {
5627 default:
5628 break;
5629 case ISD::BITCAST: {
5630 EVT DestVT = N->getValueType(ResNo: 0);
5631
5632 // Push casts through vector builds. This helps avoid emitting a large
5633 // number of copies when materializing floating point vector constants.
5634 //
5635 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
5636 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
5637 if (DestVT.isVector()) {
5638 SDValue Src = N->getOperand(Num: 0);
5639 if (Src.getOpcode() == ISD::BUILD_VECTOR &&
5640 (DCI.getDAGCombineLevel() < AfterLegalizeDAG ||
5641 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: DestVT))) {
5642 EVT SrcVT = Src.getValueType();
5643 unsigned NElts = DestVT.getVectorNumElements();
5644
5645 if (SrcVT.getVectorNumElements() == NElts) {
5646 EVT DestEltVT = DestVT.getVectorElementType();
5647
5648 SmallVector<SDValue, 8> CastedElts;
5649 SDLoc SL(N);
5650 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
5651 SDValue Elt = Src.getOperand(i: I);
5652 CastedElts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: DestEltVT, Operand: Elt));
5653 }
5654
5655 return DAG.getBuildVector(VT: DestVT, DL: SL, Ops: CastedElts);
5656 }
5657 }
5658 }
5659
5660 if (DestVT.getSizeInBits() != 64 || !DestVT.isVector())
5661 break;
5662
5663 // Fold bitcasts of constants.
5664 //
5665 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
5666 // TODO: Generalize and move to DAGCombiner
5667 SDValue Src = N->getOperand(Num: 0);
5668 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Src)) {
5669 SDLoc SL(N);
5670 if (isInt64ImmLegal(N: C, DAG))
5671 break;
5672 uint64_t CVal = C->getZExtValue();
5673 SDValue BV = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5674 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5675 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5676 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: BV);
5677 }
5678
5679 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
5680 const APInt &Val = C->getValueAPF().bitcastToAPInt();
5681 SDLoc SL(N);
5682 if (isInt64ImmLegal(N: C, DAG))
5683 break;
5684 uint64_t CVal = Val.getZExtValue();
5685 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5686 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5687 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5688
5689 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: Vec);
5690 }
5691
5692 break;
5693 }
5694 case ISD::SHL:
5695 case ISD::SRA:
5696 case ISD::SRL: {
5697 // Range metadata can be invalidated when loads are converted to legal types
5698 // (e.g. v2i64 -> v4i32).
5699 // Try to convert vector shl/sra/srl before type legalization so that range
5700 // metadata can be utilized.
5701 if (!(N->getValueType(ResNo: 0).isVector() &&
5702 DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
5703 DCI.getDAGCombineLevel() < AfterLegalizeDAG)
5704 break;
5705 if (N->getOpcode() == ISD::SHL)
5706 return performShlCombine(N, DCI);
5707 if (N->getOpcode() == ISD::SRA)
5708 return performSraCombine(N, DCI);
5709 return performSrlCombine(N, DCI);
5710 }
5711 case ISD::TRUNCATE:
5712 return performTruncateCombine(N, DCI);
5713 case ISD::MUL:
5714 return performMulCombine(N, DCI);
5715 case AMDGPUISD::MUL_U24:
5716 case AMDGPUISD::MUL_I24: {
5717 if (SDValue Simplified = simplifyMul24(Node24: N, DCI))
5718 return Simplified;
5719 break;
5720 }
5721 case AMDGPUISD::MULHI_I24:
5722 case AMDGPUISD::MULHI_U24:
5723 return simplifyMul24(Node24: N, DCI);
5724 case ISD::SMUL_LOHI:
5725 case ISD::UMUL_LOHI:
5726 return performMulLoHiCombine(N, DCI);
5727 case ISD::MULHS:
5728 return performMulhsCombine(N, DCI);
5729 case ISD::MULHU:
5730 return performMulhuCombine(N, DCI);
5731 case ISD::SELECT:
5732 return performSelectCombine(N, DCI);
5733 case ISD::FNEG:
5734 return performFNegCombine(N, DCI);
5735 case ISD::FABS:
5736 return performFAbsCombine(N, DCI);
5737 case AMDGPUISD::BFE_I32:
5738 case AMDGPUISD::BFE_U32: {
5739 assert(N->getValueType(0) == MVT::i32 &&
5740 "BFE_I32/BFE_U32 is a 32-bit operation");
5741 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
5742 if (!Width)
5743 break;
5744
5745 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
5746 if (WidthVal == 0)
5747 return DAG.getConstant(Val: 0, DL, VT: MVT::i32);
5748
5749 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
5750 if (!Offset)
5751 break;
5752
5753 SDValue BitsFrom = N->getOperand(Num: 0);
5754 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
5755
5756 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
5757
5758 if (OffsetVal == 0) {
5759 // This is already sign / zero extended, so try to fold away extra BFEs.
5760 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WidthVal);
5761 if (Signed) {
5762 if (DAG.ComputeNumSignBits(Op: BitsFrom) >= 32 - WidthVal + 1)
5763 return BitsFrom;
5764
5765 // This is a sign_extend_inreg. Replace it to take advantage of existing
5766 // DAG Combines. If not eliminated, we will match back to BFE during
5767 // selection.
5768
5769 // TODO: The sext_inreg of extended types ends, although we can could
5770 // handle them in a single BFE.
5771 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32, N1: BitsFrom,
5772 N2: DAG.getValueType(SmallVT));
5773 }
5774
5775 if (DAG.MaskedValueIsZero(Op: BitsFrom,
5776 Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - WidthVal)))
5777 return BitsFrom;
5778
5779 return DAG.getZeroExtendInReg(Op: BitsFrom, DL, VT: SmallVT);
5780 }
5781
5782 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(Val&: BitsFrom)) {
5783 if (Signed) {
5784 return constantFoldBFE<int32_t>(DAG,
5785 Src0: CVal->getSExtValue(),
5786 Offset: OffsetVal,
5787 Width: WidthVal,
5788 DL);
5789 }
5790
5791 return constantFoldBFE<uint32_t>(DAG,
5792 Src0: CVal->getZExtValue(),
5793 Offset: OffsetVal,
5794 Width: WidthVal,
5795 DL);
5796 }
5797
5798 if ((OffsetVal + WidthVal) >= 32 &&
5799 !(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
5800 SDValue ShiftVal = DAG.getConstant(Val: OffsetVal, DL, VT: MVT::i32);
5801 return DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32,
5802 N1: BitsFrom, N2: ShiftVal);
5803 }
5804
5805 if (BitsFrom.hasOneUse()) {
5806 APInt Demanded = APInt::getBitsSet(numBits: 32,
5807 loBit: OffsetVal,
5808 hiBit: OffsetVal + WidthVal);
5809
5810 KnownBits Known;
5811 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
5812 !DCI.isBeforeLegalizeOps());
5813 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5814 if (TLI.ShrinkDemandedConstant(Op: BitsFrom, DemandedBits: Demanded, TLO) ||
5815 TLI.SimplifyDemandedBits(Op: BitsFrom, DemandedBits: Demanded, Known, TLO)) {
5816 DCI.CommitTargetLoweringOpt(TLO);
5817 }
5818 }
5819
5820 break;
5821 }
5822 case ISD::LOAD:
5823 return performLoadCombine(N, DCI);
5824 case ISD::STORE:
5825 return performStoreCombine(N, DCI);
5826 case AMDGPUISD::RCP:
5827 case AMDGPUISD::RCP_IFLAG:
5828 return performRcpCombine(N, DCI);
5829 case ISD::AssertZext:
5830 case ISD::AssertSext:
5831 return performAssertSZExtCombine(N, DCI);
5832 case ISD::INTRINSIC_WO_CHAIN:
5833 return performIntrinsicWOChainCombine(N, DCI);
5834 case AMDGPUISD::FMAD_FTZ: {
5835 SDValue N0 = N->getOperand(Num: 0);
5836 SDValue N1 = N->getOperand(Num: 1);
5837 SDValue N2 = N->getOperand(Num: 2);
5838 EVT VT = N->getValueType(ResNo: 0);
5839
5840 // FMAD_FTZ is a FMAD + flush denormals to zero.
5841 // We flush the inputs, the intermediate step, and the output.
5842 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Val&: N0);
5843 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
5844 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
5845 if (N0CFP && N1CFP && N2CFP) {
5846 const auto FTZ = [](const APFloat &V) {
5847 if (V.isDenormal()) {
5848 APFloat Zero(V.getSemantics(), 0);
5849 return V.isNegative() ? -Zero : Zero;
5850 }
5851 return V;
5852 };
5853
5854 APFloat V0 = FTZ(N0CFP->getValueAPF());
5855 APFloat V1 = FTZ(N1CFP->getValueAPF());
5856 APFloat V2 = FTZ(N2CFP->getValueAPF());
5857 V0.multiply(RHS: V1, RM: APFloat::rmNearestTiesToEven);
5858 V0 = FTZ(V0);
5859 V0.add(RHS: V2, RM: APFloat::rmNearestTiesToEven);
5860 return DAG.getConstantFP(Val: FTZ(V0), DL, VT);
5861 }
5862 break;
5863 }
5864 }
5865 return SDValue();
5866}
5867
5868bool AMDGPUTargetLowering::SimplifyDemandedBitsForTargetNode(
5869 SDValue Op, const APInt &OriginalDemandedBits,
5870 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
5871 unsigned Depth) const {
5872 switch (Op.getOpcode()) {
5873 case ISD::INTRINSIC_WO_CHAIN: {
5874 switch (Op.getConstantOperandVal(i: 0)) {
5875 case Intrinsic::amdgcn_readfirstlane:
5876 case Intrinsic::amdgcn_readlane:
5877 case Intrinsic::amdgcn_wwm: {
5878 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5879 DemandedElts: OriginalDemandedElts, Known, TLO, Depth: Depth + 1))
5880 return true;
5881 break;
5882 }
5883 case Intrinsic::amdgcn_set_inactive:
5884 case Intrinsic::amdgcn_set_inactive_chain_arg: {
5885 // The result is operand 1 in active lanes and operand 2 in inactive
5886 // lanes, so the known bits are the intersection of both operands.
5887 KnownBits KnownValue, KnownInactive;
5888 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5889 DemandedElts: OriginalDemandedElts, Known&: KnownValue, TLO,
5890 Depth: Depth + 1))
5891 return true;
5892 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), DemandedBits: OriginalDemandedBits,
5893 DemandedElts: OriginalDemandedElts, Known&: KnownInactive, TLO,
5894 Depth: Depth + 1))
5895 return true;
5896 Known = KnownValue.intersectWith(RHS: KnownInactive);
5897 break;
5898 }
5899 default:
5900 break;
5901 }
5902 break;
5903 }
5904 default:
5905 break;
5906 }
5907
5908 return false;
5909}
5910
5911//===----------------------------------------------------------------------===//
5912// Helper functions
5913//===----------------------------------------------------------------------===//
5914
5915SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
5916 const TargetRegisterClass *RC,
5917 Register Reg, EVT VT,
5918 const SDLoc &SL,
5919 bool RawReg) const {
5920 MachineFunction &MF = DAG.getMachineFunction();
5921 MachineRegisterInfo &MRI = MF.getRegInfo();
5922 Register VReg;
5923
5924 if (!MRI.isLiveIn(Reg)) {
5925 VReg = MRI.createVirtualRegister(RegClass: RC);
5926 MRI.addLiveIn(Reg, vreg: VReg);
5927 } else {
5928 VReg = MRI.getLiveInVirtReg(PReg: Reg);
5929 }
5930
5931 if (RawReg)
5932 return DAG.getRegister(Reg: VReg, VT);
5933
5934 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: VReg, VT);
5935}
5936
5937// This may be called multiple times, and nothing prevents creating multiple
5938// objects at the same offset. See if we already defined this object.
5939static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
5940 int64_t Offset) {
5941 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
5942 if (MFI.getObjectOffset(ObjectIdx: I) == Offset) {
5943 assert(MFI.getObjectSize(I) == Size);
5944 return I;
5945 }
5946 }
5947
5948 return MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
5949}
5950
5951SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
5952 EVT VT,
5953 const SDLoc &SL,
5954 int64_t Offset) const {
5955 MachineFunction &MF = DAG.getMachineFunction();
5956 MachineFrameInfo &MFI = MF.getFrameInfo();
5957 int FI = getOrCreateFixedStackObject(MFI, Size: VT.getStoreSize(), Offset);
5958
5959 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
5960 SDValue Ptr = DAG.getFrameIndex(FI, VT: MVT::i32);
5961
5962 return DAG.getLoad(VT, dl: SL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: SrcPtrInfo, Alignment: Align(4),
5963 MMOFlags: MachineMemOperand::MODereferenceable |
5964 MachineMemOperand::MOInvariant);
5965}
5966
5967SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
5968 const SDLoc &SL,
5969 SDValue Chain,
5970 SDValue ArgVal,
5971 int64_t Offset) const {
5972 MachineFunction &MF = DAG.getMachineFunction();
5973 MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
5974 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5975
5976 SDValue Ptr = DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32);
5977 // Stores to the argument stack area are relative to the stack pointer.
5978 SDValue SP =
5979 DAG.getCopyFromReg(Chain, dl: SL, Reg: Info->getStackPtrOffsetReg(), VT: MVT::i32);
5980 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: SP, N2: Ptr);
5981 SDValue Store = DAG.getStore(Chain, dl: SL, Val: ArgVal, Ptr, PtrInfo: DstInfo, Alignment: Align(4),
5982 MMOFlags: MachineMemOperand::MODereferenceable);
5983 return Store;
5984}
5985
5986SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
5987 const TargetRegisterClass *RC,
5988 EVT VT, const SDLoc &SL,
5989 const ArgDescriptor &Arg) const {
5990 assert(Arg && "Attempting to load missing argument");
5991
5992 SDValue V = Arg.isRegister() ?
5993 CreateLiveInRegister(DAG, RC, Reg: Arg.getRegister(), VT, SL) :
5994 loadStackInputValue(DAG, VT, SL, Offset: Arg.getStackOffset());
5995
5996 if (!Arg.isMasked())
5997 return V;
5998
5999 unsigned Mask = Arg.getMask();
6000 unsigned Shift = llvm::countr_zero<unsigned>(Val: Mask);
6001 V = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: V,
6002 N2: DAG.getShiftAmountConstant(Val: Shift, VT, DL: SL));
6003 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT, N1: V,
6004 N2: DAG.getConstant(Val: Mask >> Shift, DL: SL, VT));
6005}
6006
6007uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
6008 uint64_t ExplicitKernArgSize, const ImplicitParameter Param) const {
6009 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
6010 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr();
6011 uint64_t ArgOffset =
6012 alignTo(Size: ExplicitKernArgSize, A: Alignment) + ExplicitArgOffset;
6013 switch (Param) {
6014 case FIRST_IMPLICIT:
6015 return ArgOffset;
6016 case PRIVATE_BASE:
6017 return ArgOffset + AMDGPU::ImplicitArg::PRIVATE_BASE_OFFSET;
6018 case SHARED_BASE:
6019 return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET;
6020 case QUEUE_PTR:
6021 return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET;
6022 }
6023 llvm_unreachable("unexpected implicit parameter type");
6024}
6025
6026uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
6027 const MachineFunction &MF, const ImplicitParameter Param) const {
6028 const AMDGPUMachineFunctionInfo *MFI =
6029 MF.getInfo<AMDGPUMachineFunctionInfo>();
6030 return getImplicitParameterOffset(ExplicitKernArgSize: MFI->getExplicitKernArgSize(), Param);
6031}
6032
6033SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
6034 SelectionDAG &DAG, int Enabled,
6035 int &RefinementSteps,
6036 bool &UseOneConstNR,
6037 bool Reciprocal) const {
6038 EVT VT = Operand.getValueType();
6039
6040 if (VT == MVT::f32) {
6041 RefinementSteps = 0;
6042 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(Operand), VT, Operand);
6043 }
6044
6045 // TODO: There is also f64 rsq instruction, but the documentation is less
6046 // clear on its precision.
6047
6048 return SDValue();
6049}
6050
6051SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
6052 SelectionDAG &DAG, int Enabled,
6053 int &RefinementSteps) const {
6054 EVT VT = Operand.getValueType();
6055
6056 if (VT == MVT::f32) {
6057 // Reciprocal, < 1 ulp error.
6058 //
6059 // This reciprocal approximation converges to < 0.5 ulp error with one
6060 // newton rhapson performed with two fused multiple adds (FMAs).
6061
6062 RefinementSteps = 0;
6063 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SDLoc(Operand), VT, Operand);
6064 }
6065
6066 // TODO: There is also f64 rcp instruction, but the documentation is less
6067 // clear on its precision.
6068
6069 return SDValue();
6070}
6071
6072static unsigned workitemIntrinsicDim(unsigned ID) {
6073 switch (ID) {
6074 case Intrinsic::amdgcn_workitem_id_x:
6075 return 0;
6076 case Intrinsic::amdgcn_workitem_id_y:
6077 return 1;
6078 case Intrinsic::amdgcn_workitem_id_z:
6079 return 2;
6080 default:
6081 llvm_unreachable("not a workitem intrinsic");
6082 }
6083}
6084
6085void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
6086 const SDValue Op, KnownBits &Known,
6087 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
6088
6089 Known.resetAll(); // Don't know anything.
6090
6091 unsigned Opc = Op.getOpcode();
6092
6093 switch (Opc) {
6094 default:
6095 break;
6096 case AMDGPUISD::CARRY:
6097 case AMDGPUISD::BORROW: {
6098 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31);
6099 break;
6100 }
6101
6102 case AMDGPUISD::BFE_I32:
6103 case AMDGPUISD::BFE_U32: {
6104 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6105 if (!CWidth)
6106 return;
6107
6108 uint32_t Width = CWidth->getZExtValue() & 0x1f;
6109
6110 if (Opc == AMDGPUISD::BFE_U32)
6111 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - Width);
6112
6113 break;
6114 }
6115 case AMDGPUISD::FP_TO_FP16: {
6116 unsigned BitWidth = Known.getBitWidth();
6117
6118 // High bits are zero.
6119 Known.Zero = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 16);
6120 break;
6121 }
6122 case AMDGPUISD::MUL_U24:
6123 case AMDGPUISD::MUL_I24: {
6124 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6125 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6126 unsigned BitWidth = Op.getScalarValueSizeInBits();
6127
6128 // Sign/Zero extend from 24 bits.
6129 if (Opc == AMDGPUISD::MUL_I24) {
6130 LHSKnown = LHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6131 RHSKnown = RHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6132 } else {
6133 LHSKnown = LHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6134 RHSKnown = RHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6135 }
6136
6137 // TODO: SelfMultiply can be poison, but not undef.
6138 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
6139 if (SelfMultiply)
6140 SelfMultiply &= DAG.isGuaranteedNotToBeUndefOrPoison(
6141 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
6142 Depth: Depth + 1);
6143
6144 Known = KnownBits::mul(LHS: LHSKnown, RHS: RHSKnown, NoUndefSelfMultiply: SelfMultiply);
6145 break;
6146 }
6147 case AMDGPUISD::PERM: {
6148 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6149 if (!CMask)
6150 return;
6151
6152 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6153 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6154 unsigned Sel = CMask->getZExtValue();
6155
6156 for (unsigned I = 0; I < 32; I += 8) {
6157 unsigned SelBits = Sel & 0xff;
6158 if (SelBits < 4) {
6159 SelBits *= 8;
6160 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6161 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6162 } else if (SelBits < 7) {
6163 SelBits = (SelBits & 3) * 8;
6164 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6165 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6166 } else if (SelBits == 0x0c) {
6167 Known.Zero |= 0xFFull << I;
6168 } else if (SelBits > 0x0c) {
6169 Known.One |= 0xFFull << I;
6170 }
6171 Sel >>= 8;
6172 }
6173 break;
6174 }
6175 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
6176 Known.Zero.setHighBits(24);
6177 break;
6178 }
6179 case AMDGPUISD::BUFFER_LOAD_USHORT: {
6180 Known.Zero.setHighBits(16);
6181 break;
6182 }
6183 case AMDGPUISD::LDS: {
6184 auto *GA = cast<GlobalAddressSDNode>(Val: Op.getOperand(i: 0).getNode());
6185 Align Alignment = GA->getGlobal()->getPointerAlignment(DL: DAG.getDataLayout());
6186
6187 Known.Zero.setHighBits(16);
6188 Known.Zero.setLowBits(Log2(A: Alignment));
6189 break;
6190 }
6191 case AMDGPUISD::SMIN3:
6192 case AMDGPUISD::SMAX3:
6193 case AMDGPUISD::SMED3:
6194 case AMDGPUISD::UMIN3:
6195 case AMDGPUISD::UMAX3:
6196 case AMDGPUISD::UMED3: {
6197 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6198 if (Known2.isUnknown())
6199 break;
6200
6201 KnownBits Known1 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6202 if (Known1.isUnknown())
6203 break;
6204
6205 KnownBits Known0 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6206 if (Known0.isUnknown())
6207 break;
6208
6209 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
6210 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
6211 Known.One = Known0.One & Known1.One & Known2.One;
6212 break;
6213 }
6214 case ISD::INTRINSIC_WO_CHAIN: {
6215 unsigned IID = Op.getConstantOperandVal(i: 0);
6216 switch (IID) {
6217 case Intrinsic::amdgcn_workitem_id_x:
6218 case Intrinsic::amdgcn_workitem_id_y:
6219 case Intrinsic::amdgcn_workitem_id_z: {
6220 unsigned MaxValue = Subtarget->getMaxWorkitemID(
6221 Kernel: DAG.getMachineFunction().getFunction(), Dimension: workitemIntrinsicDim(ID: IID));
6222 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
6223 break;
6224 }
6225 default:
6226 break;
6227 }
6228 }
6229 }
6230}
6231
6232unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
6233 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6234 unsigned Depth) const {
6235 switch (Op.getOpcode()) {
6236 case AMDGPUISD::BFE_I32: {
6237 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6238 if (!Width)
6239 return 1;
6240
6241 unsigned SignBits = 32 - (Width->getZExtValue() & 0x1f) + 1;
6242 if (!isNullConstant(V: Op.getOperand(i: 1)))
6243 return SignBits;
6244
6245 // TODO: Could probably figure something out with non-0 offsets.
6246 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6247 return std::max(a: SignBits, b: Op0SignBits);
6248 }
6249
6250 case AMDGPUISD::BFE_U32: {
6251 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6252 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
6253 }
6254
6255 case AMDGPUISD::CARRY:
6256 case AMDGPUISD::BORROW:
6257 return 31;
6258 case AMDGPUISD::BUFFER_LOAD_BYTE:
6259 return 25;
6260 case AMDGPUISD::BUFFER_LOAD_SHORT:
6261 return 17;
6262 case AMDGPUISD::BUFFER_LOAD_UBYTE:
6263 return 24;
6264 case AMDGPUISD::BUFFER_LOAD_USHORT:
6265 return 16;
6266 case AMDGPUISD::FP_TO_FP16:
6267 return 16;
6268 case AMDGPUISD::SMIN3:
6269 case AMDGPUISD::SMAX3:
6270 case AMDGPUISD::SMED3:
6271 case AMDGPUISD::UMIN3:
6272 case AMDGPUISD::UMAX3:
6273 case AMDGPUISD::UMED3: {
6274 unsigned Tmp2 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6275 if (Tmp2 == 1)
6276 return 1; // Early out.
6277
6278 unsigned Tmp1 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6279 if (Tmp1 == 1)
6280 return 1; // Early out.
6281
6282 unsigned Tmp0 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6283 if (Tmp0 == 1)
6284 return 1; // Early out.
6285
6286 return std::min(l: {Tmp0, Tmp1, Tmp2});
6287 }
6288 default:
6289 return 1;
6290 }
6291}
6292
6293unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
6294 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
6295 const MachineRegisterInfo &MRI, unsigned Depth) const {
6296 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
6297 if (!MI)
6298 return 1;
6299
6300 // TODO: Check range metadata on MMO.
6301 switch (MI->getOpcode()) {
6302 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
6303 return 25;
6304 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
6305 return 17;
6306 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
6307 return 24;
6308 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
6309 return 16;
6310 case AMDGPU::G_AMDGPU_SMED3:
6311 case AMDGPU::G_AMDGPU_UMED3: {
6312 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
6313 unsigned Tmp2 = Analysis.computeNumSignBits(R: Src2, DemandedElts, Depth: Depth + 1);
6314 if (Tmp2 == 1)
6315 return 1;
6316 unsigned Tmp1 = Analysis.computeNumSignBits(R: Src1, DemandedElts, Depth: Depth + 1);
6317 if (Tmp1 == 1)
6318 return 1;
6319 unsigned Tmp0 = Analysis.computeNumSignBits(R: Src0, DemandedElts, Depth: Depth + 1);
6320 if (Tmp0 == 1)
6321 return 1;
6322 return std::min(l: {Tmp0, Tmp1, Tmp2});
6323 }
6324 default:
6325 return 1;
6326 }
6327}
6328
6329bool AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode(
6330 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6331 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
6332 unsigned Opcode = Op.getOpcode();
6333 switch (Opcode) {
6334 case AMDGPUISD::BFE_I32:
6335 case AMDGPUISD::BFE_U32:
6336 return false;
6337 }
6338 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
6339 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
6340}
6341
6342bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
6343 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
6344 unsigned Depth) const {
6345 unsigned Opcode = Op.getOpcode();
6346 switch (Opcode) {
6347 case AMDGPUISD::FMIN_LEGACY:
6348 case AMDGPUISD::FMAX_LEGACY:
6349 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6350 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6351 case AMDGPUISD::FMUL_LEGACY:
6352 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
6353 if (SNaN)
6354 return true;
6355 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6356 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6357 }
6358 case AMDGPUISD::FMED3:
6359 case AMDGPUISD::FMIN3:
6360 case AMDGPUISD::FMAX3:
6361 case AMDGPUISD::FMINIMUM3:
6362 case AMDGPUISD::FMAXIMUM3:
6363 case AMDGPUISD::FMAD_FTZ: {
6364 if (SNaN)
6365 return true;
6366 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6367 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6368 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6369 }
6370 case AMDGPUISD::CVT_F32_UBYTE0:
6371 case AMDGPUISD::CVT_F32_UBYTE1:
6372 case AMDGPUISD::CVT_F32_UBYTE2:
6373 case AMDGPUISD::CVT_F32_UBYTE3:
6374 return true;
6375
6376 case AMDGPUISD::RCP:
6377 case AMDGPUISD::RSQ:
6378 case AMDGPUISD::RCP_LEGACY:
6379 case AMDGPUISD::RSQ_CLAMP: {
6380 if (SNaN)
6381 return true;
6382
6383 // TODO: Need is known positive check.
6384 return false;
6385 }
6386 case ISD::FLDEXP:
6387 case AMDGPUISD::FRACT: {
6388 if (SNaN)
6389 return true;
6390 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6391 }
6392 case AMDGPUISD::DIV_SCALE:
6393 case AMDGPUISD::DIV_FMAS:
6394 case AMDGPUISD::DIV_FIXUP:
6395 // TODO: Refine on operands.
6396 return SNaN;
6397 case AMDGPUISD::SIN_HW:
6398 case AMDGPUISD::COS_HW: {
6399 // TODO: Need check for infinity
6400 return SNaN;
6401 }
6402 case ISD::INTRINSIC_WO_CHAIN: {
6403 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
6404 // TODO: Handle more intrinsics
6405 switch (IntrinsicID) {
6406 case Intrinsic::amdgcn_cubeid:
6407 case Intrinsic::amdgcn_cvt_off_f32_i4:
6408 return true;
6409
6410 case Intrinsic::amdgcn_frexp_mant: {
6411 if (SNaN)
6412 return true;
6413 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6414 }
6415 case Intrinsic::amdgcn_cvt_pkrtz: {
6416 if (SNaN)
6417 return true;
6418 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6419 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6420 }
6421 case Intrinsic::amdgcn_rcp:
6422 case Intrinsic::amdgcn_rsq:
6423 case Intrinsic::amdgcn_rcp_legacy:
6424 case Intrinsic::amdgcn_rsq_legacy:
6425 case Intrinsic::amdgcn_rsq_clamp:
6426 case Intrinsic::amdgcn_tanh: {
6427 if (SNaN)
6428 return true;
6429
6430 // TODO: Need is known positive check.
6431 return false;
6432 }
6433 case Intrinsic::amdgcn_trig_preop:
6434 case Intrinsic::amdgcn_fdot2:
6435 // TODO: Refine on operand
6436 return SNaN;
6437 case Intrinsic::amdgcn_fma_legacy:
6438 if (SNaN)
6439 return true;
6440 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6441 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1) &&
6442 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 3), SNaN, Depth: Depth + 1);
6443 default:
6444 return false;
6445 }
6446 }
6447 default:
6448 return false;
6449 }
6450}
6451
6452bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
6453 Register N0, Register N1) const {
6454 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
6455}
6456