1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This is the parent TargetLowering class for hardware code gen
11/// targets.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUISelLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUMachineFunctionInfo.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUSelectionDAGInfo.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/Analysis.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/Support/AMDGPUAddrSpace.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/KnownBits.h"
30#include "llvm/Target/TargetMachine.h"
31
32using namespace llvm;
33
34#define GET_CALLING_CONV_IMPL
35#include "AMDGPUGenCallingConv.inc"
36
37static cl::opt<bool> AMDGPUBypassSlowDiv(
38 "amdgpu-bypass-slow-div",
39 cl::desc("Skip 64-bit divide for dynamic 32-bit values"),
40 cl::init(Val: true));
41
42// Find a larger type to do a load / store of a vector with.
43EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
44 unsigned StoreSize = VT.getStoreSizeInBits();
45 if (StoreSize <= 32)
46 return EVT::getIntegerVT(Context&: Ctx, BitWidth: StoreSize);
47
48 if (StoreSize % 32 == 0)
49 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i32, NumElements: StoreSize / 32);
50
51 return VT;
52}
53
54unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
55 return DAG.computeKnownBits(Op).countMaxActiveBits();
56}
57
58unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
59 // In order for this to be a signed 24-bit value, bit 23, must
60 // be a sign bit.
61 return DAG.ComputeMaxSignificantBits(Op);
62}
63
64AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
65 const TargetSubtargetInfo &STI,
66 const AMDGPUSubtarget &AMDGPUSTI)
67 : TargetLowering(TM, STI), Subtarget(&AMDGPUSTI) {
68 // Always lower memset, memcpy, and memmove intrinsics to load/store
69 // instructions, rather then generating calls to memset, mempcy or memmove.
70 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U;
71 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U;
72 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U;
73
74 // Enable ganging up loads and stores in the memcpy DAG lowering.
75 MaxGluedStoresPerMemcpy = 16;
76
77 // Lower floating point store/load to integer store/load to reduce the number
78 // of patterns in tablegen.
79 setOperationAction(Op: ISD::LOAD, VT: MVT::f32, Action: Promote);
80 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
81
82 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Promote);
83 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
84
85 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f32, Action: Promote);
86 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
87
88 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f32, Action: Promote);
89 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
90
91 setOperationAction(Op: ISD::LOAD, VT: MVT::v5f32, Action: Promote);
92 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
93
94 setOperationAction(Op: ISD::LOAD, VT: MVT::v6f32, Action: Promote);
95 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
96
97 setOperationAction(Op: ISD::LOAD, VT: MVT::v7f32, Action: Promote);
98 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
99
100 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f32, Action: Promote);
101 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
102
103 setOperationAction(Op: ISD::LOAD, VT: MVT::v9f32, Action: Promote);
104 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
105
106 setOperationAction(Op: ISD::LOAD, VT: MVT::v10f32, Action: Promote);
107 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
108
109 setOperationAction(Op: ISD::LOAD, VT: MVT::v11f32, Action: Promote);
110 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
111
112 setOperationAction(Op: ISD::LOAD, VT: MVT::v12f32, Action: Promote);
113 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
114
115 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f32, Action: Promote);
116 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
117
118 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f32, Action: Promote);
119 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
120
121 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Promote);
122 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i64, DestVT: MVT::v2i32);
123
124 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
125 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
126
127 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Promote);
128 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f64, DestVT: MVT::v2i32);
129
130 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Promote);
131 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
132
133 setOperationAction(Op: ISD::LOAD, VT: MVT::v3i64, Action: Promote);
134 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
135
136 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i64, Action: Promote);
137 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
138
139 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f64, Action: Promote);
140 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
141
142 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f64, Action: Promote);
143 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
144
145 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i64, Action: Promote);
146 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
147
148 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f64, Action: Promote);
149 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
150
151 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i64, Action: Promote);
152 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
153
154 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f64, Action: Promote);
155 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
156
157 setOperationAction(Op: ISD::LOAD, VT: MVT::i128, Action: Promote);
158 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i128, DestVT: MVT::v4i32);
159
160 // TODO: Would be better to consume as directly legal
161 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::f16, Action: Promote);
162 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::f16, DestVT: MVT::i16);
163
164 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::bf16, Action: Promote);
165 AddPromotedToType(Opc: ISD::ATOMIC_LOAD, OrigVT: MVT::bf16, DestVT: MVT::i16);
166
167 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::f16, Action: Promote);
168 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::f16, DestVT: MVT::i16);
169
170 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::bf16, Action: Promote);
171 AddPromotedToType(Opc: ISD::ATOMIC_STORE, OrigVT: MVT::bf16, DestVT: MVT::i16);
172
173 // There are no 64-bit extloads. These should be done as a 32-bit extload and
174 // an extension to 64-bit.
175 for (MVT VT : MVT::integer_valuetypes())
176 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64, MemVT: VT,
177 Action: Expand);
178
179 for (MVT VT : MVT::integer_valuetypes()) {
180 if (VT == MVT::i64)
181 continue;
182
183 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) {
184 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i1, Action: Promote);
185 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i8, Action: Legal);
186 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i16, Action: Legal);
187 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i32, Action: Expand);
188 }
189 }
190
191 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
192 for (auto MemVT :
193 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
194 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT,
195 Action: Expand);
196
197 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
198 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
199 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
200 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
201 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
202 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
203 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
204 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
205 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
206 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
207 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
208 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
209 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
210 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
211
212 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
213 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
214 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
215 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
216 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
218
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
220 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
221 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
222 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
223 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
224 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
225 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
226 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
227 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
228 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
229 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
230 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
231
232 setOperationAction(Op: ISD::STORE, VT: MVT::f32, Action: Promote);
233 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
234
235 setOperationAction(Op: ISD::STORE, VT: MVT::v2f32, Action: Promote);
236 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
237
238 setOperationAction(Op: ISD::STORE, VT: MVT::v3f32, Action: Promote);
239 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
240
241 setOperationAction(Op: ISD::STORE, VT: MVT::v4f32, Action: Promote);
242 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
243
244 setOperationAction(Op: ISD::STORE, VT: MVT::v5f32, Action: Promote);
245 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
246
247 setOperationAction(Op: ISD::STORE, VT: MVT::v6f32, Action: Promote);
248 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
249
250 setOperationAction(Op: ISD::STORE, VT: MVT::v7f32, Action: Promote);
251 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
252
253 setOperationAction(Op: ISD::STORE, VT: MVT::v8f32, Action: Promote);
254 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
255
256 setOperationAction(Op: ISD::STORE, VT: MVT::v9f32, Action: Promote);
257 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
258
259 setOperationAction(Op: ISD::STORE, VT: MVT::v10f32, Action: Promote);
260 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
261
262 setOperationAction(Op: ISD::STORE, VT: MVT::v11f32, Action: Promote);
263 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
264
265 setOperationAction(Op: ISD::STORE, VT: MVT::v12f32, Action: Promote);
266 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
267
268 setOperationAction(Op: ISD::STORE, VT: MVT::v16f32, Action: Promote);
269 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
270
271 setOperationAction(Op: ISD::STORE, VT: MVT::v32f32, Action: Promote);
272 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
273
274 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Promote);
275 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i64, DestVT: MVT::v2i32);
276
277 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
278 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
279
280 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Promote);
281 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f64, DestVT: MVT::v2i32);
282
283 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Promote);
284 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
285
286 setOperationAction(Op: ISD::STORE, VT: MVT::v3i64, Action: Promote);
287 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
288
289 setOperationAction(Op: ISD::STORE, VT: MVT::v3f64, Action: Promote);
290 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
291
292 setOperationAction(Op: ISD::STORE, VT: MVT::v4i64, Action: Promote);
293 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
294
295 setOperationAction(Op: ISD::STORE, VT: MVT::v4f64, Action: Promote);
296 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
297
298 setOperationAction(Op: ISD::STORE, VT: MVT::v8i64, Action: Promote);
299 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
300
301 setOperationAction(Op: ISD::STORE, VT: MVT::v8f64, Action: Promote);
302 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
303
304 setOperationAction(Op: ISD::STORE, VT: MVT::v16i64, Action: Promote);
305 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
306
307 setOperationAction(Op: ISD::STORE, VT: MVT::v16f64, Action: Promote);
308 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
309
310 setOperationAction(Op: ISD::STORE, VT: MVT::i128, Action: Promote);
311 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i128, DestVT: MVT::v4i32);
312
313 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i1, Action: Expand);
314 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i8, Action: Expand);
315 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
316 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i32, Action: Expand);
317
318 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i1, Action: Expand);
319 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i8, Action: Expand);
320 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i16, Action: Expand);
321 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i32, Action: Expand);
322
323 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
324 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
325 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
326 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
327 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
328 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
329 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
330 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
331 setTruncStoreAction(ValVT: MVT::v6f32, MemVT: MVT::v6f16, Action: Expand);
332 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
333 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
334 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
335 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
336 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
337 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
338
339 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
340 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
341 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
342
343 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
344 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
345 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
346
347 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i8, Action: Expand);
348
349 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
350 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
351 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i8, Action: Expand);
352 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i1, Action: Expand);
353 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
354 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
355 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
356
357 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i32, Action: Expand);
358 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i16, Action: Expand);
359 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
360 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
361 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
362
363 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i1, Action: Expand);
364 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i8, Action: Expand);
365 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i16, Action: Expand);
366
367 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i1, Action: Expand);
368 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i8, Action: Expand);
369 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i16, Action: Expand);
370
371 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i1, Action: Expand);
372 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i8, Action: Expand);
373 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i16, Action: Expand);
374
375 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
376 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
377 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
378
379 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
380 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
381 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
382 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i16, Action: Expand);
383 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
384 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
385 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i1, Action: Expand);
386
387 setOperationAction(Ops: ISD::Constant, VTs: {MVT::i32, MVT::i64}, Action: Legal);
388 setOperationAction(Ops: ISD::ConstantFP, VTs: {MVT::f32, MVT::f64}, Action: Legal);
389
390 setOperationAction(Ops: {ISD::BR_JT, ISD::BRIND}, VT: MVT::Other, Action: Expand);
391
392 // For R600, this is totally unsupported, just custom lower to produce an
393 // error.
394 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
395
396 // Library functions. These default to Expand, but we have instructions
397 // for them.
398 setOperationAction(
399 Ops: {ISD::FCEIL, ISD::FABS, ISD::FFLOOR, ISD::FROUNDEVEN, ISD::FTRUNC},
400 VTs: {MVT::f16, MVT::f32}, Action: Legal);
401 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM}, VT: MVT::f32, Action: Legal);
402
403 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Custom);
404 setOperationAction(Ops: ISD::FROUND, VTs: {MVT::f32, MVT::f64}, Action: Custom);
405 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND},
406 VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
407
408 setOperationAction(
409 Ops: {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10, ISD::FPOW},
410 VT: MVT::f32, Action: Custom);
411 setOperationAction(Ops: {ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, VT: MVT::f64, Action: Custom);
412
413 setOperationAction(Ops: ISD::FNEARBYINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
414
415 setOperationAction(Ops: ISD::FRINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
416
417 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VTs: {MVT::f16, MVT::f32, MVT::f64},
418 Action: Expand);
419
420 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
421 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
422 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Custom);
423
424 setOperationAction(Ops: {ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, VT: MVT::f16,
425 Action: Custom);
426
427 setOperationAction(Ops: ISD::FCANONICALIZE, VTs: {MVT::f32, MVT::f64}, Action: Legal);
428
429 // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
430 // scalarization code. Can be removed when IS_FPCLASS expand isn't called by
431 // default unless marked custom/legal.
432 setOperationAction(Ops: ISD::IS_FPCLASS,
433 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
434 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
435 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64,
436 MVT::v16f64},
437 Action: Custom);
438
439 // Expand to fneg + fadd.
440 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
441
442 setOperationAction(Ops: ISD::CONCAT_VECTORS,
443 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
444 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
445 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
446 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
447 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
448 Action: Custom);
449
450 setOperationAction(
451 Ops: ISD::EXTRACT_SUBVECTOR,
452 VTs: {MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32,
453 MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32,
454 MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32,
455 MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32,
456 MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
457 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
458 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64},
459 Action: Custom);
460
461 setOperationAction(Ops: {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP}, VT: MVT::f64,
462 Action: Expand);
463 setOperationAction(Ops: ISD::FP_TO_FP16, VTs: {MVT::f64, MVT::f32}, Action: Custom);
464
465 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
466 for (MVT VT : ScalarIntVTs) {
467 // These should use [SU]DIVREM, so set them to expand
468 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT,
469 Action: Expand);
470
471 // GPU does not have divrem function for signed or unsigned.
472 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT, Action: Custom);
473
474 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
475 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
476
477 setOperationAction(Ops: {ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Action: Expand);
478
479 setOperationAction(Ops: {ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT,
480 Action: Expand);
481 }
482
483 // The hardware supports 32-bit FSHR, but not FSHL.
484 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
485
486 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VTs: {MVT::i32, MVT::i64}, Action: Expand);
487
488 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT: MVT::i16, Action: Expand);
489
490 setOperationAction(Ops: {ISD::MUL, ISD::MULHU, ISD::MULHS}, VT: MVT::i64, Action: Expand);
491 setOperationAction(Ops: {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT,
492 ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
493 ISD::FP_TO_UINT_SAT},
494 VT: MVT::i64, Action: Custom);
495 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: Expand);
496
497 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i32,
498 Action: Legal);
499
500 setOperationAction(
501 Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON},
502 VT: MVT::i64, Action: Custom);
503
504 for (auto VT : {MVT::i8, MVT::i16})
505 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT, Action: Custom);
506
507 static const MVT::SimpleValueType VectorIntTypes[] = {
508 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
509 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
510
511 for (MVT VT : VectorIntTypes) {
512 // Expand the following operations for the current type by default.
513 // clang-format off
514 setOperationAction(Ops: {ISD::ADD, ISD::AND,
515 ISD::FP_TO_SINT, ISD::FP_TO_UINT,
516 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
517 ISD::MUL, ISD::MULHU,
518 ISD::MULHS, ISD::OR,
519 ISD::SHL, ISD::SRA,
520 ISD::SRL, ISD::ROTL,
521 ISD::ROTR, ISD::SUB,
522 ISD::SINT_TO_FP, ISD::UINT_TO_FP,
523 ISD::SDIV, ISD::UDIV,
524 ISD::SREM, ISD::UREM,
525 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
526 ISD::SDIVREM, ISD::UDIVREM,
527 ISD::SELECT, ISD::VSELECT,
528 ISD::SELECT_CC, ISD::XOR,
529 ISD::BSWAP, ISD::CTPOP,
530 ISD::CTTZ, ISD::CTLZ,
531 ISD::VECTOR_SHUFFLE, ISD::SETCC,
532 ISD::ADDRSPACECAST},
533 VT, Action: Expand);
534 // clang-format on
535 }
536
537 static const MVT::SimpleValueType FloatVectorTypes[] = {
538 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
539 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
540
541 for (MVT VT : FloatVectorTypes) {
542 setOperationAction(
543 Ops: {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM,
544 ISD::FADD, ISD::FCEIL, ISD::FCOS,
545 ISD::FDIV, ISD::FEXP2, ISD::FEXP,
546 ISD::FEXP10, ISD::FLOG2, ISD::FREM,
547 ISD::FLOG, ISD::FLOG10, ISD::FPOW,
548 ISD::FFLOOR, ISD::FTRUNC, ISD::FMUL,
549 ISD::FMA, ISD::FRINT, ISD::FNEARBYINT,
550 ISD::FSQRT, ISD::FSIN, ISD::FSUB,
551 ISD::FNEG, ISD::VSELECT, ISD::SELECT_CC,
552 ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, ISD::SETCC,
553 ISD::FCANONICALIZE, ISD::FROUNDEVEN},
554 VT, Action: Expand);
555 }
556
557 // This causes using an unrolled select operation rather than expansion with
558 // bit operations. This is in general better, but the alternative using BFI
559 // instructions may be better if the select sources are SGPRs.
560 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f32, Action: Promote);
561 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
562
563 setOperationAction(Op: ISD::SELECT, VT: MVT::v3f32, Action: Promote);
564 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
565
566 setOperationAction(Op: ISD::SELECT, VT: MVT::v4f32, Action: Promote);
567 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
568
569 setOperationAction(Op: ISD::SELECT, VT: MVT::v5f32, Action: Promote);
570 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
571
572 setOperationAction(Op: ISD::SELECT, VT: MVT::v6f32, Action: Promote);
573 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
574
575 setOperationAction(Op: ISD::SELECT, VT: MVT::v7f32, Action: Promote);
576 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
577
578 setOperationAction(Op: ISD::SELECT, VT: MVT::v9f32, Action: Promote);
579 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
580
581 setOperationAction(Op: ISD::SELECT, VT: MVT::v10f32, Action: Promote);
582 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
583
584 setOperationAction(Op: ISD::SELECT, VT: MVT::v11f32, Action: Promote);
585 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
586
587 setOperationAction(Op: ISD::SELECT, VT: MVT::v12f32, Action: Promote);
588 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
589
590 setSchedulingPreference(Sched::RegPressure);
591 setJumpIsExpensive(true);
592
593 setMinCmpXchgSizeInBits(32);
594 setSupportsUnalignedAtomics(false);
595
596 PredictableSelectIsExpensive = false;
597
598 // We want to find all load dependencies for long chains of stores to enable
599 // merging into very wide vectors. The problem is with vectors with > 4
600 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
601 // vectors are a legal type, even though we have to split the loads
602 // usually. When we can more precisely specify load legality per address
603 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
604 // smarter so that they can figure out what to do in 2 iterations without all
605 // N > 4 stores on the same chain.
606 GatherAllAliasesMaxDepth = 16;
607
608 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
609 // about these during lowering.
610 MaxStoresPerMemcpy = 0xffffffff;
611 MaxStoresPerMemmove = 0xffffffff;
612 MaxStoresPerMemset = 0xffffffff;
613
614 // The expansion for 64-bit division is enormous.
615 if (AMDGPUBypassSlowDiv)
616 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
617
618 setTargetDAGCombine({ISD::BITCAST, ISD::SHL,
619 ISD::SRA, ISD::SRL,
620 ISD::TRUNCATE, ISD::MUL,
621 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
622 ISD::MULHU, ISD::MULHS,
623 ISD::SELECT, ISD::SELECT_CC,
624 ISD::STORE, ISD::FADD,
625 ISD::FSUB, ISD::FNEG,
626 ISD::FABS, ISD::AssertZext,
627 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN});
628
629 setMaxAtomicSizeInBitsSupported(64);
630 setMaxDivRemBitWidthSupported(64);
631 setMaxLargeFPConvertBitWidthSupported(64);
632}
633
634//===----------------------------------------------------------------------===//
635// Target Information
636//===----------------------------------------------------------------------===//
637
638LLVM_READNONE
639static bool fnegFoldsIntoOpcode(unsigned Opc) {
640 switch (Opc) {
641 case ISD::FADD:
642 case ISD::FSUB:
643 case ISD::FMUL:
644 case ISD::FMA:
645 case ISD::FMAD:
646 case ISD::FMINNUM:
647 case ISD::FMAXNUM:
648 case ISD::FMINNUM_IEEE:
649 case ISD::FMAXNUM_IEEE:
650 case ISD::FMINIMUM:
651 case ISD::FMAXIMUM:
652 case ISD::FMINIMUMNUM:
653 case ISD::FMAXIMUMNUM:
654 case ISD::SELECT:
655 case ISD::FSIN:
656 case ISD::FTRUNC:
657 case ISD::FRINT:
658 case ISD::FNEARBYINT:
659 case ISD::FROUNDEVEN:
660 case ISD::FCANONICALIZE:
661 case AMDGPUISD::RCP:
662 case AMDGPUISD::RCP_LEGACY:
663 case AMDGPUISD::RCP_IFLAG:
664 case AMDGPUISD::SIN_HW:
665 case AMDGPUISD::FMUL_LEGACY:
666 case AMDGPUISD::FMIN_LEGACY:
667 case AMDGPUISD::FMAX_LEGACY:
668 case AMDGPUISD::FMED3:
669 // TODO: handle llvm.amdgcn.fma.legacy
670 return true;
671 case ISD::BITCAST:
672 llvm_unreachable("bitcast is special cased");
673 default:
674 return false;
675 }
676}
677
678static bool fnegFoldsIntoOp(const SDNode *N) {
679 unsigned Opc = N->getOpcode();
680 if (Opc == ISD::BITCAST) {
681 // TODO: Is there a benefit to checking the conditions performFNegCombine
682 // does? We don't for the other cases.
683 SDValue BCSrc = N->getOperand(Num: 0);
684 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
685 return BCSrc.getNumOperands() == 2 &&
686 BCSrc.getOperand(i: 1).getValueSizeInBits() == 32;
687 }
688
689 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32;
690 }
691
692 return fnegFoldsIntoOpcode(Opc);
693}
694
695/// \p returns true if the operation will definitely need to use a 64-bit
696/// encoding, and thus will use a VOP3 encoding regardless of the source
697/// modifiers.
698LLVM_READONLY
699static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
700 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) ||
701 VT == MVT::f64;
702}
703
704/// Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the
705/// type for ISD::SELECT.
706LLVM_READONLY
707static bool selectSupportsSourceMods(const SDNode *N) {
708 // TODO: Only applies if select will be vector
709 return N->getValueType(ResNo: 0) == MVT::f32;
710}
711
712// Most FP instructions support source modifiers, but this could be refined
713// slightly.
714LLVM_READONLY
715static bool hasSourceMods(const SDNode *N) {
716 if (isa<MemSDNode>(Val: N))
717 return false;
718
719 switch (N->getOpcode()) {
720 case ISD::CopyToReg:
721 case ISD::FDIV:
722 case ISD::FREM:
723 case ISD::INLINEASM:
724 case ISD::INLINEASM_BR:
725 case AMDGPUISD::DIV_SCALE:
726 case ISD::INTRINSIC_W_CHAIN:
727
728 // TODO: Should really be looking at the users of the bitcast. These are
729 // problematic because bitcasts are used to legalize all stores to integer
730 // types.
731 case ISD::BITCAST:
732 return false;
733 case ISD::INTRINSIC_WO_CHAIN: {
734 switch (N->getConstantOperandVal(Num: 0)) {
735 case Intrinsic::amdgcn_interp_p1:
736 case Intrinsic::amdgcn_interp_p2:
737 case Intrinsic::amdgcn_interp_mov:
738 case Intrinsic::amdgcn_interp_p1_f16:
739 case Intrinsic::amdgcn_interp_p2_f16:
740 return false;
741 default:
742 return true;
743 }
744 }
745 case ISD::SELECT:
746 return selectSupportsSourceMods(N);
747 default:
748 return true;
749 }
750}
751
752bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N,
753 unsigned CostThreshold) {
754 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
755 // it is truly free to use a source modifier in all cases. If there are
756 // multiple users but for each one will necessitate using VOP3, there will be
757 // a code size increase. Try to avoid increasing code size unless we know it
758 // will save on the instruction count.
759 unsigned NumMayIncreaseSize = 0;
760 MVT VT = N->getValueType(ResNo: 0).getScalarType().getSimpleVT();
761
762 assert(!N->use_empty());
763
764 // XXX - Should this limit number of uses to check?
765 for (const SDNode *U : N->users()) {
766 if (!hasSourceMods(N: U))
767 return false;
768
769 if (!opMustUseVOP3Encoding(N: U, VT)) {
770 if (++NumMayIncreaseSize > CostThreshold)
771 return false;
772 }
773 }
774
775 return true;
776}
777
778EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
779 ISD::NodeType ExtendKind) const {
780 assert(!VT.isVector() && "only scalar expected");
781
782 // Round to the next multiple of 32-bits.
783 unsigned Size = VT.getSizeInBits();
784 if (Size <= 32)
785 return MVT::i32;
786 return EVT::getIntegerVT(Context, BitWidth: 32 * ((Size + 31) / 32));
787}
788
789unsigned AMDGPUTargetLowering::getVectorIdxWidth(const DataLayout &) const {
790 return 32;
791}
792
793bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
794 return true;
795}
796
797// The backend supports 32 and 64 bit floating point immediates.
798// FIXME: Why are we reporting vectors of FP immediates as legal?
799bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
800 bool ForCodeSize) const {
801 return isTypeLegal(VT: VT.getScalarType());
802}
803
804// We don't want to shrink f64 / f32 constants.
805bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
806 EVT ScalarVT = VT.getScalarType();
807 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
808}
809
810bool AMDGPUTargetLowering::shouldReduceLoadWidth(
811 SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
812 std::optional<unsigned> ByteOffset) const {
813 // TODO: This may be worth removing. Check regression tests for diffs.
814 if (!TargetLoweringBase::shouldReduceLoadWidth(Load: N, ExtTy, NewVT, ByteOffset))
815 return false;
816
817 unsigned NewSize = NewVT.getStoreSizeInBits();
818
819 // If we are reducing to a 32-bit load or a smaller multi-dword load,
820 // this is always better.
821 if (NewSize >= 32)
822 return true;
823
824 EVT OldVT = N->getValueType(ResNo: 0);
825 unsigned OldSize = OldVT.getStoreSizeInBits();
826
827 MemSDNode *MN = cast<MemSDNode>(Val: N);
828 unsigned AS = MN->getAddressSpace();
829 // Do not shrink an aligned scalar load to sub-dword.
830 // Scalar engine cannot do sub-dword loads.
831 // Do not enable for gfx1250+ even though it has sub-dword loads because
832 // this will convert:
833 // i16 = trunc (zextload i16->i32)
834 // to:
835 // i16 = (load i16)
836 // This transformation will be reversed by LowerLOAD resulting in an infinite
837 // loop. Also, tablegen already has a pattern to match zextload i16->i32, but
838 // load i16 will not be matched since there is no instruction that does it.
839 if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) &&
840 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
841 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
842 (isa<LoadSDNode>(Val: N) && AS == AMDGPUAS::GLOBAL_ADDRESS &&
843 MN->isInvariant())) &&
844 AMDGPU::isUniformMMO(MMO: MN->getMemOperand()))
845 return false;
846
847 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
848 // extloads, so doing one requires using a buffer_load. In cases where we
849 // still couldn't use a scalar load, using the wider load shouldn't really
850 // hurt anything.
851
852 // If the old size already had to be an extload, there's no harm in continuing
853 // to reduce the width.
854 return (OldSize < 32);
855}
856
857bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
858 const SelectionDAG &DAG,
859 const MachineMemOperand &MMO) const {
860
861 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
862
863 if (LoadTy.getScalarType() == MVT::i32)
864 return false;
865
866 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
867 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
868
869 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
870 return false;
871
872 unsigned Fast = 0;
873 return allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
874 VT: CastTy, MMO, Fast: &Fast) &&
875 Fast;
876}
877
878// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
879// profitable with the expansion for 64-bit since it's generally good to
880// speculate things.
881bool AMDGPUTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
882 return true;
883}
884
885bool AMDGPUTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
886 return true;
887}
888
889bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const {
890 switch (N->getOpcode()) {
891 case ISD::EntryToken:
892 case ISD::TokenFactor:
893 return true;
894 case ISD::INTRINSIC_WO_CHAIN: {
895 unsigned IntrID = N->getConstantOperandVal(Num: 0);
896 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
897 }
898 case ISD::INTRINSIC_W_CHAIN: {
899 unsigned IntrID = N->getConstantOperandVal(Num: 1);
900 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
901 }
902 case ISD::LOAD:
903 if (cast<LoadSDNode>(Val: N)->getMemOperand()->getAddrSpace() ==
904 AMDGPUAS::CONSTANT_ADDRESS_32BIT)
905 return true;
906 return false;
907 case AMDGPUISD::SETCC: // ballot-style instruction
908 return true;
909 }
910 return false;
911}
912
913SDValue AMDGPUTargetLowering::getNegatedExpression(
914 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
915 NegatibleCost &Cost, unsigned Depth) const {
916
917 switch (Op.getOpcode()) {
918 case ISD::FMA:
919 case ISD::FMAD: {
920 // Negating a fma is not free if it has users without source mods.
921 if (!allUsesHaveSourceMods(N: Op.getNode()))
922 return SDValue();
923 break;
924 }
925 case AMDGPUISD::RCP: {
926 SDValue Src = Op.getOperand(i: 0);
927 EVT VT = Op.getValueType();
928 SDLoc SL(Op);
929
930 SDValue NegSrc = getNegatedExpression(Op: Src, DAG, LegalOperations,
931 ForCodeSize, Cost, Depth: Depth + 1);
932 if (NegSrc)
933 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: NegSrc, Flags: Op->getFlags());
934 return SDValue();
935 }
936 default:
937 break;
938 }
939
940 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps: LegalOperations,
941 OptForSize: ForCodeSize, Cost, Depth);
942}
943
944//===---------------------------------------------------------------------===//
945// Target Properties
946//===---------------------------------------------------------------------===//
947
948bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
949 assert(VT.isFloatingPoint());
950
951 // Packed operations do not have a fabs modifier.
952 // Report this based on the end legalized type.
953 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
954}
955
956bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
957 assert(VT.isFloatingPoint());
958 // Report this based on the end legalized type.
959 VT = VT.getScalarType();
960 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
961}
962
963bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
964 unsigned NumElem,
965 unsigned AS) const {
966 return true;
967}
968
969bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
970 // There are few operations which truly have vector input operands. Any vector
971 // operation is going to involve operations on each component, and a
972 // build_vector will be a copy per element, so it always makes sense to use a
973 // build_vector input in place of the extracted element to avoid a copy into a
974 // super register.
975 //
976 // We should probably only do this if all users are extracts only, but this
977 // should be the common case.
978 return true;
979}
980
981bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
982 // Truncate is just accessing a subregister.
983
984 unsigned SrcSize = Source.getSizeInBits();
985 unsigned DestSize = Dest.getSizeInBits();
986
987 return DestSize < SrcSize && DestSize % 32 == 0 ;
988}
989
990bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
991 // Truncate is just accessing a subregister.
992
993 unsigned SrcSize = Source->getScalarSizeInBits();
994 unsigned DestSize = Dest->getScalarSizeInBits();
995
996 if (DestSize== 16 && Subtarget->has16BitInsts())
997 return SrcSize >= 32;
998
999 return DestSize < SrcSize && DestSize % 32 == 0;
1000}
1001
1002bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
1003 unsigned SrcSize = Src->getScalarSizeInBits();
1004 unsigned DestSize = Dest->getScalarSizeInBits();
1005
1006 if (SrcSize == 16 && Subtarget->has16BitInsts())
1007 return DestSize >= 32;
1008
1009 return SrcSize == 32 && DestSize == 64;
1010}
1011
1012bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
1013 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
1014 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
1015 // this will enable reducing 64-bit operations the 32-bit, which is always
1016 // good.
1017
1018 if (Src == MVT::i16)
1019 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1020
1021 return Src == MVT::i32 && Dest == MVT::i64;
1022}
1023
1024bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
1025 EVT DestVT) const {
1026 switch (N->getOpcode()) {
1027 case ISD::ABS:
1028 case ISD::ADD:
1029 case ISD::SUB:
1030 case ISD::SHL:
1031 case ISD::SRL:
1032 case ISD::SRA:
1033 case ISD::AND:
1034 case ISD::OR:
1035 case ISD::XOR:
1036 case ISD::MUL:
1037 case ISD::SETCC:
1038 case ISD::SELECT:
1039 case ISD::SMIN:
1040 case ISD::SMAX:
1041 case ISD::UMIN:
1042 case ISD::UMAX:
1043 case ISD::USUBSAT:
1044 case ISD::UADDSAT:
1045 if (isTypeLegal(VT: MVT::i16) &&
1046 (!DestVT.isVector() ||
1047 !isOperationLegal(Op: ISD::ADD, VT: MVT::v2i16))) { // Check if VOP3P
1048 // Don't narrow back down to i16 if promoted to i32 already.
1049 if (!N->isDivergent() && DestVT.isInteger() &&
1050 DestVT.getScalarSizeInBits() > 1 &&
1051 DestVT.getScalarSizeInBits() <= 16 &&
1052 SrcVT.getScalarSizeInBits() > 16) {
1053 return false;
1054 }
1055 }
1056 return true;
1057 default:
1058 break;
1059 }
1060
1061 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
1062 // limited number of native 64-bit operations. Shrinking an operation to fit
1063 // in a single 32-bit register should always be helpful. As currently used,
1064 // this is much less general than the name suggests, and is only used in
1065 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
1066 // not profitable, and may actually be harmful.
1067 if (isa<LoadSDNode>(Val: N))
1068 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
1069
1070 return true;
1071}
1072
1073bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
1074 const SDNode* N, CombineLevel Level) const {
1075 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
1076 N->getOpcode() == ISD::SRL) &&
1077 "Expected shift op");
1078
1079 SDValue ShiftLHS = N->getOperand(Num: 0);
1080 if (!ShiftLHS->hasOneUse())
1081 return false;
1082
1083 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
1084 !ShiftLHS.getOperand(i: 0)->hasOneUse())
1085 return false;
1086
1087 // Always commute pre-type legalization and right shifts.
1088 // We're looking for shl(or(x,y),z) patterns.
1089 if (Level < CombineLevel::AfterLegalizeTypes ||
1090 N->getOpcode() != ISD::SHL || N->getOperand(Num: 0).getOpcode() != ISD::OR)
1091 return true;
1092
1093 // If only user is a i32 right-shift, then don't destroy a BFE pattern.
1094 if (N->getValueType(ResNo: 0) == MVT::i32 && N->hasOneUse() &&
1095 (N->user_begin()->getOpcode() == ISD::SRA ||
1096 N->user_begin()->getOpcode() == ISD::SRL))
1097 return false;
1098
1099 // Don't destroy or(shl(load_zext(),c), load_zext()) patterns.
1100 auto IsShiftAndLoad = [](SDValue LHS, SDValue RHS) {
1101 if (LHS.getOpcode() != ISD::SHL)
1102 return false;
1103 auto *RHSLd = dyn_cast<LoadSDNode>(Val&: RHS);
1104 auto *LHS0 = dyn_cast<LoadSDNode>(Val: LHS.getOperand(i: 0));
1105 auto *LHS1 = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
1106 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD &&
1107 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1108 RHSLd->getExtensionType() == ISD::ZEXTLOAD;
1109 };
1110 SDValue LHS = N->getOperand(Num: 0).getOperand(i: 0);
1111 SDValue RHS = N->getOperand(Num: 0).getOperand(i: 1);
1112 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS));
1113}
1114
1115//===---------------------------------------------------------------------===//
1116// TargetLowering Callbacks
1117//===---------------------------------------------------------------------===//
1118
1119CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
1120 bool IsVarArg) {
1121 switch (CC) {
1122 case CallingConv::AMDGPU_VS:
1123 case CallingConv::AMDGPU_GS:
1124 case CallingConv::AMDGPU_PS:
1125 case CallingConv::AMDGPU_CS:
1126 case CallingConv::AMDGPU_HS:
1127 case CallingConv::AMDGPU_ES:
1128 case CallingConv::AMDGPU_LS:
1129 return CC_AMDGPU;
1130 case CallingConv::AMDGPU_CS_Chain:
1131 case CallingConv::AMDGPU_CS_ChainPreserve:
1132 return CC_AMDGPU_CS_CHAIN;
1133 case CallingConv::C:
1134 case CallingConv::Fast:
1135 case CallingConv::Cold:
1136 return CC_AMDGPU_Func;
1137 case CallingConv::AMDGPU_Gfx:
1138 case CallingConv::AMDGPU_Gfx_WholeWave:
1139 return CC_SI_Gfx;
1140 case CallingConv::AMDGPU_KERNEL:
1141 case CallingConv::SPIR_KERNEL:
1142 default:
1143 reportFatalUsageError(reason: "unsupported calling convention for call");
1144 }
1145}
1146
1147CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
1148 bool IsVarArg) {
1149 switch (CC) {
1150 case CallingConv::AMDGPU_KERNEL:
1151 case CallingConv::SPIR_KERNEL:
1152 llvm_unreachable("kernels should not be handled here");
1153 case CallingConv::AMDGPU_VS:
1154 case CallingConv::AMDGPU_GS:
1155 case CallingConv::AMDGPU_PS:
1156 case CallingConv::AMDGPU_CS:
1157 case CallingConv::AMDGPU_CS_Chain:
1158 case CallingConv::AMDGPU_CS_ChainPreserve:
1159 case CallingConv::AMDGPU_HS:
1160 case CallingConv::AMDGPU_ES:
1161 case CallingConv::AMDGPU_LS:
1162 return RetCC_SI_Shader;
1163 case CallingConv::AMDGPU_Gfx:
1164 case CallingConv::AMDGPU_Gfx_WholeWave:
1165 return RetCC_SI_Gfx;
1166 case CallingConv::C:
1167 case CallingConv::Fast:
1168 case CallingConv::Cold:
1169 return RetCC_AMDGPU_Func;
1170 default:
1171 reportFatalUsageError(reason: "unsupported calling convention");
1172 }
1173}
1174
1175/// The SelectionDAGBuilder will automatically promote function arguments
1176/// with illegal types. However, this does not work for the AMDGPU targets
1177/// since the function arguments are stored in memory as these illegal types.
1178/// In order to handle this properly we need to get the original types sizes
1179/// from the LLVM IR Function and fixup the ISD:InputArg values before
1180/// passing them to AnalyzeFormalArguments()
1181
1182/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
1183/// input values across multiple registers. Each item in the Ins array
1184/// represents a single value that will be stored in registers. Ins[x].VT is
1185/// the value type of the value that will be stored in the register, so
1186/// whatever SDNode we lower the argument to needs to be this type.
1187///
1188/// In order to correctly lower the arguments we need to know the size of each
1189/// argument. Since Ins[x].VT gives us the size of the register that will
1190/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
1191/// for the original function argument so that we can deduce the correct memory
1192/// type to use for Ins[x]. In most cases the correct memory type will be
1193/// Ins[x].ArgVT. However, this will not always be the case. If, for example,
1194/// we have a kernel argument of type v8i8, this argument will be split into
1195/// 8 parts and each part will be represented by its own item in the Ins array.
1196/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
1197/// the argument before it was split. From this, we deduce that the memory type
1198/// for each individual part is i8. We pass the memory type as LocVT to the
1199/// calling convention analysis function and the register type (Ins[x].VT) as
1200/// the ValVT.
1201void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
1202 CCState &State,
1203 const SmallVectorImpl<ISD::InputArg> &Ins) const {
1204 const MachineFunction &MF = State.getMachineFunction();
1205 const Function &Fn = MF.getFunction();
1206 LLVMContext &Ctx = Fn.getContext();
1207 const unsigned ExplicitOffset = Subtarget->getExplicitKernelArgOffset();
1208 CallingConv::ID CC = Fn.getCallingConv();
1209
1210 Align MaxAlign = Align(1);
1211 uint64_t ExplicitArgOffset = 0;
1212 const DataLayout &DL = Fn.getDataLayout();
1213
1214 unsigned InIndex = 0;
1215
1216 for (const Argument &Arg : Fn.args()) {
1217 const bool IsByRef = Arg.hasByRefAttr();
1218 Type *BaseArgTy = Arg.getType();
1219 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1220 Align Alignment = DL.getValueOrABITypeAlignment(
1221 Alignment: IsByRef ? Arg.getParamAlign() : std::nullopt, Ty: MemArgTy);
1222 MaxAlign = std::max(a: Alignment, b: MaxAlign);
1223 uint64_t AllocSize = DL.getTypeAllocSize(Ty: MemArgTy);
1224
1225 uint64_t ArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + ExplicitOffset;
1226 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + AllocSize;
1227
1228 // We're basically throwing away everything passed into us and starting over
1229 // to get accurate in-memory offsets. The "PartOffset" is completely useless
1230 // to us as computed in Ins.
1231 //
1232 // We also need to figure out what type legalization is trying to do to get
1233 // the correct memory offsets.
1234
1235 SmallVector<EVT, 16> ValueVTs;
1236 SmallVector<uint64_t, 16> Offsets;
1237 ComputeValueVTs(TLI: *this, DL, Ty: BaseArgTy, ValueVTs, /*MemVTs=*/nullptr,
1238 FixedOffsets: &Offsets, StartingOffset: ArgOffset);
1239
1240 for (unsigned Value = 0, NumValues = ValueVTs.size();
1241 Value != NumValues; ++Value) {
1242 uint64_t BasePartOffset = Offsets[Value];
1243
1244 EVT ArgVT = ValueVTs[Value];
1245 EVT MemVT = ArgVT;
1246 MVT RegisterVT = getRegisterTypeForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1247 unsigned NumRegs = getNumRegistersForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1248
1249 if (NumRegs == 1) {
1250 // This argument is not split, so the IR type is the memory type.
1251 if (ArgVT.isExtended()) {
1252 // We have an extended type, like i24, so we should just use the
1253 // register type.
1254 MemVT = RegisterVT;
1255 } else {
1256 MemVT = ArgVT;
1257 }
1258 } else if (ArgVT.isVector() && RegisterVT.isVector() &&
1259 ArgVT.getScalarType() == RegisterVT.getScalarType()) {
1260 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
1261 // We have a vector value which has been split into a vector with
1262 // the same scalar type, but fewer elements. This should handle
1263 // all the floating-point vector types.
1264 MemVT = RegisterVT;
1265 } else if (ArgVT.isVector() &&
1266 ArgVT.getVectorNumElements() == NumRegs) {
1267 // This arg has been split so that each element is stored in a separate
1268 // register.
1269 MemVT = ArgVT.getScalarType();
1270 } else if (ArgVT.isExtended()) {
1271 // We have an extended type, like i65.
1272 MemVT = RegisterVT;
1273 } else {
1274 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
1275 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0);
1276 if (RegisterVT.isInteger()) {
1277 MemVT = EVT::getIntegerVT(Context&: State.getContext(), BitWidth: MemoryBits);
1278 } else if (RegisterVT.isVector()) {
1279 assert(!RegisterVT.getScalarType().isFloatingPoint());
1280 unsigned NumElements = RegisterVT.getVectorNumElements();
1281 assert(MemoryBits % NumElements == 0);
1282 // This vector type has been split into another vector type with
1283 // a different elements size.
1284 EVT ScalarVT = EVT::getIntegerVT(Context&: State.getContext(),
1285 BitWidth: MemoryBits / NumElements);
1286 MemVT = EVT::getVectorVT(Context&: State.getContext(), VT: ScalarVT, NumElements);
1287 } else {
1288 llvm_unreachable("cannot deduce memory type.");
1289 }
1290 }
1291
1292 // Convert one element vectors to scalar.
1293 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
1294 MemVT = MemVT.getScalarType();
1295
1296 // Round up vec3/vec5 argument.
1297 if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
1298 MemVT = MemVT.getPow2VectorType(Context&: State.getContext());
1299 } else if (!MemVT.isSimple() && !MemVT.isVector()) {
1300 MemVT = MemVT.getRoundIntegerType(Context&: State.getContext());
1301 }
1302
1303 unsigned PartOffset = 0;
1304 for (unsigned i = 0; i != NumRegs; ++i) {
1305 State.addLoc(V: CCValAssign::getCustomMem(ValNo: InIndex++, ValVT: RegisterVT,
1306 Offset: BasePartOffset + PartOffset,
1307 LocVT: MemVT.getSimpleVT(),
1308 HTP: CCValAssign::Full));
1309 PartOffset += MemVT.getStoreSize();
1310 }
1311 }
1312 }
1313}
1314
1315SDValue AMDGPUTargetLowering::LowerReturn(
1316 SDValue Chain, CallingConv::ID CallConv,
1317 bool isVarArg,
1318 const SmallVectorImpl<ISD::OutputArg> &Outs,
1319 const SmallVectorImpl<SDValue> &OutVals,
1320 const SDLoc &DL, SelectionDAG &DAG) const {
1321 // FIXME: Fails for r600 tests
1322 //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
1323 // "wave terminate should not have return values");
1324 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM, DL, VT: MVT::Other, Operand: Chain);
1325}
1326
1327//===---------------------------------------------------------------------===//
1328// Target specific lowering
1329//===---------------------------------------------------------------------===//
1330
1331/// Selects the correct CCAssignFn for a given CallingConvention value.
1332CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1333 bool IsVarArg) {
1334 return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1335}
1336
1337CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1338 bool IsVarArg) {
1339 return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1340}
1341
1342SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1343 SelectionDAG &DAG,
1344 MachineFrameInfo &MFI,
1345 int ClobberedFI) const {
1346 SmallVector<SDValue, 8> ArgChains;
1347 int64_t FirstByte = MFI.getObjectOffset(ObjectIdx: ClobberedFI);
1348 int64_t LastByte = FirstByte + MFI.getObjectSize(ObjectIdx: ClobberedFI) - 1;
1349
1350 // Include the original chain at the beginning of the list. When this is
1351 // used by target LowerCall hooks, this helps legalize find the
1352 // CALLSEQ_BEGIN node.
1353 ArgChains.push_back(Elt: Chain);
1354
1355 // Add a chain value for each stack argument corresponding
1356 for (SDNode *U : DAG.getEntryNode().getNode()->users()) {
1357 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U)) {
1358 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr())) {
1359 if (FI->getIndex() < 0) {
1360 int64_t InFirstByte = MFI.getObjectOffset(ObjectIdx: FI->getIndex());
1361 int64_t InLastByte = InFirstByte;
1362 InLastByte += MFI.getObjectSize(ObjectIdx: FI->getIndex()) - 1;
1363
1364 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1365 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1366 ArgChains.push_back(Elt: SDValue(L, 1));
1367 }
1368 }
1369 }
1370 }
1371
1372 // Build a tokenfactor for all the chains.
1373 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
1374}
1375
1376SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1377 SmallVectorImpl<SDValue> &InVals,
1378 StringRef Reason) const {
1379 SDValue Callee = CLI.Callee;
1380 SelectionDAG &DAG = CLI.DAG;
1381
1382 const Function &Fn = DAG.getMachineFunction().getFunction();
1383
1384 StringRef FuncName("<unknown>");
1385
1386 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Val&: Callee))
1387 FuncName = G->getSymbol();
1388 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
1389 FuncName = G->getGlobal()->getName();
1390
1391 DAG.getContext()->diagnose(
1392 DI: DiagnosticInfoUnsupported(Fn, Reason + FuncName, CLI.DL.getDebugLoc()));
1393
1394 if (!CLI.IsTailCall) {
1395 for (ISD::InputArg &Arg : CLI.Ins)
1396 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
1397 }
1398
1399 // FIXME: Hack because R600 doesn't handle callseq pseudos yet.
1400 if (getTargetMachine().getTargetTriple().getArch() == Triple::r600)
1401 return CLI.Chain;
1402
1403 SDValue Chain = DAG.getCALLSEQ_START(Chain: CLI.Chain, InSize: 0, OutSize: 0, DL: CLI.DL);
1404 return DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, /*InGlue=*/Glue: SDValue(), DL: CLI.DL);
1405}
1406
1407SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1408 SmallVectorImpl<SDValue> &InVals) const {
1409 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported call to function ");
1410}
1411
1412SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1413 SelectionDAG &DAG) const {
1414 const Function &Fn = DAG.getMachineFunction().getFunction();
1415
1416 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1417 Fn, "unsupported dynamic alloca", SDLoc(Op).getDebugLoc()));
1418 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()), Op.getOperand(i: 0)};
1419 return DAG.getMergeValues(Ops, dl: SDLoc());
1420}
1421
1422SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1423 SelectionDAG &DAG) const {
1424 switch (Op.getOpcode()) {
1425 default:
1426 Op->print(OS&: errs(), G: &DAG);
1427 llvm_unreachable("Custom lowering code for this "
1428 "instruction is not implemented yet!");
1429 break;
1430 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1431 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1432 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1433 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1434 case ISD::SDIVREM:
1435 return LowerSDIVREM(Op, DAG);
1436 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1437 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1438 case ISD::FRINT: return LowerFRINT(Op, DAG);
1439 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1440 case ISD::FROUNDEVEN:
1441 return LowerFROUNDEVEN(Op, DAG);
1442 case ISD::FROUND: return LowerFROUND(Op, DAG);
1443 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1444 case ISD::FLOG2:
1445 return LowerFLOG2(Op, DAG);
1446 case ISD::FLOG:
1447 case ISD::FLOG10:
1448 return LowerFLOGCommon(Op, DAG);
1449 case ISD::FEXP:
1450 case ISD::FEXP10:
1451 return lowerFEXP(Op, DAG);
1452 case ISD::FEXP2:
1453 return lowerFEXP2(Op, DAG);
1454 case ISD::FPOW:
1455 return lowerFPOW(Op, DAG);
1456 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1457 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1458 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1459 case ISD::FP_TO_SINT:
1460 case ISD::FP_TO_UINT:
1461 return LowerFP_TO_INT(Op, DAG);
1462 case ISD::FP_TO_SINT_SAT:
1463 case ISD::FP_TO_UINT_SAT:
1464 return LowerFP_TO_INT_SAT(Op, DAG);
1465 case ISD::CTTZ:
1466 case ISD::CTTZ_ZERO_POISON:
1467 case ISD::CTLZ:
1468 case ISD::CTLZ_ZERO_POISON:
1469 return LowerCTLZ_CTTZ(Op, DAG);
1470 case ISD::CTLS:
1471 return LowerCTLS(Op, DAG);
1472 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1473 }
1474 return Op;
1475}
1476
1477void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
1478 SmallVectorImpl<SDValue> &Results,
1479 SelectionDAG &DAG) const {
1480 switch (N->getOpcode()) {
1481 case ISD::SIGN_EXTEND_INREG:
1482 // Different parts of legalization seem to interpret which type of
1483 // sign_extend_inreg is the one to check for custom lowering. The extended
1484 // from type is what really matters, but some places check for custom
1485 // lowering of the result type. This results in trying to use
1486 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1487 // nothing here and let the illegal result integer be handled normally.
1488 return;
1489 case ISD::FLOG2:
1490 if (SDValue Lowered = LowerFLOG2(Op: SDValue(N, 0), DAG))
1491 Results.push_back(Elt: Lowered);
1492 return;
1493 case ISD::FLOG:
1494 case ISD::FLOG10:
1495 if (SDValue Lowered = LowerFLOGCommon(Op: SDValue(N, 0), DAG))
1496 Results.push_back(Elt: Lowered);
1497 return;
1498 case ISD::FEXP2:
1499 if (SDValue Lowered = lowerFEXP2(Op: SDValue(N, 0), DAG))
1500 Results.push_back(Elt: Lowered);
1501 return;
1502 case ISD::FEXP:
1503 case ISD::FEXP10:
1504 if (SDValue Lowered = lowerFEXP(Op: SDValue(N, 0), DAG))
1505 Results.push_back(Elt: Lowered);
1506 return;
1507 case ISD::CTLZ:
1508 case ISD::CTLZ_ZERO_POISON:
1509 if (auto Lowered = lowerCTLZResults(Op: SDValue(N, 0u), DAG))
1510 Results.push_back(Elt: Lowered);
1511 return;
1512 default:
1513 return;
1514 }
1515}
1516
1517SDValue AMDGPUTargetLowering::LowerBlockAddress(SDValue Op,
1518 SelectionDAG &DAG) const {
1519 BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val&: Op);
1520 SDLoc SL(Op);
1521 EVT VT = Op.getValueType();
1522 return DAG.getTargetBlockAddress(BA: BA->getBlockAddress(), VT, Offset: BA->getOffset(),
1523 TargetFlags: BA->getTargetFlags());
1524}
1525
1526SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
1527 SDValue Op,
1528 SelectionDAG &DAG) const {
1529
1530 const DataLayout &DL = DAG.getDataLayout();
1531 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Val&: Op);
1532 const GlobalValue *GV = G->getGlobal();
1533
1534 if (G->getAddressSpace() == AMDGPUAS::BARRIER) {
1535 const GlobalVariable *GVar = cast<GlobalVariable>(Val: GV);
1536
1537 if (!AMDGPU::isNamedBarrier(GV: *GVar)) {
1538 const Function &Fn = DAG.getMachineFunction().getFunction();
1539 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1540 Fn, "unsupported use of BARRIER address space",
1541 SDLoc(Op).getDebugLoc(), DS_Error));
1542 return DAG.getPOISON(VT: Op.getValueType());
1543 }
1544
1545 unsigned Offset = MFI->allocateBarrierGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1546 return DAG.getConstant(Val: Offset, DL: SDLoc(Op), VT: Op.getValueType());
1547 }
1548
1549 if (!MFI->isModuleEntryFunction()) {
1550 if (std::optional<uint32_t> Address =
1551 AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(
1552 GV: *GV, AS: AMDGPUAS::LOCAL_ADDRESS)) {
1553 return DAG.getConstant(Val: *Address, DL: SDLoc(Op), VT: Op.getValueType());
1554 }
1555 }
1556
1557 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1558 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
1559 if (!MFI->isModuleEntryFunction() &&
1560 GV->getName() != "llvm.amdgcn.module.lds") {
1561 SDLoc DL(Op);
1562 const Function &Fn = DAG.getMachineFunction().getFunction();
1563 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1564 Fn, "local memory global used by non-kernel function",
1565 DL.getDebugLoc(), DS_Warning));
1566
1567 // We currently don't have a way to correctly allocate LDS objects that
1568 // aren't directly associated with a kernel. We do force inlining of
1569 // functions that use local objects. However, if these dead functions are
1570 // not eliminated, we don't want a compile time error. Just emit a warning
1571 // and a trap, since there should be no callable path here.
1572 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL, VT: MVT::Other, Operand: DAG.getEntryNode());
1573 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
1574 N1: Trap, N2: DAG.getRoot());
1575 DAG.setRoot(OutputChain);
1576 return DAG.getPOISON(VT: Op.getValueType());
1577 }
1578
1579 // TODO: We could emit code to handle the initialization somewhere.
1580 // We ignore the initializer for now and legalize it to allow selection.
1581 // The initializer will anyway get errored out during assembly emission.
1582 unsigned Offset = MFI->allocateLDSGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1583 // A constant byte offset (e.g. from a GEP into an array of named barriers)
1584 // folds directly into the allocated LDS address.
1585 return DAG.getConstant(Val: Offset + G->getOffset(), DL: SDLoc(Op),
1586 VT: Op.getValueType());
1587 }
1588 return SDValue();
1589}
1590
1591SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1592 SelectionDAG &DAG) const {
1593 SmallVector<SDValue, 8> Args;
1594 SDLoc SL(Op);
1595
1596 EVT VT = Op.getValueType();
1597 if (VT.getVectorElementType().getSizeInBits() < 32) {
1598 unsigned OpBitSize = Op.getOperand(i: 0).getValueType().getSizeInBits();
1599 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1600 unsigned NewNumElt = OpBitSize / 32;
1601 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1602 : EVT::getVectorVT(Context&: *DAG.getContext(),
1603 VT: MVT::i32, NumElements: NewNumElt);
1604 for (const SDUse &U : Op->ops()) {
1605 SDValue In = U.get();
1606 SDValue NewIn = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewEltVT, Operand: In);
1607 if (NewNumElt > 1)
1608 DAG.ExtractVectorElements(Op: NewIn, Args);
1609 else
1610 Args.push_back(Elt: NewIn);
1611 }
1612
1613 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
1614 NumElements: NewNumElt * Op.getNumOperands());
1615 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1616 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: BV);
1617 }
1618 }
1619
1620 for (const SDUse &U : Op->ops())
1621 DAG.ExtractVectorElements(Op: U.get(), Args);
1622
1623 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1624}
1625
1626SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1627 SelectionDAG &DAG) const {
1628 SDLoc SL(Op);
1629 SmallVector<SDValue, 8> Args;
1630 unsigned Start = Op.getConstantOperandVal(i: 1);
1631 EVT VT = Op.getValueType();
1632 EVT SrcVT = Op.getOperand(i: 0).getValueType();
1633
1634 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
1635 unsigned NumElt = VT.getVectorNumElements();
1636 unsigned NumSrcElt = SrcVT.getVectorNumElements();
1637 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types");
1638
1639 // Extract 32-bit registers at a time.
1640 EVT NewSrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumSrcElt / 2);
1641 EVT NewVT = NumElt == 2
1642 ? MVT::i32
1643 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElt / 2);
1644 SDValue Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewSrcVT, Operand: Op.getOperand(i: 0));
1645
1646 DAG.ExtractVectorElements(Op: Tmp, Args, Start: Start / 2, Count: NumElt / 2);
1647 if (NumElt == 2)
1648 Tmp = Args[0];
1649 else
1650 Tmp = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1651
1652 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Tmp);
1653 }
1654
1655 DAG.ExtractVectorElements(Op: Op.getOperand(i: 0), Args, Start,
1656 Count: VT.getVectorNumElements());
1657
1658 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1659}
1660
1661// TODO: Handle fabs too
1662static SDValue peekFNeg(SDValue Val) {
1663 if (Val.getOpcode() == ISD::FNEG)
1664 return Val.getOperand(i: 0);
1665
1666 return Val;
1667}
1668
1669// SelectionDAG twin of AMDGPUCombinerHelper::canIgnoreLegacyMinMaxTies.
1670static bool canIgnoreLegacyMinMaxTies(const SelectionDAG &DAG,
1671 SDNodeFlags Flags, SDValue LHS,
1672 SDValue RHS) {
1673 return Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
1674 DAG.isKnownNeverLogicalZero(Op: RHS);
1675}
1676
1677SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl(
1678 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1679 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1680 SelectionDAG &DAG = DCI.DAG;
1681 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
1682 assert(CCOpcode != ISD::SETCC_INVALID && "Invalid setcc condcode!");
1683
1684 switch (CCOpcode) {
1685 case ISD::SETOLE:
1686 case ISD::SETOLT:
1687 case ISD::SETLE:
1688 case ISD::SETLT:
1689 case ISD::SETOGE:
1690 case ISD::SETOGT:
1691 case ISD::SETGE:
1692 case ISD::SETGT:
1693 // Only do this after legalization to avoid interfering with other combines
1694 // which might occur.
1695 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1696 !DCI.isCalledByLegalizer())
1697 return SDValue();
1698 break;
1699 default:
1700 break;
1701 }
1702
1703 // Canonicalize so the select returns the compare's LHS on a true predicate.
1704 if (LHS != True)
1705 CCOpcode = ISD::getSetCCInverse(Operation: CCOpcode, Type: VT);
1706
1707 unsigned Opc;
1708 bool Swap; // Emit (rhs, lhs) instead of (lhs, rhs).
1709 switch (CCOpcode) {
1710 case ISD::SETOLT:
1711 case ISD::SETLT:
1712 case ISD::SETOLE:
1713 Opc = AMDGPUISD::FMIN_LEGACY;
1714 Swap = false;
1715 break;
1716 case ISD::SETULE:
1717 case ISD::SETLE:
1718 case ISD::SETULT:
1719 Opc = AMDGPUISD::FMIN_LEGACY;
1720 Swap = true;
1721 break;
1722 case ISD::SETOGE:
1723 case ISD::SETGE:
1724 case ISD::SETOGT:
1725 Opc = AMDGPUISD::FMAX_LEGACY;
1726 Swap = false;
1727 break;
1728 case ISD::SETUGT:
1729 case ISD::SETGT:
1730 case ISD::SETUGE:
1731 Opc = AMDGPUISD::FMAX_LEGACY;
1732 Swap = true;
1733 break;
1734 default:
1735 return SDValue();
1736 }
1737
1738 // For these predicates the NaN-correct operand order is the signed zero
1739 // tie-incorrect one, so the fold needs the tie to be unobservable.
1740 if ((CCOpcode == ISD::SETOLE || CCOpcode == ISD::SETULT ||
1741 CCOpcode == ISD::SETOGT || CCOpcode == ISD::SETUGE) &&
1742 !canIgnoreLegacyMinMaxTies(DAG, Flags, LHS, RHS))
1743 return SDValue();
1744
1745 if (Swap)
1746 std::swap(a&: LHS, b&: RHS);
1747 return DAG.getNode(Opcode: Opc, DL, VT, N1: LHS, N2: RHS, Flags);
1748}
1749
1750/// Generate Min/Max node
1751SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(
1752 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1753 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1754 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
1755 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, Flags,
1756 DCI);
1757
1758 SelectionDAG &DAG = DCI.DAG;
1759
1760 // If we can't directly match this, try to see if we can fold an fneg to
1761 // match.
1762
1763 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
1764 ConstantFPSDNode *CFalse = dyn_cast<ConstantFPSDNode>(Val&: False);
1765 SDValue NegTrue = peekFNeg(Val: True);
1766
1767 // Undo the combine foldFreeOpFromSelect does if it helps us match the
1768 // fmin/fmax.
1769 //
1770 // select (fcmp olt (lhs, K)), (fneg lhs), -K
1771 // -> fneg (fmin_legacy lhs, K)
1772 //
1773 // TODO: Use getNegatedExpression
1774 if (LHS == NegTrue && CFalse && CRHS) {
1775 APFloat NegRHS = neg(X: CRHS->getValueAPF());
1776 if (NegRHS == CFalse->getValueAPF()) {
1777 SDValue Combined = combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True: NegTrue,
1778 False, CC, Flags, DCI);
1779 if (Combined)
1780 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Combined);
1781 return SDValue();
1782 }
1783 }
1784
1785 return SDValue();
1786}
1787
1788std::pair<SDValue, SDValue>
1789AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1790 SDLoc SL(Op);
1791
1792 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1793
1794 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1795 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1796
1797 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1798 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1799
1800 return std::pair(Lo, Hi);
1801}
1802
1803SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1804 SDLoc SL(Op);
1805
1806 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1807 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1808 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1809}
1810
1811// Split a vector type into two parts. The first part is a power of two vector.
1812// The second part is whatever is left over, and is a scalar if it would
1813// otherwise be a 1-vector.
1814std::pair<EVT, EVT>
1815AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1816 EVT LoVT, HiVT;
1817 EVT EltVT = VT.getVectorElementType();
1818 unsigned NumElts = VT.getVectorNumElements();
1819 unsigned LoNumElts = PowerOf2Ceil(A: (NumElts + 1) / 2);
1820 LoVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: LoNumElts);
1821 HiVT = NumElts - LoNumElts == 1
1822 ? EltVT
1823 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts - LoNumElts);
1824 return std::pair(LoVT, HiVT);
1825}
1826
1827// Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1828// scalar.
1829std::pair<SDValue, SDValue>
1830AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1831 const EVT &LoVT, const EVT &HiVT,
1832 SelectionDAG &DAG) const {
1833 EVT VT = N.getValueType();
1834 assert(LoVT.getVectorNumElements() +
1835 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
1836 VT.getVectorNumElements() &&
1837 "More vector elements requested than available!");
1838 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N,
1839 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1840
1841 unsigned LoNumElts = LoVT.getVectorNumElements();
1842
1843 if (HiVT.isVector()) {
1844 unsigned HiNumElts = HiVT.getVectorNumElements();
1845 if ((VT.getVectorNumElements() % HiNumElts) == 0) {
1846 // Avoid creating an extract_subvector with an index that isn't a multiple
1847 // of the result type.
1848 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
1849 N2: DAG.getConstant(Val: LoNumElts, DL, VT: MVT::i32));
1850 return {Lo, Hi};
1851 }
1852
1853 SmallVector<SDValue, 8> Elts;
1854 DAG.ExtractVectorElements(Op: N, Args&: Elts, /*Start=*/LoNumElts,
1855 /*Count=*/HiNumElts);
1856 SDValue Hi = DAG.getBuildVector(VT: HiVT, DL, Ops: Elts);
1857 return {Lo, Hi};
1858 }
1859
1860 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: HiVT, N1: N,
1861 N2: DAG.getVectorIdxConstant(Val: LoNumElts, DL));
1862 return {Lo, Hi};
1863}
1864
1865SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1866 SelectionDAG &DAG) const {
1867 LoadSDNode *Load = cast<LoadSDNode>(Val: Op);
1868 EVT VT = Op.getValueType();
1869 SDLoc SL(Op);
1870
1871
1872 // If this is a 2 element vector, we really want to scalarize and not create
1873 // weird 1 element vectors.
1874 if (VT.getVectorNumElements() == 2) {
1875 SDValue Ops[2];
1876 std::tie(args&: Ops[0], args&: Ops[1]) = scalarizeVectorLoad(LD: Load, DAG);
1877 return DAG.getMergeValues(Ops, dl: SL);
1878 }
1879
1880 SDValue BasePtr = Load->getBasePtr();
1881 EVT MemVT = Load->getMemoryVT();
1882
1883 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1884
1885 EVT LoVT, HiVT;
1886 EVT LoMemVT, HiMemVT;
1887 SDValue Lo, Hi;
1888
1889 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1890 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1891 std::tie(args&: Lo, args&: Hi) = splitVector(N: Op, DL: SL, LoVT, HiVT, DAG);
1892
1893 unsigned Size = LoMemVT.getStoreSize();
1894 Align BaseAlign = Load->getAlign();
1895 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1896
1897 SDValue LoLoad = DAG.getExtLoad(
1898 ExtType: Load->getExtensionType(), dl: SL, VT: LoVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1899 MemVT: LoMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1900 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Size));
1901 SDValue HiLoad = DAG.getExtLoad(
1902 ExtType: Load->getExtensionType(), dl: SL, VT: HiVT, Chain: Load->getChain(), Ptr: HiPtr,
1903 PtrInfo: SrcValue.getWithOffset(O: LoMemVT.getStoreSize()), MemVT: HiMemVT, Alignment: HiAlign,
1904 MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1905
1906 SDValue Join;
1907 if (LoVT == HiVT) {
1908 // This is the case that the vector is power of two so was evenly split.
1909 Join = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, N1: LoLoad, N2: HiLoad);
1910 } else {
1911 Join = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SL, VT, N1: DAG.getPOISON(VT), N2: LoLoad,
1912 N3: DAG.getVectorIdxConstant(Val: 0, DL: SL));
1913 Join = DAG.getNode(
1914 Opcode: HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, DL: SL,
1915 VT, N1: Join, N2: HiLoad,
1916 N3: DAG.getVectorIdxConstant(Val: LoVT.getVectorNumElements(), DL: SL));
1917 }
1918
1919 SDValue Ops[] = {Join, DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
1920 N1: LoLoad.getValue(R: 1), N2: HiLoad.getValue(R: 1))};
1921
1922 return DAG.getMergeValues(Ops, dl: SL);
1923}
1924
1925SDValue AMDGPUTargetLowering::WidenOrSplitVectorLoad(SDValue Op,
1926 SelectionDAG &DAG) const {
1927 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
1928 EVT VT = Op.getValueType();
1929 SDValue BasePtr = Load->getBasePtr();
1930 EVT MemVT = Load->getMemoryVT();
1931 SDLoc SL(Op);
1932 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1933 Align BaseAlign = Load->getAlign();
1934 unsigned NumElements = MemVT.getVectorNumElements();
1935
1936 // Widen from vec3 to vec4 when the load is at least 8-byte aligned
1937 // or 16-byte fully dereferenceable. Otherwise, split the vector load.
1938 if (NumElements != 3 ||
1939 (BaseAlign < Align(8) &&
1940 !SrcValue.isDereferenceable(Size: 16, C&: *DAG.getContext(), DL: DAG.getDataLayout())))
1941 return SplitVectorLoad(Op, DAG);
1942
1943 assert(NumElements == 3);
1944
1945 EVT WideVT =
1946 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
1947 EVT WideMemVT =
1948 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(), NumElements: 4);
1949 SDValue WideLoad = DAG.getExtLoad(
1950 ExtType: Load->getExtensionType(), dl: SL, VT: WideVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1951 MemVT: WideMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags());
1952 return DAG.getMergeValues(
1953 Ops: {DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT, N1: WideLoad,
1954 N2: DAG.getVectorIdxConstant(Val: 0, DL: SL)),
1955 WideLoad.getValue(R: 1)},
1956 dl: SL);
1957}
1958
1959SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1960 SelectionDAG &DAG) const {
1961 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
1962 SDValue Val = Store->getValue();
1963 EVT VT = Val.getValueType();
1964
1965 // If this is a 2 element vector, we really want to scalarize and not create
1966 // weird 1 element vectors.
1967 if (VT.getVectorNumElements() == 2)
1968 return scalarizeVectorStore(ST: Store, DAG);
1969
1970 EVT MemVT = Store->getMemoryVT();
1971 SDValue Chain = Store->getChain();
1972 SDValue BasePtr = Store->getBasePtr();
1973 SDLoc SL(Op);
1974
1975 EVT LoVT, HiVT;
1976 EVT LoMemVT, HiMemVT;
1977 SDValue Lo, Hi;
1978
1979 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1980 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1981 std::tie(args&: Lo, args&: Hi) = splitVector(N: Val, DL: SL, LoVT, HiVT, DAG);
1982
1983 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: LoMemVT.getStoreSize());
1984
1985 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
1986 Align BaseAlign = Store->getAlign();
1987 unsigned Size = LoMemVT.getStoreSize();
1988 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1989
1990 SDValue LoStore =
1991 DAG.getTruncStore(Chain, dl: SL, Val: Lo, Ptr: BasePtr, PtrInfo: SrcValue, SVT: LoMemVT, Alignment: BaseAlign,
1992 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
1993 SDValue HiStore = DAG.getTruncStore(
1994 Chain, dl: SL, Val: Hi, Ptr: HiPtr, PtrInfo: SrcValue.getWithOffset(O: Size), SVT: HiMemVT, Alignment: HiAlign,
1995 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
1996
1997 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: LoStore, N2: HiStore);
1998}
1999
2000// This is a shortcut for integer division because we have fast i32<->f32
2001// conversions, and fast f32 reciprocal instructions.
2002SDValue AMDGPUTargetLowering::LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG,
2003 bool Sign) const {
2004 SDLoc DL(Op);
2005 EVT VT = Op.getValueType();
2006 assert(VT == MVT::i32 && "LowerDIVREMToFloat expects an i32");
2007
2008 SDValue LHS = Op.getOperand(i: 0);
2009 SDValue RHS = Op.getOperand(i: 1);
2010 MVT IntVT = MVT::i32;
2011 MVT FltVT = MVT::f32;
2012
2013 unsigned LHSSignBits;
2014 unsigned RHSSignBits;
2015 if (Sign) {
2016 LHSSignBits = DAG.ComputeNumSignBits(Op: LHS);
2017 RHSSignBits = DAG.ComputeNumSignBits(Op: RHS);
2018 if (LHSSignBits < 9 || RHSSignBits < 9)
2019 return SDValue();
2020 } else {
2021 KnownBits LHSKnown = DAG.computeKnownBits(Op: LHS);
2022 KnownBits RHSKnown = DAG.computeKnownBits(Op: RHS);
2023
2024 LHSSignBits = LHSKnown.countMinLeadingZeros();
2025 RHSSignBits = RHSKnown.countMinLeadingZeros();
2026 }
2027
2028 unsigned BitSize = VT.getSizeInBits();
2029 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
2030 unsigned DivBits = BitSize - SignBits;
2031 if (Sign)
2032 ++DivBits;
2033
2034 // In order to avoid problems due to 1 ulp accuracy issues with v_rcp_f32,
2035 // limit LowerDIVREMToFloat to:
2036 // [-0x400000,0x3FFFFF] for Sign
2037 // [ 0x000000,0x3FFFFF] for !Sign
2038 // This matches what is done in expandDivRemToFloatImpl.
2039 if (DivBits > (Sign ? 23 : 22))
2040 return SDValue();
2041
2042 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
2043 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
2044
2045 // int ia = (int)LHS;
2046 SDValue ia = LHS;
2047
2048 // int ib, (int)RHS;
2049 SDValue ib = RHS;
2050
2051 // The calculation:
2052 // fq = fa*recip(fb)
2053 // may be too small due to the 1ulp accuracy in the recip
2054 // operation and rounding issues. Since fq is truncated to produce
2055 // an integer value it may be too small by one. This is
2056 // dealt with by incrementing fa by 1ulp:
2057 // fq = (fa+1ulp)*recip(fb)
2058 // This will increase fa's magnitude by at most 0.5
2059 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
2060 // Thus, this method is safe since fa must be incremented by at least 1.0
2061 // for the quotient to increase by one.
2062 SDValue fa = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ia);
2063 SDValue faAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: fa);
2064 SDValue faIncremented = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: faAsInt,
2065 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
2066 fa = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FltVT, Operand: faIncremented);
2067
2068 // float fb = (float)ib;
2069 SDValue fb = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ib);
2070
2071 SDValue fq = DAG.getNode(Opcode: ISD::FMUL, DL, VT: FltVT,
2072 N1: fa, N2: DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: FltVT, Operand: fb));
2073
2074 // fq = trunc(fq);
2075 fq = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: FltVT, Operand: fq);
2076
2077 // int iq = (int)fq;
2078 SDValue Div = DAG.getNode(Opcode: ToInt, DL, VT: IntVT, Operand: fq);
2079
2080 // Rem needs compensation, it's easier to recompute it
2081 SDValue Rem = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Div, N2: RHS);
2082 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Rem);
2083
2084 return DAG.getMergeValues(Ops: { Div, Rem }, dl: DL);
2085}
2086
2087void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
2088 SelectionDAG &DAG,
2089 SmallVectorImpl<SDValue> &Results) const {
2090 SDLoc DL(Op);
2091 EVT VT = Op.getValueType();
2092
2093 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
2094
2095 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2096
2097 SDValue One = DAG.getConstant(Val: 1, DL, VT: HalfVT);
2098 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HalfVT);
2099
2100 //HiLo split
2101 SDValue LHS_Lo, LHS_Hi;
2102 SDValue LHS = Op.getOperand(i: 0);
2103 std::tie(args&: LHS_Lo, args&: LHS_Hi) = DAG.SplitScalar(N: LHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2104
2105 SDValue RHS_Lo, RHS_Hi;
2106 SDValue RHS = Op.getOperand(i: 1);
2107 std::tie(args&: RHS_Lo, args&: RHS_Hi) = DAG.SplitScalar(N: RHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2108
2109 if (DAG.MaskedValueIsZero(Op: RHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32)) &&
2110 DAG.MaskedValueIsZero(Op: LHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32))) {
2111
2112 SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2113 N1: LHS_Lo, N2: RHS_Lo);
2114
2115 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 0), Zero});
2116 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 1), Zero});
2117
2118 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV));
2119 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM));
2120 return;
2121 }
2122
2123 if (isTypeLegal(VT: MVT::i64)) {
2124 // The algorithm here is based on ideas from "Software Integer Division",
2125 // Tom Rodeheffer, August 2008.
2126
2127 MachineFunction &MF = DAG.getMachineFunction();
2128 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2129
2130 // Compute denominator reciprocal.
2131 unsigned FMAD =
2132 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA
2133 : MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign()
2134 ? (unsigned)ISD::FMAD
2135 : (unsigned)AMDGPUISD::FMAD_FTZ;
2136
2137 SDValue Cvt_Lo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Lo);
2138 SDValue Cvt_Hi = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Hi);
2139 SDValue Mad1 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Cvt_Hi,
2140 N2: DAG.getConstantFP(Val: APInt(32, 0x4f800000).bitsToFloat(), DL, VT: MVT::f32),
2141 N3: Cvt_Lo);
2142 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: MVT::f32, Operand: Mad1);
2143 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Rcp,
2144 N2: DAG.getConstantFP(Val: APInt(32, 0x5f7ffffc).bitsToFloat(), DL, VT: MVT::f32));
2145 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Mul1,
2146 N2: DAG.getConstantFP(Val: APInt(32, 0x2f800000).bitsToFloat(), DL, VT: MVT::f32));
2147 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: MVT::f32, Operand: Mul2);
2148 SDValue Mad2 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Trunc,
2149 N2: DAG.getConstantFP(Val: APInt(32, 0xcf800000).bitsToFloat(), DL, VT: MVT::f32),
2150 N3: Mul1);
2151 SDValue Rcp_Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Mad2);
2152 SDValue Rcp_Hi = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Trunc);
2153 SDValue Rcp64 = DAG.getBitcast(VT,
2154 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Rcp_Lo, Rcp_Hi}));
2155
2156 SDValue Zero64 = DAG.getConstant(Val: 0, DL, VT);
2157 SDValue One64 = DAG.getConstant(Val: 1, DL, VT);
2158 SDValue Zero1 = DAG.getConstant(Val: 0, DL, VT: MVT::i1);
2159 SDVTList HalfCarryVT = DAG.getVTList(VT1: HalfVT, VT2: MVT::i1);
2160
2161 // First round of UNR (Unsigned integer Newton-Raphson).
2162 SDValue Neg_RHS = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero64, N2: RHS);
2163 SDValue Mullo1 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Rcp64);
2164 SDValue Mulhi1 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Rcp64, N2: Mullo1);
2165 SDValue Mulhi1_Lo, Mulhi1_Hi;
2166 std::tie(args&: Mulhi1_Lo, args&: Mulhi1_Hi) =
2167 DAG.SplitScalar(N: Mulhi1, DL, LoVT: HalfVT, HiVT: HalfVT);
2168 SDValue Add1_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Lo,
2169 N2: Mulhi1_Lo, N3: Zero1);
2170 SDValue Add1_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Hi,
2171 N2: Mulhi1_Hi, N3: Add1_Lo.getValue(R: 1));
2172 SDValue Add1 = DAG.getBitcast(VT,
2173 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add1_Lo, Add1_Hi}));
2174
2175 // Second round of UNR.
2176 SDValue Mullo2 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Add1);
2177 SDValue Mulhi2 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Add1, N2: Mullo2);
2178 SDValue Mulhi2_Lo, Mulhi2_Hi;
2179 std::tie(args&: Mulhi2_Lo, args&: Mulhi2_Hi) =
2180 DAG.SplitScalar(N: Mulhi2, DL, LoVT: HalfVT, HiVT: HalfVT);
2181 SDValue Add2_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Lo,
2182 N2: Mulhi2_Lo, N3: Zero1);
2183 SDValue Add2_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Hi,
2184 N2: Mulhi2_Hi, N3: Add2_Lo.getValue(R: 1));
2185 SDValue Add2 = DAG.getBitcast(VT,
2186 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add2_Lo, Add2_Hi}));
2187
2188 SDValue Mulhi3 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: LHS, N2: Add2);
2189
2190 SDValue Mul3 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: RHS, N2: Mulhi3);
2191
2192 SDValue Mul3_Lo, Mul3_Hi;
2193 std::tie(args&: Mul3_Lo, args&: Mul3_Hi) = DAG.SplitScalar(N: Mul3, DL, LoVT: HalfVT, HiVT: HalfVT);
2194 SDValue Sub1_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Lo,
2195 N2: Mul3_Lo, N3: Zero1);
2196 SDValue Sub1_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Hi,
2197 N2: Mul3_Hi, N3: Sub1_Lo.getValue(R: 1));
2198 SDValue Sub1_Mi = DAG.getNode(Opcode: ISD::SUB, DL, VT: HalfVT, N1: LHS_Hi, N2: Mul3_Hi);
2199 SDValue Sub1 = DAG.getBitcast(VT,
2200 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub1_Lo, Sub1_Hi}));
2201
2202 SDValue MinusOne = DAG.getConstant(Val: 0xffffffffu, DL, VT: HalfVT);
2203 SDValue C1 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2204 Cond: ISD::SETUGE);
2205 SDValue C2 = DAG.getSelectCC(DL, LHS: Sub1_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2206 Cond: ISD::SETUGE);
2207 SDValue C3 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: C2, False: C1, Cond: ISD::SETEQ);
2208
2209 // TODO: Here and below portions of the code can be enclosed into if/endif.
2210 // Currently control flow is unconditional and we have 4 selects after
2211 // potential endif to substitute PHIs.
2212
2213 // if C3 != 0 ...
2214 SDValue Sub2_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Lo,
2215 N2: RHS_Lo, N3: Zero1);
2216 SDValue Sub2_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Mi,
2217 N2: RHS_Hi, N3: Sub1_Lo.getValue(R: 1));
2218 SDValue Sub2_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2219 N2: Zero, N3: Sub2_Lo.getValue(R: 1));
2220 SDValue Sub2 = DAG.getBitcast(VT,
2221 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub2_Lo, Sub2_Hi}));
2222
2223 SDValue Add3 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mulhi3, N2: One64);
2224
2225 SDValue C4 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2226 Cond: ISD::SETUGE);
2227 SDValue C5 = DAG.getSelectCC(DL, LHS: Sub2_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2228 Cond: ISD::SETUGE);
2229 SDValue C6 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: C5, False: C4, Cond: ISD::SETEQ);
2230
2231 // if (C6 != 0)
2232 SDValue Add4 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Add3, N2: One64);
2233
2234 SDValue Sub3_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Lo,
2235 N2: RHS_Lo, N3: Zero1);
2236 SDValue Sub3_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2237 N2: RHS_Hi, N3: Sub2_Lo.getValue(R: 1));
2238 SDValue Sub3_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub3_Mi,
2239 N2: Zero, N3: Sub3_Lo.getValue(R: 1));
2240 SDValue Sub3 = DAG.getBitcast(VT,
2241 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub3_Lo, Sub3_Hi}));
2242
2243 // endif C6
2244 // endif C3
2245
2246 SDValue Sel1 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Add4, False: Add3, Cond: ISD::SETNE);
2247 SDValue Div = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel1, False: Mulhi3, Cond: ISD::SETNE);
2248
2249 SDValue Sel2 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Sub3, False: Sub2, Cond: ISD::SETNE);
2250 SDValue Rem = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel2, False: Sub1, Cond: ISD::SETNE);
2251
2252 Results.push_back(Elt: Div);
2253 Results.push_back(Elt: Rem);
2254
2255 return;
2256 }
2257
2258 // r600 expandion.
2259 // Get Speculative values
2260 SDValue DIV_Part = DAG.getNode(Opcode: ISD::UDIV, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2261 SDValue REM_Part = DAG.getNode(Opcode: ISD::UREM, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2262
2263 SDValue REM_Lo = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: REM_Part, False: LHS_Hi, Cond: ISD::SETEQ);
2264 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {REM_Lo, Zero});
2265 REM = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM);
2266
2267 SDValue DIV_Hi = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: DIV_Part, False: Zero, Cond: ISD::SETEQ);
2268 SDValue DIV_Lo = Zero;
2269
2270 const unsigned halfBitWidth = HalfVT.getSizeInBits();
2271
2272 for (unsigned i = 0; i < halfBitWidth; ++i) {
2273 const unsigned bitPos = halfBitWidth - i - 1;
2274 SDValue POS = DAG.getConstant(Val: bitPos, DL, VT: HalfVT);
2275 // Get value of high bit
2276 SDValue HBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: HalfVT, N1: LHS_Lo, N2: POS);
2277 HBit = DAG.getNode(Opcode: ISD::AND, DL, VT: HalfVT, N1: HBit, N2: One);
2278 HBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: HBit);
2279
2280 // Shift
2281 REM = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: REM, N2: DAG.getConstant(Val: 1, DL, VT));
2282 // Add LHS high bit
2283 REM = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: REM, N2: HBit);
2284
2285 SDValue BIT = DAG.getConstant(Val: 1ULL << bitPos, DL, VT: HalfVT);
2286 SDValue realBIT = DAG.getSelectCC(DL, LHS: REM, RHS, True: BIT, False: Zero, Cond: ISD::SETUGE);
2287
2288 DIV_Lo = DAG.getNode(Opcode: ISD::OR, DL, VT: HalfVT, N1: DIV_Lo, N2: realBIT);
2289
2290 // Update REM
2291 SDValue REM_sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: REM, N2: RHS);
2292 REM = DAG.getSelectCC(DL, LHS: REM, RHS, True: REM_sub, False: REM, Cond: ISD::SETUGE);
2293 }
2294
2295 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {DIV_Lo, DIV_Hi});
2296 DIV = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV);
2297 Results.push_back(Elt: DIV);
2298 Results.push_back(Elt: REM);
2299}
2300
2301SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
2302 SelectionDAG &DAG) const {
2303 SDLoc DL(Op);
2304 EVT VT = Op.getValueType();
2305
2306 if (VT == MVT::i64) {
2307 SmallVector<SDValue, 2> Results;
2308 LowerUDIVREM64(Op, DAG, Results);
2309 return DAG.getMergeValues(Ops: Results, dl: DL);
2310 }
2311
2312 if (VT == MVT::i32) {
2313 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: false))
2314 return Res;
2315 }
2316
2317 SDValue X = Op.getOperand(i: 0);
2318 SDValue Y = Op.getOperand(i: 1);
2319
2320 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
2321 // algorithm used here.
2322
2323 // Initial estimate of inv(y).
2324 SDValue Z = DAG.getNode(Opcode: AMDGPUISD::URECIP, DL, VT, Operand: Y);
2325
2326 // One round of UNR.
2327 SDValue NegY = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Y);
2328 SDValue NegYZ = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: NegY, N2: Z);
2329 Z = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Z,
2330 N2: DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Z, N2: NegYZ));
2331
2332 // Quotient/remainder estimate.
2333 SDValue Q = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: X, N2: Z);
2334 SDValue R =
2335 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: X, N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Q, N2: Y));
2336
2337 // First quotient/remainder refinement.
2338 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2339 SDValue One = DAG.getConstant(Val: 1, DL, VT);
2340 SDValue Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2341 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2342 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2343 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2344 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2345
2346 // Second quotient/remainder refinement.
2347 Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2348 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2349 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2350 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2351 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2352
2353 return DAG.getMergeValues(Ops: {Q, R}, dl: DL);
2354}
2355
2356SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
2357 SelectionDAG &DAG) const {
2358 SDLoc DL(Op);
2359 EVT VT = Op.getValueType();
2360
2361 SDValue LHS = Op.getOperand(i: 0);
2362 SDValue RHS = Op.getOperand(i: 1);
2363
2364 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
2365 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2366
2367 if (VT == MVT::i32) {
2368 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: true))
2369 return Res;
2370 }
2371
2372 // LHS must have > 33 sign-bits to ensure that LHS != -2147483648
2373 // Otherwise 32-bit division cannot be used safely.
2374 // -2147483648/1 and -2147483648/-1 are not equal,
2375 // but they produce the same lower 32-bit result.
2376 if (VT == MVT::i64 && DAG.ComputeNumSignBits(Op: LHS) > 33 &&
2377 DAG.ComputeNumSignBits(Op: RHS) > 32) {
2378 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2379
2380 //HiLo split
2381 SDValue LHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: LHS, N2: Zero);
2382 SDValue RHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: RHS, N2: Zero);
2383 SDValue DIVREM = DAG.getNode(Opcode: ISD::SDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2384 N1: LHS_Lo, N2: RHS_Lo);
2385 SDValue Res[2] = {
2386 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 0)),
2387 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 1))
2388 };
2389 return DAG.getMergeValues(Ops: Res, dl: DL);
2390 }
2391
2392 SDValue LHSign = DAG.getSelectCC(DL, LHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2393 SDValue RHSign = DAG.getSelectCC(DL, LHS: RHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2394 SDValue DSign = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHSign, N2: RHSign);
2395 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2396
2397 LHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: LHSign);
2398 RHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: RHS, N2: RHSign);
2399
2400 LHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHS, N2: LHSign);
2401 RHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: RHS, N2: RHSign);
2402
2403 SDValue Div = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
2404 SDValue Rem = Div.getValue(R: 1);
2405
2406 Div = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Div, N2: DSign);
2407 Rem = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Rem, N2: RSign);
2408
2409 Div = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Div, N2: DSign);
2410 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Rem, N2: RSign);
2411
2412 SDValue Res[2] = {
2413 Div,
2414 Rem
2415 };
2416 return DAG.getMergeValues(Ops: Res, dl: DL);
2417}
2418
2419SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2420 SDLoc SL(Op);
2421 SDValue Src = Op.getOperand(i: 0);
2422
2423 // result = trunc(src)
2424 // if (src > 0.0 && src != result)
2425 // result += 1.0
2426
2427 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2428
2429 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2430 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
2431
2432 EVT SetCCVT =
2433 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2434
2435 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOGT);
2436 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2437 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2438
2439 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: One, N3: Zero);
2440 // TODO: Should this propagate fast-math-flags?
2441 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2442}
2443
2444static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2445 SelectionDAG &DAG) {
2446 const unsigned FractBits = 52;
2447 const unsigned ExpBits = 11;
2448
2449 SDValue ExpPart = DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32,
2450 N1: Hi,
2451 N2: DAG.getConstant(Val: FractBits - 32, DL: SL, VT: MVT::i32),
2452 N3: DAG.getConstant(Val: ExpBits, DL: SL, VT: MVT::i32));
2453 SDValue Exp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ExpPart,
2454 N2: DAG.getConstant(Val: 1023, DL: SL, VT: MVT::i32));
2455
2456 return Exp;
2457}
2458
2459SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2460 SDLoc SL(Op);
2461 SDValue Src = Op.getOperand(i: 0);
2462
2463 assert(Op.getValueType() == MVT::f64);
2464
2465 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
2466
2467 // Extract the upper half, since this is where we will find the sign and
2468 // exponent.
2469 SDValue Hi = getHiHalf64(Op: Src, DAG);
2470
2471 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2472
2473 const unsigned FractBits = 52;
2474
2475 // Extract the sign bit.
2476 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, DL: SL, VT: MVT::i32);
2477 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Hi, N2: SignBitMask);
2478
2479 // Extend back to 64-bits.
2480 SDValue SignBit64 = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Zero, SignBit});
2481 SignBit64 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: SignBit64);
2482
2483 SDValue BcInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Src);
2484 const SDValue FractMask
2485 = DAG.getConstant(Val: (UINT64_C(1) << FractBits) - 1, DL: SL, VT: MVT::i64);
2486
2487 SDValue Shr = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: FractMask, N2: Exp);
2488 SDValue Not = DAG.getNOT(DL: SL, Val: Shr, VT: MVT::i64);
2489 SDValue Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i64, N1: BcInt, N2: Not);
2490
2491 EVT SetCCVT =
2492 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i32);
2493
2494 const SDValue FiftyOne = DAG.getConstant(Val: FractBits - 1, DL: SL, VT: MVT::i32);
2495
2496 SDValue ExpLt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: Zero, Cond: ISD::SETLT);
2497 SDValue ExpGt51 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: FiftyOne, Cond: ISD::SETGT);
2498
2499 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpLt0, N2: SignBit64, N3: Tmp0);
2500 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpGt51, N2: BcInt, N3: Tmp1);
2501
2502 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f64, Operand: Tmp2);
2503}
2504
2505SDValue AMDGPUTargetLowering::LowerFROUNDEVEN(SDValue Op,
2506 SelectionDAG &DAG) const {
2507 SDLoc SL(Op);
2508 SDValue Src = Op.getOperand(i: 0);
2509
2510 assert(Op.getValueType() == MVT::f64);
2511
2512 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2513 SDValue C1 = DAG.getConstantFP(Val: C1Val, DL: SL, VT: MVT::f64);
2514 SDValue CopySign = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MVT::f64, N1: C1, N2: Src);
2515
2516 // TODO: Should this propagate fast-math-flags?
2517
2518 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Src, N2: CopySign);
2519 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT: MVT::f64, N1: Tmp1, N2: CopySign);
2520
2521 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f64, Operand: Src);
2522
2523 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2524 SDValue C2 = DAG.getConstantFP(Val: C2Val, DL: SL, VT: MVT::f64);
2525
2526 EVT SetCCVT =
2527 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2528 SDValue Cond = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Fabs, RHS: C2, Cond: ISD::SETOGT);
2529
2530 return DAG.getSelect(DL: SL, VT: MVT::f64, Cond, LHS: Src, RHS: Tmp2);
2531}
2532
2533SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op,
2534 SelectionDAG &DAG) const {
2535 // FNEARBYINT and FRINT are the same, except in their handling of FP
2536 // exceptions. Those aren't really meaningful for us, and OpenCL only has
2537 // rint, so just treat them as equivalent.
2538 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT: Op.getValueType(),
2539 Operand: Op.getOperand(i: 0));
2540}
2541
2542SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2543 auto VT = Op.getValueType();
2544 auto Arg = Op.getOperand(i: 0u);
2545 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT, Operand: Arg);
2546}
2547
2548// XXX - May require not supporting f32 denormals?
2549
2550// Don't handle v2f16. The extra instructions to scalarize and repack around the
2551// compare and vselect end up producing worse code than scalarizing the whole
2552// operation.
2553SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2554 SDLoc SL(Op);
2555 SDValue X = Op.getOperand(i: 0);
2556 EVT VT = Op.getValueType();
2557
2558 SDValue T = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: X);
2559
2560 // TODO: Should this propagate fast-math-flags?
2561
2562 SDValue Diff = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: T);
2563
2564 SDValue AbsDiff = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Diff);
2565
2566 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2567 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2568
2569 EVT SetCCVT =
2570 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2571
2572 const SDValue Half = DAG.getConstantFP(Val: 0.5, DL: SL, VT);
2573 SDValue Cmp = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsDiff, RHS: Half, Cond: ISD::SETOGE);
2574 SDValue OneOrZeroFP = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cmp, N2: One, N3: Zero);
2575
2576 SDValue SignedOffset = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: OneOrZeroFP, N2: X);
2577 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: T, N2: SignedOffset);
2578}
2579
2580SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2581 SDLoc SL(Op);
2582 SDValue Src = Op.getOperand(i: 0);
2583
2584 // result = trunc(src);
2585 // if (src < 0.0 && src != result)
2586 // result += -1.0.
2587
2588 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2589
2590 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2591 const SDValue NegOne = DAG.getConstantFP(Val: -1.0, DL: SL, VT: MVT::f64);
2592
2593 EVT SetCCVT =
2594 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2595
2596 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOLT);
2597 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2598 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2599
2600 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: NegOne, N3: Zero);
2601 // TODO: Should this propagate fast-math-flags?
2602 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2603}
2604
2605/// Return true if it's known that \p Src can never be an f32 denormal value.
2606static bool valueIsKnownNeverF32Denorm(SDValue Src) {
2607 switch (Src.getOpcode()) {
2608 case ISD::FP_EXTEND:
2609 return Src.getOperand(i: 0).getValueType() == MVT::f16;
2610 case ISD::FP16_TO_FP:
2611 case ISD::FFREXP:
2612 case ISD::FSQRT:
2613 case AMDGPUISD::LOG:
2614 case AMDGPUISD::EXP:
2615 return true;
2616 case ISD::INTRINSIC_WO_CHAIN: {
2617 unsigned IntrinsicID = Src.getConstantOperandVal(i: 0);
2618 switch (IntrinsicID) {
2619 case Intrinsic::amdgcn_frexp_mant:
2620 case Intrinsic::amdgcn_log:
2621 case Intrinsic::amdgcn_log_clamp:
2622 case Intrinsic::amdgcn_exp2:
2623 case Intrinsic::amdgcn_sqrt:
2624 return true;
2625 default:
2626 return false;
2627 }
2628 }
2629 default:
2630 return false;
2631 }
2632
2633 llvm_unreachable("covered opcode switch");
2634}
2635
2636bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG,
2637 SDNodeFlags Flags) {
2638 return Flags.hasApproximateFuncs();
2639}
2640
2641bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG,
2642 SDValue Src,
2643 SDNodeFlags Flags) {
2644 return !valueIsKnownNeverF32Denorm(Src) &&
2645 DAG.getMachineFunction()
2646 .getDenormalMode(FPType: APFloat::IEEEsingle())
2647 .Input != DenormalMode::PreserveSign;
2648}
2649
2650SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src,
2651 SDNodeFlags Flags) const {
2652 SDLoc SL(Src);
2653 EVT VT = Src.getValueType();
2654 const fltSemantics &Semantics = VT.getFltSemantics();
2655 SDValue Inf = DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL: SL, VT);
2656
2657 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Src, Flags);
2658 SDValue IsFinite = DAG.getSetCC(
2659 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Fabs,
2660 RHS: Inf, Cond: ISD::SETOLT);
2661 return IsFinite;
2662}
2663
2664/// If denormal handling is required return the scaled input to FLOG2, and the
2665/// check for denormal range. Otherwise, return null values.
2666std::pair<SDValue, SDValue>
2667AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL,
2668 SDValue Src, SDNodeFlags Flags) const {
2669 if (!needsDenormHandlingF32(DAG, Src, Flags))
2670 return {};
2671
2672 MVT VT = MVT::f32;
2673 const fltSemantics &Semantics = APFloat::IEEEsingle();
2674 SDValue SmallestNormal =
2675 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2676
2677 SDValue IsLtSmallestNormal = DAG.getSetCC(
2678 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2679 RHS: SmallestNormal, Cond: ISD::SETOLT);
2680
2681 SDValue Scale32 = DAG.getConstantFP(Val: 0x1.0p+32, DL: SL, VT);
2682 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2683 SDValue ScaleFactor =
2684 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: Scale32, N3: One, Flags);
2685
2686 SDValue ScaledInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Src, N2: ScaleFactor, Flags);
2687 return {ScaledInput, IsLtSmallestNormal};
2688}
2689
2690SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const {
2691 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
2692 // If we have to handle denormals, scale up the input and adjust the result.
2693
2694 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
2695 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
2696
2697 SDLoc SL(Op);
2698 EVT VT = Op.getValueType();
2699 SDValue Src = Op.getOperand(i: 0);
2700 SDNodeFlags Flags = Op->getFlags();
2701
2702 if (VT == MVT::f16) {
2703 // Nothing in half is a denormal when promoted to f32.
2704 assert(!isTypeLegal(VT));
2705 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
2706 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
2707 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
2708 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
2709 }
2710
2711 auto [ScaledInput, IsLtSmallestNormal] =
2712 getScaledLogInput(DAG, SL, Src, Flags);
2713 if (!ScaledInput)
2714 return DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: Src, Flags);
2715
2716 SDValue Log2 = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2717
2718 SDValue ThirtyTwo = DAG.getConstantFP(Val: 32.0, DL: SL, VT);
2719 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2720 SDValue ResultOffset =
2721 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: ThirtyTwo, N3: Zero);
2722 return DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: Log2, N2: ResultOffset, Flags);
2723}
2724
2725static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2726 SDValue Y, SDValue C, SDNodeFlags Flags = SDNodeFlags()) {
2727 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Y, Flags);
2728 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: C, Flags);
2729}
2730
2731SDValue AMDGPUTargetLowering::LowerFLOGCommon(SDValue Op,
2732 SelectionDAG &DAG) const {
2733 SDValue X = Op.getOperand(i: 0);
2734 EVT VT = Op.getValueType();
2735 SDNodeFlags Flags = Op->getFlags();
2736 SDLoc DL(Op);
2737 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10;
2738 assert(IsLog10 || Op.getOpcode() == ISD::FLOG);
2739
2740 if (VT == MVT::f16 || Flags.hasApproximateFuncs()) {
2741 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
2742 // depending on !fpmath metadata.
2743
2744 bool PromoteToF32 = VT == MVT::f16 && (!Flags.hasApproximateFuncs() ||
2745 !isTypeLegal(VT: MVT::f16));
2746
2747 if (PromoteToF32) {
2748 // Log and multiply in f32 is always good enough for f16.
2749 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2750 }
2751
2752 SDValue Lowered = LowerFLOGUnsafe(Op: X, SL: DL, DAG, IsLog10, Flags);
2753 if (PromoteToF32) {
2754 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Lowered,
2755 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32), Flags);
2756 }
2757
2758 return Lowered;
2759 }
2760
2761 SDValue ScaledInput, IsScaled;
2762 if (VT == MVT::f16)
2763 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2764 else {
2765 std::tie(args&: ScaledInput, args&: IsScaled) = getScaledLogInput(DAG, SL: DL, Src: X, Flags);
2766 if (ScaledInput)
2767 X = ScaledInput;
2768 }
2769
2770 SDValue Y = DAG.getNode(Opcode: AMDGPUISD::LOG, DL, VT, Operand: X, Flags);
2771
2772 SDValue R;
2773 if (Subtarget->hasFastFMAF32()) {
2774 // c+cc are ln(2)/ln(10) to more than 49 bits
2775 const float c_log10 = 0x1.344134p-2f;
2776 const float cc_log10 = 0x1.09f79ep-26f;
2777
2778 // c + cc is ln(2) to more than 49 bits
2779 const float c_log = 0x1.62e42ep-1f;
2780 const float cc_log = 0x1.efa39ep-25f;
2781
2782 SDValue C = DAG.getConstantFP(Val: IsLog10 ? c_log10 : c_log, DL, VT);
2783 SDValue CC = DAG.getConstantFP(Val: IsLog10 ? cc_log10 : cc_log, DL, VT);
2784 // This adds correction terms for which contraction may lead to an increase
2785 // in the error of the approximation, so disable it.
2786 Flags.setAllowContract(false);
2787 R = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Y, N2: C, Flags);
2788 SDValue NegR = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: R, Flags);
2789 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: C, N3: NegR, Flags);
2790 SDValue FMA1 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: CC, N3: FMA0, Flags);
2791 R = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: R, N2: FMA1, Flags);
2792 } else {
2793 // ch+ct is ln(2)/ln(10) to more than 36 bits
2794 const float ch_log10 = 0x1.344000p-2f;
2795 const float ct_log10 = 0x1.3509f6p-18f;
2796
2797 // ch + ct is ln(2) to more than 36 bits
2798 const float ch_log = 0x1.62e000p-1f;
2799 const float ct_log = 0x1.0bfbe8p-15f;
2800
2801 SDValue CH = DAG.getConstantFP(Val: IsLog10 ? ch_log10 : ch_log, DL, VT);
2802 SDValue CT = DAG.getConstantFP(Val: IsLog10 ? ct_log10 : ct_log, DL, VT);
2803
2804 SDValue YAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Y);
2805 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL, VT: MVT::i32);
2806 SDValue YHInt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: YAsInt, N2: MaskConst);
2807 SDValue YH = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: YHInt);
2808 SDValue YT = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: YH, Flags);
2809 // This adds correction terms for which contraction may lead to an increase
2810 // in the error of the approximation, so disable it.
2811 Flags.setAllowContract(false);
2812 SDValue YTCT = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: YT, N2: CT, Flags);
2813 SDValue Mad0 = getMad(DAG, SL: DL, VT, X: YH, Y: CT, C: YTCT, Flags);
2814 SDValue Mad1 = getMad(DAG, SL: DL, VT, X: YT, Y: CH, C: Mad0, Flags);
2815 R = getMad(DAG, SL: DL, VT, X: YH, Y: CH, C: Mad1);
2816 }
2817
2818 const bool IsFiniteOnly = Flags.hasNoNaNs() && Flags.hasNoInfs();
2819
2820 // TODO: Check if known finite from source value.
2821 if (!IsFiniteOnly) {
2822 SDValue IsFinite = getIsFinite(DAG, Src: Y, Flags);
2823 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsFinite, N2: R, N3: Y, Flags);
2824 }
2825
2826 if (IsScaled) {
2827 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
2828 SDValue ShiftK =
2829 DAG.getConstantFP(Val: IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2830 SDValue Shift =
2831 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsScaled, N2: ShiftK, N3: Zero, Flags);
2832 R = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: R, N2: Shift, Flags);
2833 }
2834
2835 return R;
2836}
2837
2838// Do f32 fast math expansion for flog2 or flog10. This is accurate enough for a
2839// promote f16 operation.
2840SDValue AMDGPUTargetLowering::LowerFLOGUnsafe(SDValue Src, const SDLoc &SL,
2841 SelectionDAG &DAG, bool IsLog10,
2842 SDNodeFlags Flags) const {
2843 EVT VT = Src.getValueType();
2844 unsigned LogOp =
2845 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2;
2846
2847 double Log2BaseInverted =
2848 IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2849
2850 if (VT == MVT::f32) {
2851 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags);
2852 if (ScaledInput) {
2853 SDValue LogSrc = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2854 SDValue ScaledResultOffset =
2855 DAG.getConstantFP(Val: -32.0 * Log2BaseInverted, DL: SL, VT);
2856
2857 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL: SL, VT);
2858
2859 SDValue ResultOffset = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsScaled,
2860 N2: ScaledResultOffset, N3: Zero, Flags);
2861
2862 SDValue Log2Inv = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2863
2864 if (Subtarget->hasFastFMAF32())
2865 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: LogSrc, N2: Log2Inv, N3: ResultOffset,
2866 Flags);
2867 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LogSrc, N2: Log2Inv, Flags);
2868 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: ResultOffset);
2869 }
2870 }
2871
2872 SDValue Log2Operand = DAG.getNode(Opcode: LogOp, DL: SL, VT, Operand: Src, Flags);
2873 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2874
2875 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Log2Operand, N2: Log2BaseInvertedOperand,
2876 Flags);
2877}
2878
2879// This expansion gives a result slightly better than 1ulp.
2880SDValue AMDGPUTargetLowering::lowerFEXPF64(SDValue Op,
2881 SelectionDAG &DAG) const {
2882 SDLoc DL(Op);
2883 SDValue X = Op.getOperand(i: 0);
2884
2885 // TODO: Check if reassoc is safe. There is an output change in exp2 and
2886 // exp10, which slightly increases ulp.
2887 SDNodeFlags Flags = Op->getFlags() & ~SDNodeFlags::AllowReassociation;
2888
2889 SDValue DN, F, T;
2890
2891 if (Op.getOpcode() == ISD::FEXP2) {
2892 // dn = rint(x)
2893 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: X, Flags);
2894 // f = x - dn
2895 F = DAG.getNode(Opcode: ISD::FSUB, DL, VT: MVT::f64, N1: X, N2: DN, Flags);
2896 // t = f*C1 + f*C2
2897 SDValue C1 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2898 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2899 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C2, Flags);
2900 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C1, N3: Mul2, Flags);
2901 } else if (Op.getOpcode() == ISD::FEXP10) {
2902 // dn = rint(x * C1)
2903 SDValue C1 = DAG.getConstantFP(Val: 0x1.a934f0979a371p+1, DL, VT: MVT::f64);
2904 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2905 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2906
2907 // f = FMA(-dn, C2, FMA(-dn, C3, x))
2908 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2909 SDValue C2 = DAG.getConstantFP(Val: -0x1.9dc1da994fd21p-59, DL, VT: MVT::f64);
2910 SDValue C3 = DAG.getConstantFP(Val: 0x1.34413509f79ffp-2, DL, VT: MVT::f64);
2911 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2912 F = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2913
2914 // t = FMA(f, C4, f*C5)
2915 SDValue C4 = DAG.getConstantFP(Val: 0x1.26bb1bbb55516p+1, DL, VT: MVT::f64);
2916 SDValue C5 = DAG.getConstantFP(Val: -0x1.f48ad494ea3e9p-53, DL, VT: MVT::f64);
2917 SDValue MulF = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C5, Flags);
2918 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C4, N3: MulF, Flags);
2919 } else { // ISD::FEXP
2920 // dn = rint(x * C1)
2921 SDValue C1 = DAG.getConstantFP(Val: 0x1.71547652b82fep+0, DL, VT: MVT::f64);
2922 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2923 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2924
2925 // t = FMA(-dn, C2, FMA(-dn, C3, x))
2926 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2927 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2928 SDValue C3 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2929 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2930 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2931 }
2932
2933 // Polynomial expansion for p
2934 SDValue P = DAG.getConstantFP(Val: 0x1.ade156a5dcb37p-26, DL, VT: MVT::f64);
2935 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2936 N3: DAG.getConstantFP(Val: 0x1.28af3fca7ab0cp-22, DL, VT: MVT::f64),
2937 Flags);
2938 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2939 N3: DAG.getConstantFP(Val: 0x1.71dee623fde64p-19, DL, VT: MVT::f64),
2940 Flags);
2941 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2942 N3: DAG.getConstantFP(Val: 0x1.a01997c89e6b0p-16, DL, VT: MVT::f64),
2943 Flags);
2944 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2945 N3: DAG.getConstantFP(Val: 0x1.a01a014761f6ep-13, DL, VT: MVT::f64),
2946 Flags);
2947 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2948 N3: DAG.getConstantFP(Val: 0x1.6c16c1852b7b0p-10, DL, VT: MVT::f64),
2949 Flags);
2950 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2951 N3: DAG.getConstantFP(Val: 0x1.1111111122322p-7, DL, VT: MVT::f64), Flags);
2952 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2953 N3: DAG.getConstantFP(Val: 0x1.55555555502a1p-5, DL, VT: MVT::f64), Flags);
2954 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2955 N3: DAG.getConstantFP(Val: 0x1.5555555555511p-3, DL, VT: MVT::f64), Flags);
2956 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2957 N3: DAG.getConstantFP(Val: 0x1.000000000000bp-1, DL, VT: MVT::f64), Flags);
2958
2959 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT: MVT::f64);
2960
2961 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
2962 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
2963
2964 // z = ldexp(p, (int)dn)
2965 SDValue DNInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: MVT::i32, Operand: DN);
2966 SDValue Z = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: P, N2: DNInt, Flags);
2967
2968 // Overflow/underflow guards
2969 SDValue CondHi = DAG.getSetCC(
2970 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: 1024.0, DL, VT: MVT::f64), Cond: ISD::SETULE);
2971
2972 if (!Flags.hasNoInfs()) {
2973 SDValue PInf = DAG.getConstantFP(Val: std::numeric_limits<double>::infinity(),
2974 DL, VT: MVT::f64);
2975 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondHi, LHS: Z, RHS: PInf, Flags);
2976 }
2977
2978 SDValue CondLo = DAG.getSetCC(
2979 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: -1075.0, DL, VT: MVT::f64), Cond: ISD::SETUGE);
2980 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
2981 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondLo, LHS: Z, RHS: Zero, Flags);
2982
2983 return Z;
2984}
2985
2986SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const {
2987 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
2988 // If we have to handle denormals, scale up the input and adjust the result.
2989
2990 EVT VT = Op.getValueType();
2991 if (VT == MVT::f64)
2992 return lowerFEXPF64(Op, DAG);
2993
2994 SDLoc SL(Op);
2995 SDValue Src = Op.getOperand(i: 0);
2996 SDNodeFlags Flags = Op->getFlags();
2997
2998 if (VT == MVT::f16) {
2999 // Nothing in half is a denormal when promoted to f32.
3000 assert(!isTypeLegal(MVT::f16));
3001 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3002 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
3003 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
3004 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3005 }
3006
3007 assert(VT == MVT::f32);
3008
3009 if (!needsDenormHandlingF32(DAG, Src, Flags))
3010 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3011
3012 // bool needs_scaling = x < -0x1.f80000p+6f;
3013 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3014
3015 // -nextafter(128.0, -1)
3016 SDValue RangeCheckConst = DAG.getConstantFP(Val: -0x1.f80000p+6f, DL: SL, VT);
3017
3018 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3019
3020 SDValue NeedsScaling =
3021 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: RangeCheckConst, Cond: ISD::SETOLT);
3022
3023 SDValue SixtyFour = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3024 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3025
3026 SDValue AddOffset =
3027 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: SixtyFour, N3: Zero);
3028
3029 SDValue AddInput = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Src, N2: AddOffset, Flags);
3030 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: AddInput, Flags);
3031
3032 SDValue TwoExpNeg64 = DAG.getConstantFP(Val: 0x1.0p-64f, DL: SL, VT);
3033 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
3034 SDValue ResultScale =
3035 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: TwoExpNeg64, N3: One);
3036
3037 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScale, Flags);
3038}
3039
3040SDValue AMDGPUTargetLowering::lowerFEXPUnsafeImpl(SDValue X, const SDLoc &SL,
3041 SelectionDAG &DAG,
3042 SDNodeFlags Flags,
3043 bool IsExp10) const {
3044 // exp(x) -> exp2(M_LOG2E_F * x);
3045 // exp10(x) -> exp2(log2(10) * x);
3046 EVT VT = X.getValueType();
3047 SDValue Const =
3048 DAG.getConstantFP(Val: IsExp10 ? 0x1.a934f0p+1f : numbers::log2e, DL: SL, VT);
3049
3050 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Const, Flags);
3051 return DAG.getNode(Opcode: VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
3052 : (unsigned)ISD::FEXP2,
3053 DL: SL, VT, Operand: Mul, Flags);
3054}
3055
3056SDValue AMDGPUTargetLowering::lowerFEXPUnsafe(SDValue X, const SDLoc &SL,
3057 SelectionDAG &DAG,
3058 SDNodeFlags Flags) const {
3059 EVT VT = X.getValueType();
3060 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags))
3061 return lowerFEXPUnsafeImpl(X, SL, DAG, Flags, /*IsExp10=*/false);
3062
3063 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3064
3065 SDValue Threshold = DAG.getConstantFP(Val: -0x1.5d58a0p+6f, DL: SL, VT);
3066 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3067
3068 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3069
3070 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3071
3072 SDValue AdjustedX =
3073 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3074
3075 const SDValue Log2E = DAG.getConstantFP(Val: numbers::log2e, DL: SL, VT);
3076 SDValue ExpInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: Log2E, Flags);
3077
3078 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: ExpInput, Flags);
3079
3080 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.969d48p-93f, DL: SL, VT);
3081 SDValue AdjustedResult =
3082 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScaleFactor, Flags);
3083
3084 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: Exp2,
3085 Flags);
3086}
3087
3088/// Emit approx-funcs appropriate lowering for exp10. inf/nan should still be
3089/// handled correctly.
3090SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL,
3091 SelectionDAG &DAG,
3092 SDNodeFlags Flags) const {
3093 const EVT VT = X.getValueType();
3094
3095 const unsigned Exp2Op = VT == MVT::f32 ? static_cast<unsigned>(AMDGPUISD::EXP)
3096 : static_cast<unsigned>(ISD::FEXP2);
3097
3098 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags)) {
3099 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
3100 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3101 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3102
3103 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K0, Flags);
3104 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3105 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K1, Flags);
3106 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3107 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1);
3108 }
3109
3110 // bool s = x < -0x1.2f7030p+5f;
3111 // x += s ? 0x1.0p+5f : 0.0f;
3112 // exp10 = exp2(x * 0x1.a92000p+1f) *
3113 // exp2(x * 0x1.4f0978p-11f) *
3114 // (s ? 0x1.9f623ep-107f : 1.0f);
3115
3116 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3117
3118 SDValue Threshold = DAG.getConstantFP(Val: -0x1.2f7030p+5f, DL: SL, VT);
3119 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3120
3121 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+5f, DL: SL, VT);
3122 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3123 SDValue AdjustedX =
3124 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3125
3126 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3127 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3128
3129 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K0, Flags);
3130 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3131 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K1, Flags);
3132 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3133
3134 SDValue MulExps = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1, Flags);
3135
3136 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.9f623ep-107f, DL: SL, VT);
3137 SDValue AdjustedResult =
3138 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: MulExps, N2: ResultScaleFactor, Flags);
3139
3140 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: MulExps,
3141 Flags);
3142}
3143
3144SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
3145 EVT VT = Op.getValueType();
3146
3147 if (VT == MVT::f64)
3148 return lowerFEXPF64(Op, DAG);
3149
3150 SDLoc SL(Op);
3151 SDValue X = Op.getOperand(i: 0);
3152 SDNodeFlags Flags = Op->getFlags();
3153 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10;
3154
3155 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
3156 // library behavior. Also, is known-not-daz source sufficient?
3157 if (allowApproxFunc(DAG, Flags)) { // TODO: Does this really require fast?
3158 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags)
3159 : lowerFEXPUnsafe(X, SL, DAG, Flags);
3160 }
3161
3162 if (VT.getScalarType() == MVT::f16) {
3163 if (VT.isVector())
3164 return SDValue();
3165
3166 // Nothing in half is a denormal when promoted to f32.
3167 //
3168 // exp(f16 x) ->
3169 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
3170 //
3171 // exp10(f16 x) ->
3172 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
3173 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: X, Flags);
3174 SDValue Lowered = lowerFEXPUnsafeImpl(X: Ext, SL, DAG, Flags, IsExp10);
3175 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Lowered,
3176 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3177 }
3178
3179 assert(VT == MVT::f32);
3180
3181 // Algorithm:
3182 //
3183 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
3184 //
3185 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
3186 // n = 64*m + j, 0 <= j < 64
3187 //
3188 // e^x = 2^((64*m + j + f)/64)
3189 // = (2^m) * (2^(j/64)) * 2^(f/64)
3190 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
3191 //
3192 // f = x*(64/ln(2)) - n
3193 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
3194 //
3195 // e^x = (2^m) * (2^(j/64)) * e^r
3196 //
3197 // (2^(j/64)) is precomputed
3198 //
3199 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3200 // e^r = 1 + q
3201 //
3202 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3203 //
3204 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
3205 SDNodeFlags FlagsNoContract = Flags;
3206 FlagsNoContract.setAllowContract(false);
3207
3208 SDValue PH, PL;
3209 if (Subtarget->hasFastFMAF32()) {
3210 const float c_exp = numbers::log2ef;
3211 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
3212 const float c_exp10 = 0x1.a934f0p+1f;
3213 const float cc_exp10 = 0x1.2f346ep-24f;
3214
3215 SDValue C = DAG.getConstantFP(Val: IsExp10 ? c_exp10 : c_exp, DL: SL, VT);
3216 SDValue CC = DAG.getConstantFP(Val: IsExp10 ? cc_exp10 : cc_exp, DL: SL, VT);
3217
3218 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: C, Flags);
3219 SDValue NegPH = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: PH, Flags);
3220 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: C, N3: NegPH, Flags);
3221 PL = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: CC, N3: FMA0, Flags);
3222 } else {
3223 const float ch_exp = 0x1.714000p+0f;
3224 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
3225
3226 const float ch_exp10 = 0x1.a92000p+1f;
3227 const float cl_exp10 = 0x1.4f0978p-11f;
3228
3229 SDValue CH = DAG.getConstantFP(Val: IsExp10 ? ch_exp10 : ch_exp, DL: SL, VT);
3230 SDValue CL = DAG.getConstantFP(Val: IsExp10 ? cl_exp10 : cl_exp, DL: SL, VT);
3231
3232 SDValue XAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: X);
3233 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL: SL, VT: MVT::i32);
3234 SDValue XHAsInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: XAsInt, N2: MaskConst);
3235 SDValue XH = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: XHAsInt);
3236 SDValue XL = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: XH, Flags);
3237
3238 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XH, N2: CH, Flags);
3239
3240 SDValue XLCL = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XL, N2: CL, Flags);
3241 SDValue Mad0 = getMad(DAG, SL, VT, X: XL, Y: CH, C: XLCL, Flags);
3242 PL = getMad(DAG, SL, VT, X: XH, Y: CL, C: Mad0, Flags);
3243 }
3244
3245 SDValue E = DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SL, VT, Operand: PH, Flags);
3246
3247 // It is unsafe to contract this fsub into the PH multiply.
3248 SDValue PHSubE = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: PH, N2: E, Flags: FlagsNoContract);
3249
3250 SDValue A = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: PHSubE, N2: PL, Flags);
3251 SDValue IntE = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: SL, VT: MVT::i32, Operand: E);
3252 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: A, Flags);
3253
3254 SDValue R = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: Exp2, N2: IntE, Flags);
3255
3256 SDValue UnderflowCheckConst =
3257 DAG.getConstantFP(Val: IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, DL: SL, VT);
3258
3259 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3260 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3261 SDValue Underflow =
3262 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: UnderflowCheckConst, Cond: ISD::SETOLT);
3263
3264 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Underflow, N2: Zero, N3: R);
3265
3266 if (!Flags.hasNoInfs()) {
3267 SDValue OverflowCheckConst =
3268 DAG.getConstantFP(Val: IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, DL: SL, VT);
3269 SDValue Overflow =
3270 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: OverflowCheckConst, Cond: ISD::SETOGT);
3271 SDValue Inf =
3272 DAG.getConstantFP(Val: APFloat::getInf(Sem: APFloat::IEEEsingle()), DL: SL, VT);
3273 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Overflow, N2: Inf, N3: R);
3274 }
3275
3276 return R;
3277}
3278
3279// No pow instruction or libcall to fall back on. fmul_legacy returns 0 for a
3280// zero operand even against an infinity or a NaN, so pow(x, 0) and pow(1, y)
3281// fall out as exp2(0) = 1.
3282SDValue AMDGPUTargetLowering::lowerFPOW(SDValue Op, SelectionDAG &DAG) const {
3283 EVT VT = Op.getValueType();
3284 assert(VT == MVT::f32);
3285
3286 SDLoc SL(Op);
3287 SDValue X = Op.getOperand(i: 0);
3288 SDValue Y = Op.getOperand(i: 1);
3289 SDNodeFlags Flags = Op->getFlags();
3290
3291 // log2(0) is -inf, which exp2 turns back into a finite result, so the core
3292 // goes infinite for inputs a ninf fpow still asserts about, like pow(0, 2).
3293 SDNodeFlags CoreFlags = Flags;
3294 CoreFlags.setNoInfs(false);
3295
3296 // Fast expansion: ignores denormals, NaN for a negative base.
3297 if (allowApproxFunc(DAG, Flags)) {
3298 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: X, Flags: CoreFlags);
3299 SDValue Mul =
3300 DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL: SL, VT, N1: Y, N2: Log, Flags: CoreFlags);
3301 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: Mul, Flags: CoreFlags);
3302 }
3303
3304 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: X, Flags);
3305 SDValue Log = DAG.getNode(Opcode: ISD::FLOG2, DL: SL, VT, Operand: Abs, Flags: CoreFlags);
3306 SDValue Mul = DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL: SL, VT, N1: Y, N2: Log, Flags: CoreFlags);
3307 SDValue R = DAG.getNode(Opcode: ISD::FEXP2, DL: SL, VT, Operand: Mul, Flags: CoreFlags);
3308
3309 // A base that is never negative needs neither the sign fixup nor the NaN.
3310 if (DAG.computeKnownFPClass(Op: X, InterestedClasses: fcNegative).signBitIsZeroOrNaN())
3311 return R;
3312
3313 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3314
3315 // Infinities count as integers, and every f32 >= 2^24 in magnitude is even.
3316 SDValue YTrunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: Y);
3317 SDValue YIsInt = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: YTrunc, RHS: Y, Cond: ISD::SETOEQ);
3318 SDValue YHalf =
3319 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Y, N2: DAG.getConstantFP(Val: 0.5, DL: SL, VT));
3320 SDValue YHalfTrunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: YHalf);
3321 SDValue YIsOdd =
3322 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: YIsInt,
3323 N2: DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: YHalfTrunc, RHS: YHalf, Cond: ISD::SETONE));
3324
3325 // pow(-x, odd y) = -pow(x, y). Selecting the copysign lets even y fold it.
3326 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: YIsOdd,
3327 N2: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: R, N2: X), N3: R);
3328
3329 if (Flags.hasNoNaNs())
3330 return R;
3331
3332 // A negative finite base to a non-integral power is NaN. -inf is excluded:
3333 // the core already gives pow(+inf, y). So are subnormals when flushed.
3334 FPClassTest NegFiniteMask = fcNegNormal;
3335 if (!DAG.getMachineFunction()
3336 .getDenormalMode(FPType: APFloat::IEEEsingle())
3337 .inputsAreZero())
3338 NegFiniteMask |= fcNegSubnormal;
3339 SDValue XNegFinite =
3340 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: SL, VT: SetCCVT, N1: X,
3341 N2: DAG.getTargetConstant(Val: NegFiniteMask, DL: SL, VT: MVT::i32));
3342 // Not a SETONE compare: pow(-1, NaN) needs the NaN-true behavior of !SETOEQ.
3343 SDValue NegNonInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: XNegFinite,
3344 N2: DAG.getNOT(DL: SL, Val: YIsInt, VT: SetCCVT));
3345 SDValue NaN =
3346 DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()), DL: SL, VT);
3347 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NegNonInt, N2: NaN, N3: R);
3348}
3349
3350static bool isCtlzOpc(unsigned Opc) {
3351 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
3352}
3353
3354static bool isCttzOpc(unsigned Opc) {
3355 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_POISON;
3356}
3357
3358SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op,
3359 SelectionDAG &DAG) const {
3360 auto SL = SDLoc(Op);
3361 auto Opc = Op.getOpcode();
3362 auto Arg = Op.getOperand(i: 0u);
3363 auto ResultVT = Op.getValueType();
3364
3365 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3366 return {};
3367
3368 assert(isCtlzOpc(Opc));
3369 assert(ResultVT == Arg.getValueType());
3370
3371 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3372 SDValue NumExtBits = DAG.getConstant(Val: 32u - NumBits, DL: SL, VT: MVT::i32);
3373 SDValue NewOp;
3374
3375 if (Opc == ISD::CTLZ_ZERO_POISON) {
3376 NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3377 NewOp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3378 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3379 } else {
3380 NewOp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3381 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3382 NewOp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3383 }
3384
3385 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ResultVT, Operand: NewOp);
3386}
3387
3388SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
3389 SDLoc SL(Op);
3390 SDValue Src = Op.getOperand(i: 0);
3391
3392 assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode()));
3393 bool Ctlz = isCtlzOpc(Opc: Op.getOpcode());
3394 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;
3395
3396 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_POISON ||
3397 Op.getOpcode() == ISD::CTTZ_ZERO_POISON;
3398 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3399
3400 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3401 // (ctlz hi:lo) -> (umin (ffbh src), 32)
3402 // (cttz hi:lo) -> (umin (ffbl src), 32)
3403 // (ctlz_zero_poison src) -> (ffbh src)
3404 // (cttz_zero_poison src) -> (ffbl src)
3405
3406 // 64-bit scalar version produce 32-bit result
3407 // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64)
3408 // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64)
3409 // (ctlz_zero_poison src) -> (S_FLBIT_I32_B64 src)
3410 // (cttz_zero_poison src) -> (S_FF1_I32_B64 src)
3411 SDValue NewOpr = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Src);
3412 if (!ZeroUndef) {
3413 const SDValue ConstVal = DAG.getConstant(
3414 Val: Op.getValueType().getScalarSizeInBits(), DL: SL, VT: MVT::i32);
3415 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: ConstVal);
3416 }
3417 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: Src.getValueType(), Operand: NewOpr);
3418 }
3419
3420 SDValue Lo, Hi;
3421 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3422
3423 SDValue OprLo = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Lo);
3424 SDValue OprHi = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Hi);
3425
3426 // (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64)
3427 // (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64)
3428 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
3429 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
3430
3431 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT;
3432 const SDValue Const32 = DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32);
3433 if (Ctlz)
3434 OprLo = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprLo, N2: Const32);
3435 else
3436 OprHi = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprHi, N2: Const32);
3437
3438 SDValue NewOpr;
3439 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: OprLo, N2: OprHi);
3440 if (!ZeroUndef) {
3441 const SDValue Const64 = DAG.getConstant(Val: 64, DL: SL, VT: MVT::i32);
3442 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: Const64);
3443 }
3444
3445 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: NewOpr);
3446}
3447
3448SDValue AMDGPUTargetLowering::LowerCTLS(SDValue Op, SelectionDAG &DAG) const {
3449 SDLoc SL(Op);
3450 SDValue Src = Op.getOperand(i: 0);
3451 assert(Src.getValueType() == MVT::i32 && "LowerCTLS only supports i32");
3452 SDValue Ffbh = DAG.getNode(
3453 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3454 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Src);
3455 SDValue Clamped = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: Ffbh,
3456 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3457 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: Clamped,
3458 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i32));
3459}
3460
3461SDValue AMDGPUTargetLowering::LowerINT_TO_FP16(SDValue Op, SelectionDAG &DAG,
3462 EVT FP16Ty) const {
3463 assert(FP16Ty == MVT::f16 || FP16Ty == MVT::bf16);
3464 SDLoc SL(Op);
3465 SDValue Src = Op.getOperand(i: 0);
3466 SDValue ToF32 = DAG.getNode(Opcode: Op.getOpcode(), DL: SL, VT: MVT::f32, Operand: Src);
3467 SDValue FPRoundFlag = DAG.getIntPtrConstant(Val: 0, DL: SL, /*isTarget=*/true);
3468 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: FP16Ty, N1: ToF32, N2: FPRoundFlag);
3469}
3470
3471SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
3472 bool Signed) const {
3473 // The regular method converting a 64-bit integer to float roughly consists of
3474 // 2 steps: normalization and rounding. In fact, after normalization, the
3475 // conversion from a 64-bit integer to a float is essentially the same as the
3476 // one from a 32-bit integer. The only difference is that it has more
3477 // trailing bits to be rounded. To leverage the native 32-bit conversion, a
3478 // 64-bit integer could be preprocessed and fit into a 32-bit integer then
3479 // converted into the correct float number. The basic steps for the unsigned
3480 // conversion are illustrated in the following pseudo code:
3481 //
3482 // f32 uitofp(i64 u) {
3483 // i32 hi, lo = split(u);
3484 // // Only count the leading zeros in hi as we have native support of the
3485 // // conversion from i32 to f32. If hi is all 0s, the conversion is
3486 // // reduced to a 32-bit one automatically.
3487 // i32 shamt = clz(hi); // Return 32 if hi is all 0s.
3488 // u <<= shamt;
3489 // hi, lo = split(u);
3490 // hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo.
3491 // // convert it as a 32-bit integer and scale the result back.
3492 // return uitofp(hi) * 2^(32 - shamt);
3493 // }
3494 //
3495 // The signed one follows the same principle but uses 'ffbh_i32' to count its
3496 // sign bits instead. If 'ffbh_i32' is not available, its absolute value is
3497 // converted instead followed by negation based its sign bit.
3498
3499 SDLoc SL(Op);
3500 SDValue Src = Op.getOperand(i: 0);
3501
3502 SDValue Lo, Hi;
3503 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3504 SDValue Sign;
3505 SDValue ShAmt;
3506 if (Signed && Subtarget->isGCN()) {
3507 // We also need to consider the sign bit in Lo if Hi has just sign bits,
3508 // i.e. Hi is 0 or -1. However, that only needs to take the MSB into
3509 // account. That is, the maximal shift is
3510 // - 32 if Lo and Hi have opposite signs;
3511 // - 33 if Lo and Hi have the same sign.
3512 //
3513 // Or, MaxShAmt = 33 + OppositeSign, where
3514 //
3515 // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is
3516 // - -1 if Lo and Hi have opposite signs; and
3517 // - 0 otherwise.
3518 //
3519 // All in all, ShAmt is calculated as
3520 //
3521 // umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1.
3522 //
3523 // or
3524 //
3525 // umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31).
3526 //
3527 // to reduce the critical path.
3528 SDValue OppositeSign = DAG.getNode(
3529 Opcode: ISD::SRA, DL: SL, VT: MVT::i32, N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: Lo, N2: Hi),
3530 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3531 SDValue MaxShAmt =
3532 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3533 N2: OppositeSign);
3534 // Count the leading sign bits.
3535 ShAmt = DAG.getNode(
3536 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3537 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Hi);
3538 // Different from unsigned conversion, the shift should be one bit less to
3539 // preserve the sign bit.
3540 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ShAmt,
3541 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
3542 ShAmt = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: ShAmt, N2: MaxShAmt);
3543 } else {
3544 if (Signed) {
3545 // Without 'ffbh_i32', only leading zeros could be counted. Take the
3546 // absolute value first.
3547 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: Src,
3548 N2: DAG.getConstant(Val: 63, DL: SL, VT: MVT::i64));
3549 SDValue Abs =
3550 DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64,
3551 N1: DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: Src, N2: Sign), N2: Sign);
3552 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Abs, DAG);
3553 }
3554 // Count the leading zeros.
3555 ShAmt = DAG.getNode(Opcode: ISD::CTLZ, DL: SL, VT: MVT::i32, Operand: Hi);
3556 // The shift amount for signed integers is [0, 32].
3557 }
3558 // Normalize the given 64-bit integer.
3559 SDValue Norm = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i64, N1: Src, N2: ShAmt);
3560 // Split it again.
3561 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Norm, DAG);
3562 // Calculate the adjust bit for rounding.
3563 // (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo)
3564 SDValue Adjust = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32,
3565 N1: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32), N2: Lo);
3566 // Get the 32-bit normalized integer.
3567 Norm = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Hi, N2: Adjust);
3568 // Convert the normalized 32-bit integer into f32.
3569
3570 bool UseLDEXP = isOperationLegal(Op: ISD::FLDEXP, VT: MVT::f32);
3571 unsigned Opc = Signed && UseLDEXP ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3572 SDValue FVal = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::f32, Operand: Norm);
3573
3574 // Finally, need to scale back the converted floating number as the original
3575 // 64-bit integer is converted as a 32-bit one.
3576 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3577 N2: ShAmt);
3578 // On GCN, use LDEXP directly.
3579 if (UseLDEXP)
3580 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f32, N1: FVal, N2: ShAmt);
3581
3582 // Otherwise, align 'ShAmt' to the exponent part and add it into the exponent
3583 // part directly to emulate the multiplication of 2^ShAmt. That 8-bit
3584 // exponent is enough to avoid overflowing into the sign bit.
3585 SDValue Exp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ShAmt,
3586 N2: DAG.getConstant(Val: 23, DL: SL, VT: MVT::i32));
3587 SDValue IVal =
3588 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32,
3589 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: FVal), N2: Exp);
3590 if (Signed) {
3591 // Set the sign bit.
3592 Sign = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32,
3593 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Sign),
3594 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3595 IVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: IVal, N2: Sign);
3596 }
3597 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: IVal);
3598}
3599
3600SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
3601 bool Signed) const {
3602 SDLoc SL(Op);
3603 SDValue Src = Op.getOperand(i: 0);
3604
3605 SDValue Lo, Hi;
3606 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3607
3608 SDValue CvtHi = DAG.getNode(Opcode: Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
3609 DL: SL, VT: MVT::f64, Operand: Hi);
3610
3611 SDValue CvtLo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: SL, VT: MVT::f64, Operand: Lo);
3612
3613 SDValue LdExp = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f64, N1: CvtHi,
3614 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3615 // TODO: Should this propagate fast-math-flags?
3616 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: LdExp, N2: CvtLo);
3617}
3618
3619SDValue AMDGPUTargetLowering::lowerINT_TO_FPImpl(SDValue Op, SelectionDAG &DAG,
3620 bool Signed) const {
3621 EVT DestVT = Op.getValueType();
3622 SDValue Src = Op.getOperand(i: 0);
3623 EVT SrcVT = Src.getValueType();
3624 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3625 unsigned CvtOpc = Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3626
3627 if (SrcVT == MVT::i16) {
3628 if (DestVT == MVT::f16)
3629 return Op;
3630
3631 SDLoc DL(Op);
3632 SDValue Ext = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i32, Operand: Src);
3633 return DAG.getNode(Opcode: CvtOpc, DL, VT: DestVT, Operand: Ext);
3634 }
3635
3636 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3637 return LowerINT_TO_FP16(Op, DAG, FP16Ty: DestVT);
3638
3639 if (SrcVT != MVT::i64)
3640 return Op;
3641
3642 if (DestVT == MVT::f32)
3643 return LowerINT_TO_FP32(Op, DAG, Signed);
3644
3645 assert(DestVT == MVT::f64);
3646 return LowerINT_TO_FP64(Op, DAG, Signed);
3647}
3648
3649SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
3650 SelectionDAG &DAG) const {
3651 return lowerINT_TO_FPImpl(Op, DAG, Signed: false);
3652}
3653
3654SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
3655 SelectionDAG &DAG) const {
3656 return lowerINT_TO_FPImpl(Op, DAG, Signed: true);
3657}
3658
3659SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
3660 bool Signed) const {
3661 SDLoc SL(Op);
3662
3663 SDValue Src = Op.getOperand(i: 0);
3664 EVT SrcVT = Src.getValueType();
3665
3666 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3667
3668 // The basic idea of converting a floating point number into a pair of 32-bit
3669 // integers is illustrated as follows:
3670 //
3671 // tf := trunc(val);
3672 // hif := floor(tf * 2^-32);
3673 // lof := tf - hif * 2^32; // lof is always positive due to floor.
3674 // hi := fptoi(hif);
3675 // lo := fptoi(lof);
3676 //
3677 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: SrcVT, Operand: Src);
3678 SDValue Sign;
3679 if (Signed && SrcVT == MVT::f32) {
3680 // However, a 32-bit floating point number has only 23 bits mantissa and
3681 // it's not enough to hold all the significant bits of `lof` if val is
3682 // negative. To avoid the loss of precision, We need to take the absolute
3683 // value after truncating and flip the result back based on the original
3684 // signedness.
3685 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i32,
3686 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Trunc),
3687 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3688 Trunc = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: SrcVT, Operand: Trunc);
3689 }
3690
3691 SDValue K0, K1;
3692 if (SrcVT == MVT::f64) {
3693 K0 = DAG.getConstantFP(
3694 Val: llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), DL: SL,
3695 VT: SrcVT);
3696 K1 = DAG.getConstantFP(
3697 Val: llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), DL: SL,
3698 VT: SrcVT);
3699 } else {
3700 K0 = DAG.getConstantFP(
3701 Val: llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), DL: SL, VT: SrcVT);
3702 K1 = DAG.getConstantFP(
3703 Val: llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), DL: SL, VT: SrcVT);
3704 }
3705 // TODO: Should this propagate fast-math-flags?
3706 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: SrcVT, N1: Trunc, N2: K0);
3707
3708 SDValue FloorMul = DAG.getNode(Opcode: ISD::FFLOOR, DL: SL, VT: SrcVT, Operand: Mul);
3709
3710 SDValue Fma = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: SrcVT, N1: FloorMul, N2: K1, N3: Trunc);
3711
3712 SDValue Hi = DAG.getNode(Opcode: (Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
3713 : ISD::FP_TO_UINT,
3714 DL: SL, VT: MVT::i32, Operand: FloorMul);
3715 SDValue Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SL, VT: MVT::i32, Operand: Fma);
3716
3717 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3718 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Lo, Hi}));
3719
3720 if (Signed && SrcVT == MVT::f32) {
3721 assert(Sign);
3722 // Flip the result based on the signedness, which is either all 0s or 1s.
3723 Sign = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3724 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Sign, Sign}));
3725 // r := xor(r, sign) - sign;
3726 Result =
3727 DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i64,
3728 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64, N1: Result, N2: Sign), N2: Sign);
3729 }
3730
3731 return Result;
3732}
3733
3734SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
3735 SDLoc DL(Op);
3736 SDValue N0 = Op.getOperand(i: 0);
3737
3738 // Convert to target node to get known bits
3739 if (N0.getValueType() == MVT::f32)
3740 return DAG.getNode(Opcode: AMDGPUISD::FP_TO_FP16, DL, VT: Op.getValueType(), Operand: N0);
3741
3742 if (Op->getFlags().hasApproximateFuncs()) {
3743 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
3744 return SDValue();
3745 }
3746
3747 return LowerF64ToF16Safe(Src: N0, DL, DAG);
3748}
3749
3750// return node in i32
3751SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
3752 SelectionDAG &DAG) const {
3753 assert(Src.getSimpleValueType() == MVT::f64);
3754
3755 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
3756 // TODO: We can generate better code for True16.
3757 const unsigned ExpMask = 0x7ff;
3758 const unsigned ExpBiasf64 = 1023;
3759 const unsigned ExpBiasf16 = 15;
3760 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3761 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
3762 SDValue U = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Src);
3763 SDValue UH = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: U,
3764 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3765 UH = DAG.getZExtOrTrunc(Op: UH, DL, VT: MVT::i32);
3766 U = DAG.getZExtOrTrunc(Op: U, DL, VT: MVT::i32);
3767 SDValue E = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3768 N2: DAG.getConstant(Val: 20, DL, VT: MVT::i64));
3769 E = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: E,
3770 N2: DAG.getConstant(Val: ExpMask, DL, VT: MVT::i32));
3771 // Subtract the fp64 exponent bias (1023) to get the real exponent and
3772 // add the f16 bias (15) to get the biased exponent for the f16 format.
3773 E = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: E,
3774 N2: DAG.getConstant(Val: -ExpBiasf64 + ExpBiasf16, DL, VT: MVT::i32));
3775
3776 SDValue M = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3777 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32));
3778 M = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M,
3779 N2: DAG.getConstant(Val: 0xffe, DL, VT: MVT::i32));
3780
3781 SDValue MaskedSig = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: UH,
3782 N2: DAG.getConstant(Val: 0x1ff, DL, VT: MVT::i32));
3783 MaskedSig = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: MaskedSig, N2: U);
3784
3785 SDValue Lo40Set = DAG.getSelectCC(DL, LHS: MaskedSig, RHS: Zero, True: Zero, False: One, Cond: ISD::SETEQ);
3786 M = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M, N2: Lo40Set);
3787
3788 // (M != 0 ? 0x0200 : 0) | 0x7c00;
3789 SDValue I = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
3790 N1: DAG.getSelectCC(DL, LHS: M, RHS: Zero, True: DAG.getConstant(Val: 0x0200, DL, VT: MVT::i32),
3791 False: Zero, Cond: ISD::SETNE), N2: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32));
3792
3793 // N = M | (E << 12);
3794 SDValue N = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3795 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: E,
3796 N2: DAG.getConstant(Val: 12, DL, VT: MVT::i32)));
3797
3798 // B = clamp(1-E, 0, 13);
3799 SDValue OneSubExp = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
3800 N1: One, N2: E);
3801 SDValue B = DAG.getNode(Opcode: ISD::SMAX, DL, VT: MVT::i32, N1: OneSubExp, N2: Zero);
3802 B = DAG.getNode(Opcode: ISD::SMIN, DL, VT: MVT::i32, N1: B,
3803 N2: DAG.getConstant(Val: 13, DL, VT: MVT::i32));
3804
3805 SDValue SigSetHigh = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3806 N2: DAG.getConstant(Val: 0x1000, DL, VT: MVT::i32));
3807
3808 SDValue D = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SigSetHigh, N2: B);
3809 SDValue D0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: D, N2: B);
3810 SDValue D1 = DAG.getSelectCC(DL, LHS: D0, RHS: SigSetHigh, True: One, False: Zero, Cond: ISD::SETNE);
3811 D = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: D, N2: D1);
3812
3813 SDValue V = DAG.getSelectCC(DL, LHS: E, RHS: One, True: D, False: N, Cond: ISD::SETLT);
3814 SDValue VLow3 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: V,
3815 N2: DAG.getConstant(Val: 0x7, DL, VT: MVT::i32));
3816 V = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: V,
3817 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3818 SDValue V0 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 3, DL, VT: MVT::i32),
3819 True: One, False: Zero, Cond: ISD::SETEQ);
3820 SDValue V1 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 5, DL, VT: MVT::i32),
3821 True: One, False: Zero, Cond: ISD::SETGT);
3822 V1 = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: V0, N2: V1);
3823 V = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: V, N2: V1);
3824
3825 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 30, DL, VT: MVT::i32),
3826 True: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32), False: V, Cond: ISD::SETGT);
3827 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 1039, DL, VT: MVT::i32),
3828 True: I, False: V, Cond: ISD::SETEQ);
3829
3830 // Extract the sign bit.
3831 SDValue Sign = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3832 N2: DAG.getConstant(Val: 16, DL, VT: MVT::i32));
3833 Sign = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Sign,
3834 N2: DAG.getConstant(Val: 0x8000, DL, VT: MVT::i32));
3835
3836 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Sign, N2: V);
3837}
3838
3839SDValue AMDGPUTargetLowering::LowerFP_TO_INT(const SDValue Op,
3840 SelectionDAG &DAG) const {
3841 SDValue Src = Op.getOperand(i: 0);
3842 unsigned OpOpcode = Op.getOpcode();
3843 EVT SrcVT = Src.getValueType();
3844 EVT DestVT = Op.getValueType();
3845
3846 // Will be selected natively
3847 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3848 return Op;
3849
3850 if (SrcVT == MVT::bf16 || (SrcVT == MVT::f16 && DestVT == MVT::i32)) {
3851 SDLoc DL(Op);
3852 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3853 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DestVT, Operand: PromotedSrc);
3854 }
3855
3856 // Promote i16 to i32
3857 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3858 SDLoc DL(Op);
3859
3860 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3861 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToInt32);
3862 }
3863
3864 if (DestVT != MVT::i64)
3865 return Op;
3866
3867 if (SrcVT == MVT::f16 ||
3868 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
3869 SDLoc DL(Op);
3870
3871 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3872 unsigned Ext =
3873 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3874 return DAG.getNode(Opcode: Ext, DL, VT: MVT::i64, Operand: FpToInt32);
3875 }
3876
3877 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3878 return LowerFP_TO_INT64(Op, DAG, Signed: OpOpcode == ISD::FP_TO_SINT);
3879
3880 return SDValue();
3881}
3882
3883SDValue AMDGPUTargetLowering::LowerFP_TO_INT_SAT(const SDValue Op,
3884 SelectionDAG &DAG) const {
3885 SDValue Src = Op.getOperand(i: 0);
3886 unsigned OpOpcode = Op.getOpcode();
3887 EVT SrcVT = Src.getValueType();
3888 EVT DstVT = Op.getValueType();
3889 SDValue SatVTOp = Op.getNode()->getOperand(Num: 1);
3890 EVT SatVT = cast<VTSDNode>(Val&: SatVTOp)->getVT();
3891 SDLoc DL(Op);
3892
3893 uint64_t DstWidth = DstVT.getScalarSizeInBits();
3894 uint64_t SatWidth = SatVT.getScalarSizeInBits();
3895 assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
3896
3897 // Scalar cases will be selected natively to v_cvt_/s_cvt_ instructions.
3898 // v2f32 -> v2i16 will be selected natively to v_cvt_pk_[iu]16_f32.
3899 if (SatWidth == DstWidth) {
3900 if ((DstVT == MVT::i32 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
3901 (DstVT == MVT::i16 && (SrcVT == MVT::f16 || SrcVT == MVT::f32)) ||
3902 (DstVT == MVT::v2i16 && SrcVT == MVT::v2f32))
3903 return Op;
3904 }
3905
3906 // Vectors can only be selected natively.
3907 if (DstVT.isVector())
3908 return SDValue();
3909
3910 // Perform all saturation at selected width (i16 or i32) and truncate
3911 if (SatWidth < DstWidth && SatWidth <= 32) {
3912 // For f16 conversion with sub-i16 saturation perform saturation
3913 // at i16, if available in the target. This removes the need for extra f16
3914 // to f32 conversion. For all the others use i32.
3915 MVT ResultVT =
3916 Subtarget->has16BitInsts() && SrcVT == MVT::f16 && SatWidth < 16
3917 ? MVT::i16
3918 : MVT::i32;
3919
3920 const SDValue ResultVTOp = DAG.getValueType(ResultVT);
3921 const uint64_t ResultWidth = ResultVT.getScalarSizeInBits();
3922
3923 // First, convert input float into selected integer (i16 or i32)
3924 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: ResultVT, N1: Src, N2: ResultVTOp);
3925 SDValue IntSatVal;
3926
3927 // Then, clamp at the saturation width using either i16 or i32 instructions
3928 if (OpOpcode == ISD::FP_TO_SINT_SAT) {
3929 SDValue MinConst = DAG.getConstant(
3930 Val: APInt::getSignedMaxValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3931 SDValue MaxConst = DAG.getConstant(
3932 Val: APInt::getSignedMinValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3933 SDValue MinVal = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3934 IntSatVal = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ResultVT, N1: MinVal, N2: MaxConst);
3935 } else {
3936 SDValue MinConst = DAG.getConstant(
3937 Val: APInt::getMaxValue(numBits: SatWidth).zext(width: ResultWidth), DL, VT: ResultVT);
3938 IntSatVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3939 }
3940
3941 // Finally, after saturating at i16 or i32 fit into the destination type
3942 return DAG.getExtOrTrunc(IsSigned: OpOpcode == ISD::FP_TO_SINT_SAT, Op: IntSatVal, DL,
3943 VT: DstVT);
3944 }
3945
3946 // SatWidth == DstWidth or SatWidth > 32
3947
3948 // Saturate at i32 for i64 dst and f16/bf16 src (will invoke f16 promotion
3949 // below)
3950 if (DstVT == MVT::i64 &&
3951 (SrcVT == MVT::f16 || SrcVT == MVT::bf16 ||
3952 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP))) {
3953 const SDValue Int32VTOp = DAG.getValueType(MVT::i32);
3954 return DAG.getNode(Opcode: OpOpcode, DL, VT: DstVT, N1: Src, N2: Int32VTOp);
3955 }
3956
3957 // Promote f16/bf16 src to f32 for i32 conversion
3958 if (DstVT == MVT::i32 && (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
3959 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3960 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DstVT, N1: PromotedSrc, N2: SatVTOp);
3961 }
3962
3963 // For DstWidth < 16, promote i1 and i8 dst to i16 (if legal) with sub-i16
3964 // saturation. For DstWidth == 16, promote i16 dst to i32 with sub-i32
3965 // saturation; this covers i16.f32 and i16.f64
3966 if (DstWidth < 32) {
3967 // Note: this triggers SatWidth < DstWidth above to generate saturated
3968 // truncate by requesting MVT::i16/i32 destination with SatWidth < 16/32.
3969 MVT PromoteVT =
3970 (DstWidth < 16 && Subtarget->has16BitInsts()) ? MVT::i16 : MVT::i32;
3971 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: PromoteVT, N1: Src, N2: SatVTOp);
3972 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: FpToInt);
3973 }
3974
3975 // TODO: can we implement i64 dst for f32/f64?
3976
3977 return SDValue();
3978}
3979
3980SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
3981 SelectionDAG &DAG) const {
3982 EVT ExtraVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3983 MVT VT = Op.getSimpleValueType();
3984 MVT ScalarVT = VT.getScalarType();
3985
3986 assert(VT.isVector());
3987
3988 SDValue Src = Op.getOperand(i: 0);
3989 SDLoc DL(Op);
3990
3991 // TODO: Don't scalarize on Evergreen?
3992 unsigned NElts = VT.getVectorNumElements();
3993 SmallVector<SDValue, 8> Args;
3994 DAG.ExtractVectorElements(Op: Src, Args, Start: 0, Count: NElts);
3995
3996 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
3997 for (unsigned I = 0; I < NElts; ++I)
3998 Args[I] = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: ScalarVT, N1: Args[I], N2: VTOp);
3999
4000 return DAG.getBuildVector(VT, DL, Ops: Args);
4001}
4002
4003//===----------------------------------------------------------------------===//
4004// Custom DAG optimizations
4005//===----------------------------------------------------------------------===//
4006
4007static bool isU24(SDValue Op, SelectionDAG &DAG) {
4008 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
4009}
4010
4011static bool isI24(SDValue Op, SelectionDAG &DAG) {
4012 EVT VT = Op.getValueType();
4013 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
4014 // as unsigned 24-bit values.
4015 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24;
4016}
4017
4018static SDValue simplifyMul24(SDNode *Node24,
4019 TargetLowering::DAGCombinerInfo &DCI) {
4020 SelectionDAG &DAG = DCI.DAG;
4021 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4022 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
4023
4024 SDValue LHS = IsIntrin ? Node24->getOperand(Num: 1) : Node24->getOperand(Num: 0);
4025 SDValue RHS = IsIntrin ? Node24->getOperand(Num: 2) : Node24->getOperand(Num: 1);
4026 unsigned NewOpcode = Node24->getOpcode();
4027 if (IsIntrin) {
4028 unsigned IID = Node24->getConstantOperandVal(Num: 0);
4029 switch (IID) {
4030 case Intrinsic::amdgcn_mul_i24:
4031 NewOpcode = AMDGPUISD::MUL_I24;
4032 break;
4033 case Intrinsic::amdgcn_mul_u24:
4034 NewOpcode = AMDGPUISD::MUL_U24;
4035 break;
4036 case Intrinsic::amdgcn_mulhi_i24:
4037 NewOpcode = AMDGPUISD::MULHI_I24;
4038 break;
4039 case Intrinsic::amdgcn_mulhi_u24:
4040 NewOpcode = AMDGPUISD::MULHI_U24;
4041 break;
4042 default:
4043 llvm_unreachable("Expected 24-bit mul intrinsic");
4044 }
4045 }
4046
4047 APInt Demanded = APInt::getLowBitsSet(numBits: LHS.getValueSizeInBits(), loBitsSet: 24);
4048
4049 if (isNullConstant(V: LHS) || isNullConstant(V: RHS))
4050 return DAG.getConstant(Val: 0, DL: SDLoc(Node24), VT: Node24->getValueType(ResNo: 0));
4051
4052 // First try SimplifyDemandedBits which can simplify the nodes used by our
4053 // operands if this node is the only user.
4054 if (LHS.hasOneUse() && TLI.SimplifyDemandedBits(Op: LHS, DemandedBits: Demanded, DCI))
4055 return SDValue(Node24, 0);
4056 if (RHS.hasOneUse() && TLI.SimplifyDemandedBits(Op: RHS, DemandedBits: Demanded, DCI))
4057 return SDValue(Node24, 0);
4058
4059 // Then try SimplifyMultipleUseDemandedBits which allows the operands to have
4060 // other uses, but will only perform simplifications that involve bypassing
4061 // some nodes for this user.
4062 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(Op: LHS, DemandedBits: Demanded, DAG);
4063 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(Op: RHS, DemandedBits: Demanded, DAG);
4064 if (DemandedLHS || DemandedRHS)
4065 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(Node24), VTList: Node24->getVTList(),
4066 N1: DemandedLHS ? DemandedLHS : LHS,
4067 N2: DemandedRHS ? DemandedRHS : RHS);
4068
4069 return SDValue();
4070}
4071
4072template <typename IntTy>
4073static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
4074 uint32_t Width, const SDLoc &DL) {
4075 if (Width + Offset < 32) {
4076 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
4077 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
4078 if constexpr (std::is_signed_v<IntTy>) {
4079 return DAG.getSignedConstant(Val: Result, DL, VT: MVT::i32);
4080 } else {
4081 return DAG.getConstant(Result, DL, MVT::i32);
4082 }
4083 }
4084
4085 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
4086}
4087
4088static bool hasVolatileUser(SDNode *Val) {
4089 for (SDNode *U : Val->users()) {
4090 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: U)) {
4091 if (M->isVolatile())
4092 return true;
4093 }
4094 }
4095
4096 return false;
4097}
4098
4099bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
4100 // i32 vectors are the canonical memory type.
4101 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
4102 return false;
4103
4104 if (!VT.isByteSized())
4105 return false;
4106
4107 unsigned Size = VT.getStoreSize();
4108
4109 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
4110 return false;
4111
4112 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
4113 return false;
4114
4115 return true;
4116}
4117
4118// Replace load of an illegal type with a bitcast from a load of a friendlier
4119// type.
4120SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
4121 DAGCombinerInfo &DCI) const {
4122 if (!DCI.isBeforeLegalize())
4123 return SDValue();
4124
4125 LoadSDNode *LN = cast<LoadSDNode>(Val: N);
4126 if (!LN->isSimple() || !ISD::isNormalLoad(N: LN) || hasVolatileUser(Val: LN))
4127 return SDValue();
4128
4129 SDLoc SL(N);
4130 SelectionDAG &DAG = DCI.DAG;
4131 EVT VT = LN->getMemoryVT();
4132
4133 unsigned Size = VT.getStoreSize();
4134 Align Alignment = LN->getAlign();
4135 if (Alignment < Size && isTypeLegal(VT)) {
4136 unsigned IsFast;
4137 unsigned AS = LN->getAddressSpace();
4138
4139 // Expand unaligned loads earlier than legalization. Due to visitation order
4140 // problems during legalization, the emitted instructions to pack and unpack
4141 // the bytes again are not eliminated in the case of an unaligned copy.
4142 if (!allowsMisalignedMemoryAccesses(
4143 VT, AddrSpace: AS, Alignment, Flags: LN->getMemOperand()->getFlags(), &IsFast)) {
4144 if (VT.isVector())
4145 return SplitVectorLoad(Op: SDValue(LN, 0), DAG);
4146
4147 SDValue Ops[2];
4148 std::tie(args&: Ops[0], args&: Ops[1]) = expandUnalignedLoad(LD: LN, DAG);
4149
4150 return DAG.getMergeValues(Ops, dl: SDLoc(N));
4151 }
4152
4153 if (!IsFast)
4154 return SDValue();
4155 }
4156
4157 if (!shouldCombineMemoryType(VT))
4158 return SDValue();
4159
4160 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4161
4162 SDValue NewLoad
4163 = DAG.getLoad(VT: NewVT, dl: SL, Chain: LN->getChain(),
4164 Ptr: LN->getBasePtr(), MMO: LN->getMemOperand());
4165
4166 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewLoad);
4167 DCI.CombineTo(N, Res0: BC, Res1: NewLoad.getValue(R: 1));
4168 return SDValue(N, 0);
4169}
4170
4171// Replace store of an illegal type with a store of a bitcast to a friendlier
4172// type.
4173SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
4174 DAGCombinerInfo &DCI) const {
4175 if (!DCI.isBeforeLegalize())
4176 return SDValue();
4177
4178 StoreSDNode *SN = cast<StoreSDNode>(Val: N);
4179 if (!SN->isSimple() || !ISD::isNormalStore(N: SN))
4180 return SDValue();
4181
4182 EVT VT = SN->getMemoryVT();
4183 unsigned Size = VT.getStoreSize();
4184
4185 SDLoc SL(N);
4186 SelectionDAG &DAG = DCI.DAG;
4187 Align Alignment = SN->getAlign();
4188 if (Alignment < Size && isTypeLegal(VT)) {
4189 unsigned IsFast;
4190 unsigned AS = SN->getAddressSpace();
4191
4192 // Expand unaligned stores earlier than legalization. Due to visitation
4193 // order problems during legalization, the emitted instructions to pack and
4194 // unpack the bytes again are not eliminated in the case of an unaligned
4195 // copy.
4196 if (!allowsMisalignedMemoryAccesses(
4197 VT, AddrSpace: AS, Alignment, Flags: SN->getMemOperand()->getFlags(), &IsFast)) {
4198 if (VT.isVector())
4199 return SplitVectorStore(Op: SDValue(SN, 0), DAG);
4200
4201 return expandUnalignedStore(ST: SN, DAG);
4202 }
4203
4204 if (!IsFast)
4205 return SDValue();
4206 }
4207
4208 if (!shouldCombineMemoryType(VT))
4209 return SDValue();
4210
4211 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4212 SDValue Val = SN->getValue();
4213
4214 // DCI.AddToWorklist(Val.getNode());
4215
4216 bool OtherUses = !Val.hasOneUse();
4217 SDValue CastVal = DAG.getBitcast(VT: NewVT, V: Val);
4218 if (OtherUses) {
4219 SDValue CastBack = DAG.getBitcast(VT, V: CastVal);
4220 DAG.ReplaceAllUsesOfValueWith(From: Val, To: CastBack);
4221 }
4222
4223 return DAG.getStore(Chain: SN->getChain(), dl: SL, Val: CastVal,
4224 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
4225}
4226
4227// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
4228// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
4229// issues.
4230SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
4231 DAGCombinerInfo &DCI) const {
4232 SelectionDAG &DAG = DCI.DAG;
4233 SDValue N0 = N->getOperand(Num: 0);
4234
4235 // (vt2 (assertzext (truncate vt0:x), vt1)) ->
4236 // (vt2 (truncate (assertzext vt0:x, vt1)))
4237 if (N0.getOpcode() == ISD::TRUNCATE) {
4238 SDValue N1 = N->getOperand(Num: 1);
4239 EVT ExtVT = cast<VTSDNode>(Val&: N1)->getVT();
4240 SDLoc SL(N);
4241
4242 SDValue Src = N0.getOperand(i: 0);
4243 EVT SrcVT = Src.getValueType();
4244 if (SrcVT.bitsGE(VT: ExtVT)) {
4245 SDValue NewInReg = DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: SrcVT, N1: Src, N2: N1);
4246 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: N->getValueType(ResNo: 0), Operand: NewInReg);
4247 }
4248 }
4249
4250 return SDValue();
4251}
4252
4253SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
4254 SDNode *N, DAGCombinerInfo &DCI) const {
4255 unsigned IID = N->getConstantOperandVal(Num: 0);
4256 switch (IID) {
4257 case Intrinsic::amdgcn_mul_i24:
4258 case Intrinsic::amdgcn_mul_u24:
4259 case Intrinsic::amdgcn_mulhi_i24:
4260 case Intrinsic::amdgcn_mulhi_u24:
4261 return simplifyMul24(Node24: N, DCI);
4262 case Intrinsic::amdgcn_fract:
4263 case Intrinsic::amdgcn_rsq:
4264 case Intrinsic::amdgcn_rcp_legacy:
4265 case Intrinsic::amdgcn_rsq_legacy:
4266 case Intrinsic::amdgcn_rsq_clamp:
4267 case Intrinsic::amdgcn_tanh:
4268 case Intrinsic::amdgcn_prng_b32: {
4269 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
4270 SDValue Src = N->getOperand(Num: 1);
4271 return Src.isUndef() ? Src : SDValue();
4272 }
4273 case Intrinsic::amdgcn_frexp_exp: {
4274 // frexp_exp (fneg x) -> frexp_exp x
4275 // frexp_exp (fabs x) -> frexp_exp x
4276 // frexp_exp (fneg (fabs x)) -> frexp_exp x
4277 SDValue Src = N->getOperand(Num: 1);
4278 SDValue PeekSign = peekFPSignOps(Val: Src);
4279 if (PeekSign == Src)
4280 return SDValue();
4281 return SDValue(DCI.DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: PeekSign),
4282 0);
4283 }
4284 default:
4285 return SDValue();
4286 }
4287}
4288
4289/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
4290/// binary operation \p Opc to it with the corresponding constant operands.
4291SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
4292 DAGCombinerInfo &DCI, const SDLoc &SL,
4293 unsigned Opc, SDValue LHS,
4294 uint32_t ValLo, uint32_t ValHi) const {
4295 SelectionDAG &DAG = DCI.DAG;
4296 SDValue Lo, Hi;
4297 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: LHS, DAG);
4298
4299 SDValue LoRHS = DAG.getConstant(Val: ValLo, DL: SL, VT: MVT::i32);
4300 SDValue HiRHS = DAG.getConstant(Val: ValHi, DL: SL, VT: MVT::i32);
4301
4302 SDValue LoAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Lo, N2: LoRHS);
4303 SDValue HiAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Hi, N2: HiRHS);
4304
4305 // Re-visit the ands. It's possible we eliminated one of them and it could
4306 // simplify the vector.
4307 DCI.AddToWorklist(N: Lo.getNode());
4308 DCI.AddToWorklist(N: Hi.getNode());
4309
4310 SDValue Vec = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoAnd, HiAnd});
4311 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
4312}
4313
4314SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
4315 DAGCombinerInfo &DCI) const {
4316 EVT VT = N->getValueType(ResNo: 0);
4317 SDValue LHS = N->getOperand(Num: 0);
4318 SDValue RHS = N->getOperand(Num: 1);
4319 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4320 SDLoc SL(N);
4321 SelectionDAG &DAG = DCI.DAG;
4322
4323 unsigned RHSVal;
4324 if (CRHS) {
4325 RHSVal = CRHS->getZExtValue();
4326 if (!RHSVal)
4327 return LHS;
4328
4329 switch (LHS->getOpcode()) {
4330 default:
4331 break;
4332 case ISD::ZERO_EXTEND:
4333 case ISD::SIGN_EXTEND:
4334 case ISD::ANY_EXTEND: {
4335 SDValue X = LHS->getOperand(Num: 0);
4336
4337 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
4338 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16)) {
4339 // Prefer build_vector as the canonical form if packed types are legal.
4340 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
4341 SDValue Vec = DAG.getBuildVector(
4342 VT: MVT::v2i16, DL: SL,
4343 Ops: {DAG.getConstant(Val: 0, DL: SL, VT: MVT::i16), LHS->getOperand(Num: 0)});
4344 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Vec);
4345 }
4346
4347 // shl (ext x) => zext (shl x), if shift does not overflow int
4348 if (VT != MVT::i64)
4349 break;
4350 KnownBits Known = DAG.computeKnownBits(Op: X);
4351 unsigned LZ = Known.countMinLeadingZeros();
4352 if (LZ < RHSVal)
4353 break;
4354 EVT XVT = X.getValueType();
4355 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: XVT, N1: X, N2: SDValue(CRHS, 0));
4356 return DAG.getZExtOrTrunc(Op: Shl, DL: SL, VT);
4357 }
4358 }
4359 }
4360
4361 if (VT.getScalarType() != MVT::i64)
4362 return SDValue();
4363
4364 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4365 // common case, splitting this into a move and a 32-bit shift is faster and
4366 // the same code size.
4367 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4368
4369 EVT ElementType = VT.getScalarType();
4370 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4371 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4372
4373 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4374 return SDValue();
4375 SDValue ShiftAmt;
4376
4377 if (CRHS) {
4378 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4379 VT: TargetType);
4380 } else {
4381 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4382 const SDValue ShiftMask =
4383 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4384 // This AND instruction will clamp out of bounds shift values.
4385 // It will also be removed during later instruction selection.
4386 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4387 }
4388
4389 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: LHS);
4390 SDValue NewShift =
4391 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: TargetType, N1: Lo, N2: ShiftAmt, Flags: N->getFlags());
4392
4393 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4394 SDValue Vec;
4395
4396 if (VT.isVector()) {
4397 EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4398 unsigned NElts = TargetType.getVectorNumElements();
4399 SmallVector<SDValue, 8> HiOps;
4400 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4401
4402 DAG.ExtractVectorElements(Op: NewShift, Args&: HiOps, Start: 0, Count: NElts);
4403 for (unsigned I = 0; I != NElts; ++I)
4404 HiAndLoOps[2 * I + 1] = HiOps[I];
4405 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4406 } else {
4407 EVT ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4408 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {Zero, NewShift});
4409 }
4410 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4411}
4412
4413SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
4414 DAGCombinerInfo &DCI) const {
4415 SDValue RHS = N->getOperand(Num: 1);
4416 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4417 EVT VT = N->getValueType(ResNo: 0);
4418 SDValue LHS = N->getOperand(Num: 0);
4419 SelectionDAG &DAG = DCI.DAG;
4420 SDLoc SL(N);
4421
4422 if (VT.getScalarType() != MVT::i64)
4423 return SDValue();
4424
4425 // For C >= 32
4426 // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
4427
4428 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4429 // common case, splitting this into a move and a 32-bit shift is faster and
4430 // the same code size.
4431 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4432
4433 EVT ElementType = VT.getScalarType();
4434 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4435 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4436
4437 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4438 return SDValue();
4439
4440 SDValue ShiftFullAmt =
4441 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4442 SDValue ShiftAmt;
4443 if (CRHS) {
4444 unsigned RHSVal = CRHS->getZExtValue();
4445 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4446 VT: TargetType);
4447 } else if (Known.getMinValue().getZExtValue() ==
4448 (ElementType.getSizeInBits() - 1)) {
4449 ShiftAmt = ShiftFullAmt;
4450 } else {
4451 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4452 const SDValue ShiftMask =
4453 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4454 // This AND instruction will clamp out of bounds shift values.
4455 // It will also be removed during later instruction selection.
4456 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4457 }
4458
4459 EVT ConcatType;
4460 SDValue Hi;
4461 SDLoc LHSSL(LHS);
4462 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4463 if (VT.isVector()) {
4464 unsigned NElts = TargetType.getVectorNumElements();
4465 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4466 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4467 SmallVector<SDValue, 8> HiOps(NElts);
4468 SmallVector<SDValue, 16> HiAndLoOps;
4469
4470 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, Start: 0, Count: NElts * 2);
4471 for (unsigned I = 0; I != NElts; ++I) {
4472 HiOps[I] = HiAndLoOps[2 * I + 1];
4473 }
4474 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4475 } else {
4476 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4477 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4478 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4479 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4480 }
4481
4482 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
4483 SDValue NewShift, HiShift;
4484 if (KnownLHS.isNegative()) {
4485 HiShift = DAG.getAllOnesConstant(DL: SL, VT: TargetType);
4486 NewShift =
4487 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4488 } else if (CRHS &&
4489 CRHS->getZExtValue() == (ElementType.getSizeInBits() - 1)) {
4490 NewShift = HiShift =
4491 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4492 } else {
4493 Hi = DAG.getFreeze(V: Hi);
4494 HiShift = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftFullAmt);
4495 NewShift =
4496 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4497 }
4498
4499 SDValue Vec;
4500 if (VT.isVector()) {
4501 unsigned NElts = TargetType.getVectorNumElements();
4502 SmallVector<SDValue, 8> HiOps;
4503 SmallVector<SDValue, 8> LoOps;
4504 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
4505
4506 DAG.ExtractVectorElements(Op: HiShift, Args&: HiOps, Start: 0, Count: NElts);
4507 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4508 for (unsigned I = 0; I != NElts; ++I) {
4509 HiAndLoOps[2 * I + 1] = HiOps[I];
4510 HiAndLoOps[2 * I] = LoOps[I];
4511 }
4512 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4513 } else {
4514 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, HiShift});
4515 }
4516 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4517}
4518
4519SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
4520 DAGCombinerInfo &DCI) const {
4521 SDValue RHS = N->getOperand(Num: 1);
4522 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4523 EVT VT = N->getValueType(ResNo: 0);
4524 SDValue LHS = N->getOperand(Num: 0);
4525 SelectionDAG &DAG = DCI.DAG;
4526 SDLoc SL(N);
4527 unsigned RHSVal;
4528
4529 if (CRHS) {
4530 RHSVal = CRHS->getZExtValue();
4531
4532 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
4533 // this improves the ability to match BFE patterns in isel.
4534 if (LHS.getOpcode() == ISD::AND) {
4535 if (auto *Mask = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
4536 unsigned MaskIdx, MaskLen;
4537 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4538 MaskIdx == RHSVal) {
4539 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT,
4540 N1: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 0),
4541 N2: N->getOperand(Num: 1)),
4542 N2: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 1),
4543 N2: N->getOperand(Num: 1)));
4544 }
4545 }
4546 }
4547 }
4548
4549 if (VT.getScalarType() != MVT::i64)
4550 return SDValue();
4551
4552 // for C >= 32
4553 // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
4554
4555 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4556 // common case, splitting this into a move and a 32-bit shift is faster and
4557 // the same code size.
4558 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4559
4560 EVT ElementType = VT.getScalarType();
4561 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4562 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4563
4564 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4565 return SDValue();
4566
4567 SDValue ShiftAmt;
4568 if (CRHS) {
4569 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4570 VT: TargetType);
4571 } else {
4572 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4573 const SDValue ShiftMask =
4574 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4575 // This AND instruction will clamp out of bounds shift values.
4576 // It will also be removed during later instruction selection.
4577 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4578 }
4579
4580 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4581 EVT ConcatType;
4582 SDValue Hi;
4583 SDLoc LHSSL(LHS);
4584 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4585 if (VT.isVector()) {
4586 unsigned NElts = TargetType.getVectorNumElements();
4587 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4588 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4589 SmallVector<SDValue, 8> HiOps(NElts);
4590 SmallVector<SDValue, 16> HiAndLoOps;
4591
4592 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, /*Start=*/0, Count: NElts * 2);
4593 for (unsigned I = 0; I != NElts; ++I)
4594 HiOps[I] = HiAndLoOps[2 * I + 1];
4595 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4596 } else {
4597 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4598 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4599 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4600 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4601 }
4602
4603 SDValue NewShift =
4604 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4605
4606 SDValue Vec;
4607 if (VT.isVector()) {
4608 unsigned NElts = TargetType.getVectorNumElements();
4609 SmallVector<SDValue, 8> LoOps;
4610 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4611
4612 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4613 for (unsigned I = 0; I != NElts; ++I)
4614 HiAndLoOps[2 * I] = LoOps[I];
4615 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4616 } else {
4617 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, Zero});
4618 }
4619 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4620}
4621
4622SDValue AMDGPUTargetLowering::performTruncateCombine(
4623 SDNode *N, DAGCombinerInfo &DCI) const {
4624 SDLoc SL(N);
4625 SelectionDAG &DAG = DCI.DAG;
4626 EVT VT = N->getValueType(ResNo: 0);
4627 SDValue Src = N->getOperand(Num: 0);
4628
4629 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
4630 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
4631 SDValue Vec = Src.getOperand(i: 0);
4632 if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
4633 SDValue Elt0 = Vec.getOperand(i: 0);
4634 EVT EltVT = Elt0.getValueType();
4635 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
4636 if (EltVT.isFloatingPoint()) {
4637 Elt0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL,
4638 VT: EltVT.changeTypeToInteger(), Operand: Elt0);
4639 }
4640
4641 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Elt0);
4642 }
4643 }
4644 }
4645
4646 // Equivalent of above for accessing the high element of a vector as an
4647 // integer operation.
4648 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
4649 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
4650 if (auto *K = isConstOrConstSplat(N: Src.getOperand(i: 1))) {
4651 SDValue BV = stripBitcast(Val: Src.getOperand(i: 0));
4652 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
4653 EVT SrcEltVT = BV.getOperand(i: 0).getValueType();
4654 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
4655 unsigned BitIndex = K->getZExtValue();
4656 unsigned PartIndex = BitIndex / SrcEltSize;
4657
4658 if (PartIndex * SrcEltSize == BitIndex &&
4659 PartIndex < BV.getNumOperands()) {
4660 if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
4661 SDValue SrcElt =
4662 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcEltVT.changeTypeToInteger(),
4663 Operand: BV.getOperand(i: PartIndex));
4664 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: SrcElt);
4665 }
4666 }
4667 }
4668 }
4669 }
4670
4671 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
4672 //
4673 // i16 (trunc (srl i64:x, K)), K <= 16 ->
4674 // i16 (trunc (srl (i32 (trunc x), K)))
4675 if (VT.getScalarSizeInBits() < 32) {
4676 EVT SrcVT = Src.getValueType();
4677 if (SrcVT.getScalarSizeInBits() > 32 &&
4678 (Src.getOpcode() == ISD::SRL ||
4679 Src.getOpcode() == ISD::SRA ||
4680 Src.getOpcode() == ISD::SHL)) {
4681 SDValue Amt = Src.getOperand(i: 1);
4682 KnownBits Known = DAG.computeKnownBits(Op: Amt);
4683
4684 // - For left shifts, do the transform as long as the shift
4685 // amount is still legal for i32, so when ShiftAmt < 32 (<= 31)
4686 // - For right shift, do it if ShiftAmt <= (32 - Size) to avoid
4687 // losing information stored in the high bits when truncating.
4688 const unsigned MaxCstSize =
4689 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
4690 if (Known.getMaxValue().ule(RHS: MaxCstSize)) {
4691 EVT MidVT = VT.isVector() ?
4692 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
4693 NumElements: VT.getVectorNumElements()) : MVT::i32;
4694
4695 EVT NewShiftVT = getShiftAmountTy(LHSTy: MidVT, DL: DAG.getDataLayout());
4696 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MidVT,
4697 Operand: Src.getOperand(i: 0));
4698 DCI.AddToWorklist(N: Trunc.getNode());
4699
4700 if (Amt.getValueType() != NewShiftVT) {
4701 Amt = DAG.getZExtOrTrunc(Op: Amt, DL: SL, VT: NewShiftVT);
4702 DCI.AddToWorklist(N: Amt.getNode());
4703 }
4704
4705 SDValue ShrunkShift = DAG.getNode(Opcode: Src.getOpcode(), DL: SL, VT: MidVT,
4706 N1: Trunc, N2: Amt);
4707 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: ShrunkShift);
4708 }
4709 }
4710 }
4711
4712 return SDValue();
4713}
4714
4715// We need to specifically handle i64 mul here to avoid unnecessary conversion
4716// instructions. If we only match on the legalized i64 mul expansion,
4717// SimplifyDemandedBits will be unable to remove them because there will be
4718// multiple uses due to the separate mul + mulh[su].
4719static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
4720 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
4721 if (Size <= 32) {
4722 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4723 return DAG.getNode(Opcode: MulOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4724 }
4725
4726 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4727 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
4728
4729 SDValue MulLo = DAG.getNode(Opcode: MulLoOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4730 SDValue MulHi = DAG.getNode(Opcode: MulHiOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4731
4732 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: MulLo, N2: MulHi);
4733}
4734
4735/// If \p V is an add of a constant 1, returns the other operand. Otherwise
4736/// return SDValue().
4737static SDValue getAddOneOp(const SDNode *V) {
4738 if (V->getOpcode() != ISD::ADD)
4739 return SDValue();
4740
4741 return isOneConstant(V: V->getOperand(Num: 1)) ? V->getOperand(Num: 0) : SDValue();
4742}
4743
4744SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
4745 DAGCombinerInfo &DCI) const {
4746 assert(N->getOpcode() == ISD::MUL);
4747 EVT VT = N->getValueType(ResNo: 0);
4748
4749 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4750 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4751 // unnecessarily). isDivergent() is used as an approximation of whether the
4752 // value is in an SGPR.
4753 if (!N->isDivergent())
4754 return SDValue();
4755
4756 unsigned Size = VT.getSizeInBits();
4757 if (VT.isVector() || Size > 64)
4758 return SDValue();
4759
4760 SelectionDAG &DAG = DCI.DAG;
4761 SDLoc DL(N);
4762
4763 SDValue N0 = N->getOperand(Num: 0);
4764 SDValue N1 = N->getOperand(Num: 1);
4765
4766 // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
4767 // matching.
4768
4769 // mul x, (add y, 1) -> add (mul x, y), x
4770 auto IsFoldableAdd = [](SDValue V) -> SDValue {
4771 SDValue AddOp = getAddOneOp(V: V.getNode());
4772 if (!AddOp)
4773 return SDValue();
4774
4775 if (V.hasOneUse() || all_of(Range: V->users(), P: [](const SDNode *U) -> bool {
4776 return U->getOpcode() == ISD::MUL;
4777 }))
4778 return AddOp;
4779
4780 return SDValue();
4781 };
4782
4783 // FIXME: The selection pattern is not properly checking for commuted
4784 // operands, so we have to place the mul in the LHS
4785 if (SDValue MulOper = IsFoldableAdd(N0)) {
4786 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1, N2: MulOper);
4787 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N1);
4788 }
4789
4790 if (SDValue MulOper = IsFoldableAdd(N1)) {
4791 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N0, N2: MulOper);
4792 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N0);
4793 }
4794
4795 // There are i16 integer mul/mad.
4796 if (isTypeLegal(VT: MVT::i16) && VT.getScalarType().bitsLE(VT: MVT::i16))
4797 return SDValue();
4798
4799 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4800 // in the source into any_extends if the result of the mul is truncated. Since
4801 // we can assume the high bits are whatever we want, use the underlying value
4802 // to avoid the unknown high bits from interfering.
4803 if (N0.getOpcode() == ISD::ANY_EXTEND)
4804 N0 = N0.getOperand(i: 0);
4805
4806 if (N1.getOpcode() == ISD::ANY_EXTEND)
4807 N1 = N1.getOperand(i: 0);
4808
4809 SDValue Mul;
4810
4811 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4812 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4813 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4814 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: false);
4815 } else if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4816 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4817 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4818 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: true);
4819 } else {
4820 return SDValue();
4821 }
4822
4823 // We need to use sext even for MUL_U24, because MUL_U24 is used
4824 // for signed multiply of 8 and 16-bit types.
4825 return DAG.getSExtOrTrunc(Op: Mul, DL, VT);
4826}
4827
4828SDValue
4829AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N,
4830 DAGCombinerInfo &DCI) const {
4831 if (N->getValueType(ResNo: 0) != MVT::i32)
4832 return SDValue();
4833
4834 SelectionDAG &DAG = DCI.DAG;
4835 SDLoc DL(N);
4836
4837 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
4838 SDValue N0 = N->getOperand(Num: 0);
4839 SDValue N1 = N->getOperand(Num: 1);
4840
4841 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4842 // in the source into any_extends if the result of the mul is truncated. Since
4843 // we can assume the high bits are whatever we want, use the underlying value
4844 // to avoid the unknown high bits from interfering.
4845 if (N0.getOpcode() == ISD::ANY_EXTEND)
4846 N0 = N0.getOperand(i: 0);
4847 if (N1.getOpcode() == ISD::ANY_EXTEND)
4848 N1 = N1.getOperand(i: 0);
4849
4850 // Try to use two fast 24-bit multiplies (one for each half of the result)
4851 // instead of one slow extending multiply.
4852 unsigned LoOpcode = 0;
4853 unsigned HiOpcode = 0;
4854 if (Signed) {
4855 if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4856 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4857 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4858 LoOpcode = AMDGPUISD::MUL_I24;
4859 HiOpcode = AMDGPUISD::MULHI_I24;
4860 }
4861 } else {
4862 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4863 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4864 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4865 LoOpcode = AMDGPUISD::MUL_U24;
4866 HiOpcode = AMDGPUISD::MULHI_U24;
4867 }
4868 }
4869 if (!LoOpcode)
4870 return SDValue();
4871
4872 SDValue Lo = DAG.getNode(Opcode: LoOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4873 SDValue Hi = DAG.getNode(Opcode: HiOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4874 DCI.CombineTo(N, Res0: Lo, Res1: Hi);
4875 return SDValue(N, 0);
4876}
4877
4878SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
4879 DAGCombinerInfo &DCI) const {
4880 EVT VT = N->getValueType(ResNo: 0);
4881
4882 if (!Subtarget->hasMulI24() || VT.isVector())
4883 return SDValue();
4884
4885 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4886 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4887 // unnecessarily). isDivergent() is used as an approximation of whether the
4888 // value is in an SGPR.
4889 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4890 // valu op anyway)
4891 if (Subtarget->hasSMulHi() && !N->isDivergent())
4892 return SDValue();
4893
4894 SelectionDAG &DAG = DCI.DAG;
4895 SDLoc DL(N);
4896
4897 SDValue N0 = N->getOperand(Num: 0);
4898 SDValue N1 = N->getOperand(Num: 1);
4899
4900 if (!isI24(Op: N0, DAG) || !isI24(Op: N1, DAG))
4901 return SDValue();
4902
4903 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4904 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4905
4906 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_I24, DL, VT: MVT::i32, N1: N0, N2: N1);
4907 DCI.AddToWorklist(N: Mulhi.getNode());
4908 return DAG.getSExtOrTrunc(Op: Mulhi, DL, VT);
4909}
4910
4911SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
4912 DAGCombinerInfo &DCI) const {
4913 EVT VT = N->getValueType(ResNo: 0);
4914
4915 if (VT.isVector() || VT.getSizeInBits() > 32 || !Subtarget->hasMulU24())
4916 return SDValue();
4917
4918 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4919 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4920 // unnecessarily). isDivergent() is used as an approximation of whether the
4921 // value is in an SGPR.
4922 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4923 // valu op anyway)
4924 if (!N->isDivergent() && Subtarget->hasSMulHi())
4925 return SDValue();
4926
4927 SelectionDAG &DAG = DCI.DAG;
4928 SDLoc DL(N);
4929
4930 SDValue N0 = N->getOperand(Num: 0);
4931 SDValue N1 = N->getOperand(Num: 1);
4932
4933 if (!isU24(Op: N0, DAG) || !isU24(Op: N1, DAG))
4934 return SDValue();
4935
4936 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4937 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4938
4939 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_U24, DL, VT: MVT::i32, N1: N0, N2: N1);
4940 DCI.AddToWorklist(N: Mulhi.getNode());
4941 return DAG.getZExtOrTrunc(Op: Mulhi, DL, VT);
4942}
4943
4944SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
4945 SDValue Op,
4946 const SDLoc &DL,
4947 unsigned Opc) const {
4948 EVT VT = Op.getValueType();
4949 if (VT.bitsGT(VT: MVT::i32))
4950 return SDValue();
4951
4952 if (VT != MVT::i32)
4953 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: Op);
4954
4955 SDValue FFBX = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, Operand: Op);
4956 if (VT != MVT::i32)
4957 FFBX = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FFBX);
4958
4959 return FFBX;
4960}
4961
4962// The native instructions return -1 on 0 input. Optimize out a select that
4963// produces -1 on 0.
4964//
4965// TODO: If zero is not undef, we could also do this if the output is compared
4966// against the bitwidth.
4967//
4968// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
4969SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
4970 SDValue LHS, SDValue RHS,
4971 DAGCombinerInfo &DCI) const {
4972 if (!isNullConstant(V: Cond.getOperand(i: 1)))
4973 return SDValue();
4974
4975 SelectionDAG &DAG = DCI.DAG;
4976 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
4977 SDValue CmpLHS = Cond.getOperand(i: 0);
4978
4979 // select (setcc x, 0, eq), -1, (ctlz_zero_poison x) -> ffbh_u32 x
4980 // select (setcc x, 0, eq), -1, (cttz_zero_poison x) -> ffbl_u32 x
4981 if (CCOpcode == ISD::SETEQ &&
4982 (isCtlzOpc(Opc: RHS.getOpcode()) || isCttzOpc(Opc: RHS.getOpcode())) &&
4983 RHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: LHS)) {
4984 unsigned Opc =
4985 isCttzOpc(Opc: RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4986 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4987 }
4988
4989 // select (setcc x, 0, ne), (ctlz_zero_poison x), -1 -> ffbh_u32 x
4990 // select (setcc x, 0, ne), (cttz_zero_poison x), -1 -> ffbl_u32 x
4991 if (CCOpcode == ISD::SETNE &&
4992 (isCtlzOpc(Opc: LHS.getOpcode()) || isCttzOpc(Opc: LHS.getOpcode())) &&
4993 LHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: RHS)) {
4994 unsigned Opc =
4995 isCttzOpc(Opc: LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4996
4997 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4998 }
4999
5000 return SDValue();
5001}
5002
5003static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
5004 unsigned Op,
5005 const SDLoc &SL,
5006 SDValue Cond,
5007 SDValue N1,
5008 SDValue N2) {
5009 SelectionDAG &DAG = DCI.DAG;
5010 EVT VT = N1.getValueType();
5011
5012 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cond,
5013 N2: N1.getOperand(i: 0), N3: N2.getOperand(i: 0));
5014 DCI.AddToWorklist(N: NewSelect.getNode());
5015 return DAG.getNode(Opcode: Op, DL: SL, VT, Operand: NewSelect);
5016}
5017
5018// Pull a free FP operation out of a select so it may fold into uses.
5019//
5020// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
5021// select c, (fneg x), k -> fneg (select c, x, (fneg k))
5022//
5023// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
5024// select c, (fabs x), +k -> fabs (select c, x, k)
5025SDValue
5026AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
5027 SDValue N) const {
5028 SelectionDAG &DAG = DCI.DAG;
5029 SDValue Cond = N.getOperand(i: 0);
5030 SDValue LHS = N.getOperand(i: 1);
5031 SDValue RHS = N.getOperand(i: 2);
5032
5033 EVT VT = N.getValueType();
5034 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
5035 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
5036 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5037 return SDValue();
5038
5039 return distributeOpThroughSelect(DCI, Op: LHS.getOpcode(),
5040 SL: SDLoc(N), Cond, N1: LHS, N2: RHS);
5041 }
5042
5043 bool Inv = false;
5044 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
5045 std::swap(a&: LHS, b&: RHS);
5046 Inv = true;
5047 }
5048
5049 // TODO: Support vector constants.
5050 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
5051 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS &&
5052 !selectSupportsSourceMods(N: N.getNode())) {
5053 SDLoc SL(N);
5054 // If one side is an fneg/fabs and the other is a constant, we can push the
5055 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
5056 SDValue NewLHS = LHS.getOperand(i: 0);
5057 SDValue NewRHS = RHS;
5058
5059 // Careful: if the neg can be folded up, don't try to pull it back down.
5060 bool ShouldFoldNeg = true;
5061
5062 if (NewLHS.hasOneUse()) {
5063 unsigned Opc = NewLHS.getOpcode();
5064 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(N: NewLHS.getNode()))
5065 ShouldFoldNeg = false;
5066 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
5067 ShouldFoldNeg = false;
5068 }
5069
5070 if (ShouldFoldNeg) {
5071 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative())
5072 return SDValue();
5073
5074 // We're going to be forced to use a source modifier anyway, there's no
5075 // point to pulling the negate out unless we can get a size reduction by
5076 // negating the constant.
5077 //
5078 // TODO: Generalize to use getCheaperNegatedExpression which doesn't know
5079 // about cheaper constants.
5080 if (NewLHS.getOpcode() == ISD::FABS &&
5081 getConstantNegateCost(C: CRHS) != NegatibleCost::Cheaper)
5082 return SDValue();
5083
5084 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5085 return SDValue();
5086
5087 if (LHS.getOpcode() == ISD::FNEG)
5088 NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5089
5090 if (Inv)
5091 std::swap(a&: NewLHS, b&: NewRHS);
5092
5093 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT,
5094 N1: Cond, N2: NewLHS, N3: NewRHS);
5095 DCI.AddToWorklist(N: NewSelect.getNode());
5096 return DAG.getNode(Opcode: LHS.getOpcode(), DL: SL, VT, Operand: NewSelect);
5097 }
5098 }
5099
5100 return SDValue();
5101}
5102
5103SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
5104 DAGCombinerInfo &DCI) const {
5105 if (SDValue Folded = foldFreeOpFromSelect(DCI, N: SDValue(N, 0)))
5106 return Folded;
5107
5108 SDValue Cond = N->getOperand(Num: 0);
5109 if (Cond.getOpcode() != ISD::SETCC)
5110 return SDValue();
5111
5112 EVT VT = N->getValueType(ResNo: 0);
5113 SDValue LHS = Cond.getOperand(i: 0);
5114 SDValue RHS = Cond.getOperand(i: 1);
5115 SDValue CC = Cond.getOperand(i: 2);
5116
5117 SDValue True = N->getOperand(Num: 1);
5118 SDValue False = N->getOperand(Num: 2);
5119
5120 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
5121 SelectionDAG &DAG = DCI.DAG;
5122 if (DAG.isConstantValueOfAnyType(N: True) &&
5123 !DAG.isConstantValueOfAnyType(N: False)) {
5124 // Swap cmp + select pair to move constant to false input.
5125 // This will allow using VOPC cndmasks more often.
5126 // select (setcc x, y), k, x -> select (setccinv x, y), x, k
5127
5128 SDLoc SL(N);
5129 ISD::CondCode NewCC =
5130 getSetCCInverse(Operation: cast<CondCodeSDNode>(Val&: CC)->get(), Type: LHS.getValueType());
5131
5132 SDValue NewCond = DAG.getSetCC(DL: SL, VT: Cond.getValueType(), LHS, RHS, Cond: NewCC);
5133 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NewCond, N2: False, N3: True);
5134 }
5135
5136 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
5137 SDValue MinMax = combineFMinMaxLegacy(DL: SDLoc(N), VT, LHS, RHS, True, False,
5138 CC, Flags: N->getFlags(), DCI);
5139 // Revisit this node so we can catch min3/max3/med3 patterns.
5140 //DCI.AddToWorklist(MinMax.getNode());
5141 return MinMax;
5142 }
5143 }
5144
5145 // There's no reason to not do this if the condition has other uses.
5146 return performCtlz_CttzCombine(SL: SDLoc(N), Cond, LHS: True, RHS: False, DCI);
5147}
5148
5149static bool isInv2Pi(const APFloat &APF) {
5150 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
5151 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
5152 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
5153
5154 return APF.bitwiseIsEqual(RHS: KF16) ||
5155 APF.bitwiseIsEqual(RHS: KF32) ||
5156 APF.bitwiseIsEqual(RHS: KF64);
5157}
5158
5159// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
5160// additional cost to negate them.
5161TargetLowering::NegatibleCost
5162AMDGPUTargetLowering::getConstantNegateCost(const ConstantFPSDNode *C) const {
5163 if (C->isZero())
5164 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5165
5166 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(APF: C->getValueAPF()))
5167 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5168
5169 return NegatibleCost::Neutral;
5170}
5171
5172bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
5173 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5174 return getConstantNegateCost(C) == NegatibleCost::Expensive;
5175 return false;
5176}
5177
5178static unsigned inverseMinMax(unsigned Opc) {
5179 switch (Opc) {
5180 case ISD::FMAXNUM:
5181 return ISD::FMINNUM;
5182 case ISD::FMINNUM:
5183 return ISD::FMAXNUM;
5184 case ISD::FMAXNUM_IEEE:
5185 return ISD::FMINNUM_IEEE;
5186 case ISD::FMINNUM_IEEE:
5187 return ISD::FMAXNUM_IEEE;
5188 case ISD::FMAXIMUM:
5189 return ISD::FMINIMUM;
5190 case ISD::FMINIMUM:
5191 return ISD::FMAXIMUM;
5192 case ISD::FMAXIMUMNUM:
5193 return ISD::FMINIMUMNUM;
5194 case ISD::FMINIMUMNUM:
5195 return ISD::FMAXIMUMNUM;
5196 case AMDGPUISD::FMAX_LEGACY:
5197 return AMDGPUISD::FMIN_LEGACY;
5198 case AMDGPUISD::FMIN_LEGACY:
5199 return AMDGPUISD::FMAX_LEGACY;
5200 default:
5201 llvm_unreachable("invalid min/max opcode");
5202 }
5203}
5204
5205/// \return true if it's profitable to try to push an fneg into its source
5206/// instruction.
5207bool AMDGPUTargetLowering::shouldFoldFNegIntoSrc(SDNode *N, SDValue N0) {
5208 // If the input has multiple uses and we can either fold the negate down, or
5209 // the other uses cannot, give up. This both prevents unprofitable
5210 // transformations and infinite loops: we won't repeatedly try to fold around
5211 // a negate that has no 'good' form.
5212 if (N0.hasOneUse()) {
5213 // This may be able to fold into the source, but at a code size cost. Don't
5214 // fold if the fold into the user is free.
5215 if (allUsesHaveSourceMods(N, CostThreshold: 0))
5216 return false;
5217 } else {
5218 if (fnegFoldsIntoOp(N: N0.getNode()) &&
5219 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N: N0.getNode())))
5220 return false;
5221 }
5222
5223 return true;
5224}
5225
5226SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
5227 DAGCombinerInfo &DCI) const {
5228 SelectionDAG &DAG = DCI.DAG;
5229 SDValue N0 = N->getOperand(Num: 0);
5230 EVT VT = N->getValueType(ResNo: 0);
5231
5232 unsigned Opc = N0.getOpcode();
5233
5234 if (!shouldFoldFNegIntoSrc(N, N0))
5235 return SDValue();
5236
5237 bool MayIgnoreSignedZeroForAllUses =
5238 N0->getFlags().hasNoSignedZeros() ||
5239 (N0.hasOneUse() && N->getFlags().hasNoSignedZeros());
5240
5241 SDLoc SL(N);
5242 switch (Opc) {
5243 case ISD::FADD: {
5244 if (!MayIgnoreSignedZeroForAllUses)
5245 return SDValue();
5246
5247 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
5248 SDValue LHS = N0.getOperand(i: 0);
5249 SDValue RHS = N0.getOperand(i: 1);
5250
5251 if (LHS.getOpcode() != ISD::FNEG)
5252 LHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5253 else
5254 LHS = LHS.getOperand(i: 0);
5255
5256 if (RHS.getOpcode() != ISD::FNEG)
5257 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5258 else
5259 RHS = RHS.getOperand(i: 0);
5260
5261 SDValue Res = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5262 if (Res.getOpcode() != ISD::FADD)
5263 return SDValue(); // Op got folded away.
5264 if (!N0.hasOneUse())
5265 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5266 return Res;
5267 }
5268 case ISD::FMUL:
5269 case AMDGPUISD::FMUL_LEGACY: {
5270 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
5271 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
5272 SDValue LHS = N0.getOperand(i: 0);
5273 SDValue RHS = N0.getOperand(i: 1);
5274
5275 if (LHS.getOpcode() == ISD::FNEG)
5276 LHS = LHS.getOperand(i: 0);
5277 else if (RHS.getOpcode() == ISD::FNEG)
5278 RHS = RHS.getOperand(i: 0);
5279 else
5280 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5281
5282 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5283 if (Res.getOpcode() != Opc)
5284 return SDValue(); // Op got folded away.
5285 if (!N0.hasOneUse())
5286 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5287 return Res;
5288 }
5289 case ISD::FMA:
5290 case ISD::FMAD: {
5291 // TODO: handle llvm.amdgcn.fma.legacy
5292 if (!MayIgnoreSignedZeroForAllUses)
5293 return SDValue();
5294
5295 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
5296 SDValue LHS = N0.getOperand(i: 0);
5297 SDValue MHS = N0.getOperand(i: 1);
5298 SDValue RHS = N0.getOperand(i: 2);
5299
5300 if (LHS.getOpcode() == ISD::FNEG)
5301 LHS = LHS.getOperand(i: 0);
5302 else if (MHS.getOpcode() == ISD::FNEG)
5303 MHS = MHS.getOperand(i: 0);
5304 else
5305 MHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: MHS);
5306
5307 if (RHS.getOpcode() != ISD::FNEG)
5308 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5309 else
5310 RHS = RHS.getOperand(i: 0);
5311
5312 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: MHS, N3: RHS);
5313 if (Res.getOpcode() != Opc)
5314 return SDValue(); // Op got folded away.
5315 if (!N0.hasOneUse())
5316 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5317 return Res;
5318 }
5319 case ISD::FMAXNUM:
5320 case ISD::FMINNUM:
5321 case ISD::FMAXNUM_IEEE:
5322 case ISD::FMINNUM_IEEE:
5323 case ISD::FMINIMUM:
5324 case ISD::FMAXIMUM:
5325 case ISD::FMINIMUMNUM:
5326 case ISD::FMAXIMUMNUM:
5327 case AMDGPUISD::FMAX_LEGACY:
5328 case AMDGPUISD::FMIN_LEGACY: {
5329 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
5330 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
5331 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
5332 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
5333
5334 SDValue LHS = N0.getOperand(i: 0);
5335 SDValue RHS = N0.getOperand(i: 1);
5336
5337 // 0 doesn't have a negated inline immediate.
5338 // TODO: This constant check should be generalized to other operations.
5339 if (isConstantCostlierToNegate(N: RHS))
5340 return SDValue();
5341
5342 // Swapping min<->max flips which operand a signed zero tie selects.
5343 if ((Opc == AMDGPUISD::FMIN_LEGACY || Opc == AMDGPUISD::FMAX_LEGACY) &&
5344 !canIgnoreLegacyMinMaxTies(DAG, Flags: N0->getFlags(), LHS, RHS))
5345 return SDValue();
5346
5347 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5348 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5349 unsigned Opposite = inverseMinMax(Opc);
5350
5351 SDValue Res = DAG.getNode(Opcode: Opposite, DL: SL, VT, N1: NegLHS, N2: NegRHS, Flags: N0->getFlags());
5352 if (Res.getOpcode() != Opposite)
5353 return SDValue(); // Op got folded away.
5354 if (!N0.hasOneUse())
5355 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5356 return Res;
5357 }
5358 case AMDGPUISD::FMED3: {
5359 // med3 sorts a NaN input as smaller than everything regardless of its sign,
5360 // so negating all operands does not sign-flip the median when an input may
5361 // be NaN.
5362 if (!N0->getFlags().hasNoNaNs())
5363 return SDValue();
5364
5365 SDValue Ops[3];
5366 for (unsigned I = 0; I < 3; ++I)
5367 Ops[I] = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N0->getOperand(Num: I), Flags: N0->getFlags());
5368
5369 SDValue Res = DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT, Ops, Flags: N0->getFlags());
5370 if (Res.getOpcode() != AMDGPUISD::FMED3)
5371 return SDValue(); // Op got folded away.
5372
5373 if (!N0.hasOneUse()) {
5374 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res);
5375 DAG.ReplaceAllUsesWith(From: N0, To: Neg);
5376
5377 for (SDNode *U : Neg->users())
5378 DCI.AddToWorklist(N: U);
5379 }
5380
5381 return Res;
5382 }
5383 case ISD::FP_EXTEND:
5384 case ISD::FTRUNC:
5385 case ISD::FRINT:
5386 case ISD::FNEARBYINT: // XXX - Should fround be handled?
5387 case ISD::FROUNDEVEN:
5388 case ISD::FSIN:
5389 case ISD::FCANONICALIZE:
5390 case AMDGPUISD::RCP:
5391 case AMDGPUISD::RCP_LEGACY:
5392 case AMDGPUISD::RCP_IFLAG:
5393 case AMDGPUISD::SIN_HW: {
5394 SDValue CvtSrc = N0.getOperand(i: 0);
5395 if (CvtSrc.getOpcode() == ISD::FNEG) {
5396 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
5397 // (fneg (rcp (fneg x))) -> (rcp x)
5398 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: CvtSrc.getOperand(i: 0));
5399 }
5400
5401 if (!N0.hasOneUse())
5402 return SDValue();
5403
5404 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
5405 // (fneg (rcp x)) -> (rcp (fneg x))
5406 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5407 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: Neg, Flags: N0->getFlags());
5408 }
5409 case ISD::FP_ROUND: {
5410 SDValue CvtSrc = N0.getOperand(i: 0);
5411
5412 if (CvtSrc.getOpcode() == ISD::FNEG) {
5413 // (fneg (fp_round (fneg x))) -> (fp_round x)
5414 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT,
5415 N1: CvtSrc.getOperand(i: 0), N2: N0.getOperand(i: 1));
5416 }
5417
5418 if (!N0.hasOneUse())
5419 return SDValue();
5420
5421 // (fneg (fp_round x)) -> (fp_round (fneg x))
5422 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5423 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Neg, N2: N0.getOperand(i: 1));
5424 }
5425 case ISD::FP16_TO_FP: {
5426 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
5427 // f16, but legalization of f16 fneg ends up pulling it out of the source.
5428 // Put the fneg back as a legal source operation that can be matched later.
5429 SDLoc SL(N);
5430
5431 SDValue Src = N0.getOperand(i: 0);
5432 EVT SrcVT = Src.getValueType();
5433
5434 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
5435 SDValue IntFNeg = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: SrcVT, N1: Src,
5436 N2: DAG.getConstant(Val: 0x8000, DL: SL, VT: SrcVT));
5437 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFNeg);
5438 }
5439 case ISD::SELECT: {
5440 // fneg (select c, a, b) -> select c, (fneg a), (fneg b)
5441 // TODO: Invert conditions of foldFreeOpFromSelect
5442 return SDValue();
5443 }
5444 case ISD::BITCAST: {
5445 SDLoc SL(N);
5446 SDValue BCSrc = N0.getOperand(i: 0);
5447 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
5448 SDValue HighBits = BCSrc.getOperand(i: BCSrc.getNumOperands() - 1);
5449 if (VT != MVT::f64 || HighBits.getValueType().getSizeInBits() != 32 ||
5450 !fnegFoldsIntoOp(N: HighBits.getNode()))
5451 return SDValue();
5452
5453 // f64 fneg only really needs to operate on the high half of of the
5454 // register, so try to force it to an f32 operation to help make use of
5455 // source modifiers.
5456 //
5457 //
5458 // fneg (f64 (bitcast (build_vector x, y))) ->
5459 // f64 (bitcast (build_vector (bitcast i32:x to f32),
5460 // (fneg (bitcast i32:y to f32)))
5461
5462 SDValue CastHi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: HighBits);
5463 SDValue NegHi = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: CastHi);
5464 SDValue CastBack =
5465 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: HighBits.getValueType(), Operand: NegHi);
5466
5467 SmallVector<SDValue, 8> Ops(BCSrc->ops());
5468 Ops.back() = CastBack;
5469 DCI.AddToWorklist(N: NegHi.getNode());
5470 SDValue Build =
5471 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: BCSrc.getValueType(), Ops);
5472 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Build);
5473
5474 if (!N0.hasOneUse())
5475 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Result));
5476 return Result;
5477 }
5478
5479 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 &&
5480 BCSrc.hasOneUse()) {
5481 // fneg (bitcast (f32 (select cond, i32:lhs, i32:rhs))) ->
5482 // select cond, (bitcast i32:lhs to f32), (bitcast i32:rhs to f32)
5483
5484 // TODO: Cast back result for multiple uses is beneficial in some cases.
5485
5486 SDValue LHS =
5487 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 1));
5488 SDValue RHS =
5489 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 2));
5490
5491 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: LHS);
5492 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHS);
5493
5494 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: BCSrc.getOperand(i: 0), N2: NegLHS,
5495 N3: NegRHS);
5496 }
5497
5498 return SDValue();
5499 }
5500 default:
5501 return SDValue();
5502 }
5503}
5504
5505SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
5506 DAGCombinerInfo &DCI) const {
5507 SelectionDAG &DAG = DCI.DAG;
5508 SDValue N0 = N->getOperand(Num: 0);
5509
5510 if (!N0.hasOneUse())
5511 return SDValue();
5512
5513 switch (N0.getOpcode()) {
5514 case ISD::FP16_TO_FP: {
5515 assert(!isTypeLegal(MVT::f16) && "should only see if f16 is illegal");
5516 SDLoc SL(N);
5517 SDValue Src = N0.getOperand(i: 0);
5518 EVT SrcVT = Src.getValueType();
5519
5520 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
5521 SDValue IntFAbs = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SrcVT, N1: Src,
5522 N2: DAG.getConstant(Val: 0x7fff, DL: SL, VT: SrcVT));
5523 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFAbs);
5524 }
5525 case ISD::FP_ROUND: {
5526 SDLoc SL(N);
5527 SDValue CvtSrc = N0.getOperand(i: 0);
5528
5529 // fabs (fp_round x) -> fp_round (fabs x)
5530 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc,
5531 Flags: N->getFlags());
5532 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: N->getValueType(ResNo: 0), N1: Abs,
5533 N2: N0.getOperand(i: 1), Flags: N0->getFlags());
5534 }
5535 default:
5536 return SDValue();
5537 }
5538}
5539
5540SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
5541 DAGCombinerInfo &DCI) const {
5542 const auto *CFP = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
5543 if (!CFP)
5544 return SDValue();
5545
5546 std::optional<APFloat> Result = AMDGPU::evaluateRcp(Val: CFP->getValueAPF());
5547 if (!Result)
5548 return SDValue();
5549
5550 return DCI.DAG.getConstantFP(Val: *Result, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
5551}
5552
5553bool AMDGPUTargetLowering::isInt64ImmLegal(SDNode *N, SelectionDAG &DAG) const {
5554 if (!Subtarget->isGCN())
5555 return false;
5556
5557 ConstantSDNode *SDConstant = dyn_cast<ConstantSDNode>(Val: N);
5558 ConstantFPSDNode *SDFPConstant = dyn_cast<ConstantFPSDNode>(Val: N);
5559 auto &ST = DAG.getSubtarget<GCNSubtarget>();
5560 const auto *TII = ST.getInstrInfo();
5561
5562 if (!ST.hasVMovB64Inst() || (!SDConstant && !SDFPConstant))
5563 return false;
5564
5565 if (ST.has64BitLiterals())
5566 return true;
5567
5568 if (SDConstant) {
5569 const APInt &APVal = SDConstant->getAPIntValue();
5570 return isUInt<32>(x: APVal.getZExtValue()) || TII->isInlineConstant(Imm: APVal);
5571 }
5572
5573 APInt Val = SDFPConstant->getValueAPF().bitcastToAPInt();
5574 return isUInt<32>(x: Val.getZExtValue()) || TII->isInlineConstant(Imm: Val);
5575}
5576
5577SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
5578 DAGCombinerInfo &DCI) const {
5579 SelectionDAG &DAG = DCI.DAG;
5580 SDLoc DL(N);
5581
5582 switch(N->getOpcode()) {
5583 default:
5584 break;
5585 case ISD::BITCAST: {
5586 EVT DestVT = N->getValueType(ResNo: 0);
5587
5588 // Push casts through vector builds. This helps avoid emitting a large
5589 // number of copies when materializing floating point vector constants.
5590 //
5591 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
5592 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
5593 if (DestVT.isVector()) {
5594 SDValue Src = N->getOperand(Num: 0);
5595 if (Src.getOpcode() == ISD::BUILD_VECTOR &&
5596 (DCI.getDAGCombineLevel() < AfterLegalizeDAG ||
5597 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: DestVT))) {
5598 EVT SrcVT = Src.getValueType();
5599 unsigned NElts = DestVT.getVectorNumElements();
5600
5601 if (SrcVT.getVectorNumElements() == NElts) {
5602 EVT DestEltVT = DestVT.getVectorElementType();
5603
5604 SmallVector<SDValue, 8> CastedElts;
5605 SDLoc SL(N);
5606 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
5607 SDValue Elt = Src.getOperand(i: I);
5608 CastedElts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: DestEltVT, Operand: Elt));
5609 }
5610
5611 return DAG.getBuildVector(VT: DestVT, DL: SL, Ops: CastedElts);
5612 }
5613 }
5614 }
5615
5616 if (DestVT.getSizeInBits() != 64 || !DestVT.isVector())
5617 break;
5618
5619 // Fold bitcasts of constants.
5620 //
5621 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
5622 // TODO: Generalize and move to DAGCombiner
5623 SDValue Src = N->getOperand(Num: 0);
5624 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Src)) {
5625 SDLoc SL(N);
5626 if (isInt64ImmLegal(N: C, DAG))
5627 break;
5628 uint64_t CVal = C->getZExtValue();
5629 SDValue BV = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5630 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5631 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5632 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: BV);
5633 }
5634
5635 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
5636 const APInt &Val = C->getValueAPF().bitcastToAPInt();
5637 SDLoc SL(N);
5638 if (isInt64ImmLegal(N: C, DAG))
5639 break;
5640 uint64_t CVal = Val.getZExtValue();
5641 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5642 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5643 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5644
5645 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: Vec);
5646 }
5647
5648 break;
5649 }
5650 case ISD::SHL:
5651 case ISD::SRA:
5652 case ISD::SRL: {
5653 // Range metadata can be invalidated when loads are converted to legal types
5654 // (e.g. v2i64 -> v4i32).
5655 // Try to convert vector shl/sra/srl before type legalization so that range
5656 // metadata can be utilized.
5657 if (!(N->getValueType(ResNo: 0).isVector() &&
5658 DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
5659 DCI.getDAGCombineLevel() < AfterLegalizeDAG)
5660 break;
5661 if (N->getOpcode() == ISD::SHL)
5662 return performShlCombine(N, DCI);
5663 if (N->getOpcode() == ISD::SRA)
5664 return performSraCombine(N, DCI);
5665 return performSrlCombine(N, DCI);
5666 }
5667 case ISD::TRUNCATE:
5668 return performTruncateCombine(N, DCI);
5669 case ISD::MUL:
5670 return performMulCombine(N, DCI);
5671 case AMDGPUISD::MUL_U24:
5672 case AMDGPUISD::MUL_I24: {
5673 if (SDValue Simplified = simplifyMul24(Node24: N, DCI))
5674 return Simplified;
5675 break;
5676 }
5677 case AMDGPUISD::MULHI_I24:
5678 case AMDGPUISD::MULHI_U24:
5679 return simplifyMul24(Node24: N, DCI);
5680 case ISD::SMUL_LOHI:
5681 case ISD::UMUL_LOHI:
5682 return performMulLoHiCombine(N, DCI);
5683 case ISD::MULHS:
5684 return performMulhsCombine(N, DCI);
5685 case ISD::MULHU:
5686 return performMulhuCombine(N, DCI);
5687 case ISD::SELECT:
5688 return performSelectCombine(N, DCI);
5689 case ISD::FNEG:
5690 return performFNegCombine(N, DCI);
5691 case ISD::FABS:
5692 return performFAbsCombine(N, DCI);
5693 case AMDGPUISD::BFE_I32:
5694 case AMDGPUISD::BFE_U32: {
5695 assert(N->getValueType(0) == MVT::i32 &&
5696 "BFE_I32/BFE_U32 is a 32-bit operation");
5697 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
5698 if (!Width)
5699 break;
5700
5701 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
5702 if (WidthVal == 0)
5703 return DAG.getConstant(Val: 0, DL, VT: MVT::i32);
5704
5705 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
5706 if (!Offset)
5707 break;
5708
5709 SDValue BitsFrom = N->getOperand(Num: 0);
5710 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
5711
5712 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
5713
5714 if (OffsetVal == 0) {
5715 // This is already sign / zero extended, so try to fold away extra BFEs.
5716 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WidthVal);
5717 if (Signed) {
5718 if (DAG.ComputeNumSignBits(Op: BitsFrom) >= 32 - WidthVal + 1)
5719 return BitsFrom;
5720
5721 // This is a sign_extend_inreg. Replace it to take advantage of existing
5722 // DAG Combines. If not eliminated, we will match back to BFE during
5723 // selection.
5724
5725 // TODO: The sext_inreg of extended types ends, although we can could
5726 // handle them in a single BFE.
5727 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32, N1: BitsFrom,
5728 N2: DAG.getValueType(SmallVT));
5729 }
5730
5731 if (DAG.MaskedValueIsZero(Op: BitsFrom,
5732 Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - WidthVal)))
5733 return BitsFrom;
5734
5735 return DAG.getZeroExtendInReg(Op: BitsFrom, DL, VT: SmallVT);
5736 }
5737
5738 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(Val&: BitsFrom)) {
5739 if (Signed) {
5740 return constantFoldBFE<int32_t>(DAG,
5741 Src0: CVal->getSExtValue(),
5742 Offset: OffsetVal,
5743 Width: WidthVal,
5744 DL);
5745 }
5746
5747 return constantFoldBFE<uint32_t>(DAG,
5748 Src0: CVal->getZExtValue(),
5749 Offset: OffsetVal,
5750 Width: WidthVal,
5751 DL);
5752 }
5753
5754 if ((OffsetVal + WidthVal) >= 32 &&
5755 !(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
5756 SDValue ShiftVal = DAG.getConstant(Val: OffsetVal, DL, VT: MVT::i32);
5757 return DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32,
5758 N1: BitsFrom, N2: ShiftVal);
5759 }
5760
5761 if (BitsFrom.hasOneUse()) {
5762 APInt Demanded = APInt::getBitsSet(numBits: 32,
5763 loBit: OffsetVal,
5764 hiBit: OffsetVal + WidthVal);
5765
5766 KnownBits Known;
5767 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
5768 !DCI.isBeforeLegalizeOps());
5769 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5770 if (TLI.ShrinkDemandedConstant(Op: BitsFrom, DemandedBits: Demanded, TLO) ||
5771 TLI.SimplifyDemandedBits(Op: BitsFrom, DemandedBits: Demanded, Known, TLO)) {
5772 DCI.CommitTargetLoweringOpt(TLO);
5773 }
5774 }
5775
5776 break;
5777 }
5778 case ISD::LOAD:
5779 return performLoadCombine(N, DCI);
5780 case ISD::STORE:
5781 return performStoreCombine(N, DCI);
5782 case AMDGPUISD::RCP:
5783 case AMDGPUISD::RCP_IFLAG:
5784 return performRcpCombine(N, DCI);
5785 case ISD::AssertZext:
5786 case ISD::AssertSext:
5787 return performAssertSZExtCombine(N, DCI);
5788 case ISD::INTRINSIC_WO_CHAIN:
5789 return performIntrinsicWOChainCombine(N, DCI);
5790 case AMDGPUISD::FMAD_FTZ: {
5791 SDValue N0 = N->getOperand(Num: 0);
5792 SDValue N1 = N->getOperand(Num: 1);
5793 SDValue N2 = N->getOperand(Num: 2);
5794 EVT VT = N->getValueType(ResNo: 0);
5795
5796 // FMAD_FTZ is a FMAD + flush denormals to zero.
5797 // We flush the inputs, the intermediate step, and the output.
5798 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Val&: N0);
5799 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
5800 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
5801 if (N0CFP && N1CFP && N2CFP) {
5802 const auto FTZ = [](const APFloat &V) {
5803 if (V.isDenormal()) {
5804 APFloat Zero(V.getSemantics(), 0);
5805 return V.isNegative() ? -Zero : Zero;
5806 }
5807 return V;
5808 };
5809
5810 APFloat V0 = FTZ(N0CFP->getValueAPF());
5811 APFloat V1 = FTZ(N1CFP->getValueAPF());
5812 APFloat V2 = FTZ(N2CFP->getValueAPF());
5813 V0.multiply(RHS: V1, RM: APFloat::rmNearestTiesToEven);
5814 V0 = FTZ(V0);
5815 V0.add(RHS: V2, RM: APFloat::rmNearestTiesToEven);
5816 return DAG.getConstantFP(Val: FTZ(V0), DL, VT);
5817 }
5818 break;
5819 }
5820 }
5821 return SDValue();
5822}
5823
5824bool AMDGPUTargetLowering::SimplifyDemandedBitsForTargetNode(
5825 SDValue Op, const APInt &OriginalDemandedBits,
5826 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
5827 unsigned Depth) const {
5828 switch (Op.getOpcode()) {
5829 case ISD::INTRINSIC_WO_CHAIN: {
5830 switch (Op.getConstantOperandVal(i: 0)) {
5831 case Intrinsic::amdgcn_readfirstlane:
5832 case Intrinsic::amdgcn_readlane:
5833 case Intrinsic::amdgcn_wwm: {
5834 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5835 DemandedElts: OriginalDemandedElts, Known, TLO, Depth: Depth + 1))
5836 return true;
5837 break;
5838 }
5839 case Intrinsic::amdgcn_set_inactive:
5840 case Intrinsic::amdgcn_set_inactive_chain_arg: {
5841 // The result is operand 1 in active lanes and operand 2 in inactive
5842 // lanes, so the known bits are the intersection of both operands.
5843 KnownBits KnownValue, KnownInactive;
5844 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5845 DemandedElts: OriginalDemandedElts, Known&: KnownValue, TLO,
5846 Depth: Depth + 1))
5847 return true;
5848 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), DemandedBits: OriginalDemandedBits,
5849 DemandedElts: OriginalDemandedElts, Known&: KnownInactive, TLO,
5850 Depth: Depth + 1))
5851 return true;
5852 Known = KnownValue.intersectWith(RHS: KnownInactive);
5853 break;
5854 }
5855 default:
5856 break;
5857 }
5858 break;
5859 }
5860 default:
5861 break;
5862 }
5863
5864 return false;
5865}
5866
5867//===----------------------------------------------------------------------===//
5868// Helper functions
5869//===----------------------------------------------------------------------===//
5870
5871SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
5872 const TargetRegisterClass *RC,
5873 Register Reg, EVT VT,
5874 const SDLoc &SL,
5875 bool RawReg) const {
5876 MachineFunction &MF = DAG.getMachineFunction();
5877 MachineRegisterInfo &MRI = MF.getRegInfo();
5878 Register VReg;
5879
5880 if (!MRI.isLiveIn(Reg)) {
5881 VReg = MRI.createVirtualRegister(RegClass: RC);
5882 MRI.addLiveIn(Reg, vreg: VReg);
5883 } else {
5884 VReg = MRI.getLiveInVirtReg(PReg: Reg);
5885 }
5886
5887 if (RawReg)
5888 return DAG.getRegister(Reg: VReg, VT);
5889
5890 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: VReg, VT);
5891}
5892
5893// This may be called multiple times, and nothing prevents creating multiple
5894// objects at the same offset. See if we already defined this object.
5895static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
5896 int64_t Offset) {
5897 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
5898 if (MFI.getObjectOffset(ObjectIdx: I) == Offset) {
5899 assert(MFI.getObjectSize(I) == Size);
5900 return I;
5901 }
5902 }
5903
5904 return MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
5905}
5906
5907SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
5908 EVT VT,
5909 const SDLoc &SL,
5910 int64_t Offset) const {
5911 MachineFunction &MF = DAG.getMachineFunction();
5912 MachineFrameInfo &MFI = MF.getFrameInfo();
5913 int FI = getOrCreateFixedStackObject(MFI, Size: VT.getStoreSize(), Offset);
5914
5915 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
5916 SDValue Ptr = DAG.getFrameIndex(FI, VT: MVT::i32);
5917
5918 return DAG.getLoad(VT, dl: SL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: SrcPtrInfo, Alignment: Align(4),
5919 MMOFlags: MachineMemOperand::MODereferenceable |
5920 MachineMemOperand::MOInvariant);
5921}
5922
5923SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
5924 const SDLoc &SL,
5925 SDValue Chain,
5926 SDValue ArgVal,
5927 int64_t Offset) const {
5928 MachineFunction &MF = DAG.getMachineFunction();
5929 MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
5930 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5931
5932 SDValue Ptr = DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32);
5933 // Stores to the argument stack area are relative to the stack pointer.
5934 SDValue SP =
5935 DAG.getCopyFromReg(Chain, dl: SL, Reg: Info->getStackPtrOffsetReg(), VT: MVT::i32);
5936 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: SP, N2: Ptr);
5937 SDValue Store = DAG.getStore(Chain, dl: SL, Val: ArgVal, Ptr, PtrInfo: DstInfo, Alignment: Align(4),
5938 MMOFlags: MachineMemOperand::MODereferenceable);
5939 return Store;
5940}
5941
5942SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
5943 const TargetRegisterClass *RC,
5944 EVT VT, const SDLoc &SL,
5945 const ArgDescriptor &Arg) const {
5946 assert(Arg && "Attempting to load missing argument");
5947
5948 SDValue V = Arg.isRegister() ?
5949 CreateLiveInRegister(DAG, RC, Reg: Arg.getRegister(), VT, SL) :
5950 loadStackInputValue(DAG, VT, SL, Offset: Arg.getStackOffset());
5951
5952 if (!Arg.isMasked())
5953 return V;
5954
5955 unsigned Mask = Arg.getMask();
5956 unsigned Shift = llvm::countr_zero<unsigned>(Val: Mask);
5957 V = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: V,
5958 N2: DAG.getShiftAmountConstant(Val: Shift, VT, DL: SL));
5959 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT, N1: V,
5960 N2: DAG.getConstant(Val: Mask >> Shift, DL: SL, VT));
5961}
5962
5963uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5964 uint64_t ExplicitKernArgSize, const ImplicitParameter Param) const {
5965 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
5966 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr();
5967 uint64_t ArgOffset =
5968 alignTo(Size: ExplicitKernArgSize, A: Alignment) + ExplicitArgOffset;
5969 switch (Param) {
5970 case FIRST_IMPLICIT:
5971 return ArgOffset;
5972 case PRIVATE_BASE:
5973 return ArgOffset + AMDGPU::ImplicitArg::PRIVATE_BASE_OFFSET;
5974 case SHARED_BASE:
5975 return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET;
5976 case QUEUE_PTR:
5977 return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET;
5978 }
5979 llvm_unreachable("unexpected implicit parameter type");
5980}
5981
5982uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5983 const MachineFunction &MF, const ImplicitParameter Param) const {
5984 const AMDGPUMachineFunctionInfo *MFI =
5985 MF.getInfo<AMDGPUMachineFunctionInfo>();
5986 return getImplicitParameterOffset(ExplicitKernArgSize: MFI->getExplicitKernArgSize(), Param);
5987}
5988
5989SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
5990 SelectionDAG &DAG, int Enabled,
5991 int &RefinementSteps,
5992 bool &UseOneConstNR,
5993 bool Reciprocal) const {
5994 EVT VT = Operand.getValueType();
5995
5996 if (VT == MVT::f32) {
5997 RefinementSteps = 0;
5998 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(Operand), VT, Operand);
5999 }
6000
6001 // TODO: There is also f64 rsq instruction, but the documentation is less
6002 // clear on its precision.
6003
6004 return SDValue();
6005}
6006
6007SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
6008 SelectionDAG &DAG, int Enabled,
6009 int &RefinementSteps) const {
6010 EVT VT = Operand.getValueType();
6011
6012 if (VT == MVT::f32) {
6013 // Reciprocal, < 1 ulp error.
6014 //
6015 // This reciprocal approximation converges to < 0.5 ulp error with one
6016 // newton rhapson performed with two fused multiple adds (FMAs).
6017
6018 RefinementSteps = 0;
6019 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SDLoc(Operand), VT, Operand);
6020 }
6021
6022 // TODO: There is also f64 rcp instruction, but the documentation is less
6023 // clear on its precision.
6024
6025 return SDValue();
6026}
6027
6028static unsigned workitemIntrinsicDim(unsigned ID) {
6029 switch (ID) {
6030 case Intrinsic::amdgcn_workitem_id_x:
6031 return 0;
6032 case Intrinsic::amdgcn_workitem_id_y:
6033 return 1;
6034 case Intrinsic::amdgcn_workitem_id_z:
6035 return 2;
6036 default:
6037 llvm_unreachable("not a workitem intrinsic");
6038 }
6039}
6040
6041void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
6042 const SDValue Op, KnownBits &Known,
6043 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
6044
6045 Known.resetAll(); // Don't know anything.
6046
6047 unsigned Opc = Op.getOpcode();
6048
6049 switch (Opc) {
6050 default:
6051 break;
6052 case AMDGPUISD::CARRY:
6053 case AMDGPUISD::BORROW: {
6054 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31);
6055 break;
6056 }
6057
6058 case AMDGPUISD::BFE_I32:
6059 case AMDGPUISD::BFE_U32: {
6060 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6061 if (!CWidth)
6062 return;
6063
6064 uint32_t Width = CWidth->getZExtValue() & 0x1f;
6065
6066 if (Opc == AMDGPUISD::BFE_U32)
6067 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - Width);
6068
6069 break;
6070 }
6071 case AMDGPUISD::FP_TO_FP16: {
6072 unsigned BitWidth = Known.getBitWidth();
6073
6074 // High bits are zero.
6075 Known.Zero = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 16);
6076 break;
6077 }
6078 case AMDGPUISD::MUL_U24:
6079 case AMDGPUISD::MUL_I24: {
6080 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6081 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6082 unsigned BitWidth = Op.getScalarValueSizeInBits();
6083
6084 // Sign/Zero extend from 24 bits.
6085 if (Opc == AMDGPUISD::MUL_I24) {
6086 LHSKnown = LHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6087 RHSKnown = RHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6088 } else {
6089 LHSKnown = LHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6090 RHSKnown = RHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6091 }
6092
6093 // TODO: SelfMultiply can be poison, but not undef.
6094 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
6095 if (SelfMultiply)
6096 SelfMultiply &= DAG.isGuaranteedNotToBeUndefOrPoison(
6097 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
6098 Depth: Depth + 1);
6099
6100 Known = KnownBits::mul(LHS: LHSKnown, RHS: RHSKnown, NoUndefSelfMultiply: SelfMultiply);
6101 break;
6102 }
6103 case AMDGPUISD::PERM: {
6104 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6105 if (!CMask)
6106 return;
6107
6108 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6109 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6110 unsigned Sel = CMask->getZExtValue();
6111
6112 for (unsigned I = 0; I < 32; I += 8) {
6113 unsigned SelBits = Sel & 0xff;
6114 if (SelBits < 4) {
6115 SelBits *= 8;
6116 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6117 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6118 } else if (SelBits < 7) {
6119 SelBits = (SelBits & 3) * 8;
6120 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6121 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6122 } else if (SelBits == 0x0c) {
6123 Known.Zero |= 0xFFull << I;
6124 } else if (SelBits > 0x0c) {
6125 Known.One |= 0xFFull << I;
6126 }
6127 Sel >>= 8;
6128 }
6129 break;
6130 }
6131 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
6132 Known.Zero.setHighBits(24);
6133 break;
6134 }
6135 case AMDGPUISD::BUFFER_LOAD_USHORT: {
6136 Known.Zero.setHighBits(16);
6137 break;
6138 }
6139 case AMDGPUISD::LDS: {
6140 auto *GA = cast<GlobalAddressSDNode>(Val: Op.getOperand(i: 0).getNode());
6141 Align Alignment = GA->getGlobal()->getPointerAlignment(DL: DAG.getDataLayout());
6142
6143 Known.Zero.setHighBits(16);
6144 Known.Zero.setLowBits(Log2(A: Alignment));
6145 break;
6146 }
6147 case AMDGPUISD::SMIN3:
6148 case AMDGPUISD::SMAX3:
6149 case AMDGPUISD::SMED3:
6150 case AMDGPUISD::UMIN3:
6151 case AMDGPUISD::UMAX3:
6152 case AMDGPUISD::UMED3: {
6153 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6154 if (Known2.isUnknown())
6155 break;
6156
6157 KnownBits Known1 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6158 if (Known1.isUnknown())
6159 break;
6160
6161 KnownBits Known0 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6162 if (Known0.isUnknown())
6163 break;
6164
6165 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
6166 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
6167 Known.One = Known0.One & Known1.One & Known2.One;
6168 break;
6169 }
6170 case ISD::INTRINSIC_WO_CHAIN: {
6171 unsigned IID = Op.getConstantOperandVal(i: 0);
6172 switch (IID) {
6173 case Intrinsic::amdgcn_workitem_id_x:
6174 case Intrinsic::amdgcn_workitem_id_y:
6175 case Intrinsic::amdgcn_workitem_id_z: {
6176 unsigned MaxValue = Subtarget->getMaxWorkitemID(
6177 Kernel: DAG.getMachineFunction().getFunction(), Dimension: workitemIntrinsicDim(ID: IID));
6178 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
6179 break;
6180 }
6181 case Intrinsic::amdgcn_readfirstlane:
6182 case Intrinsic::amdgcn_readlane:
6183 // Result is the data operand's value from some lane.
6184 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6185 break;
6186 default:
6187 break;
6188 }
6189 }
6190 }
6191}
6192
6193unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
6194 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6195 unsigned Depth) const {
6196 switch (Op.getOpcode()) {
6197 case AMDGPUISD::BFE_I32: {
6198 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6199 if (!Width)
6200 return 1;
6201
6202 unsigned SignBits = 32 - (Width->getZExtValue() & 0x1f) + 1;
6203 if (!isNullConstant(V: Op.getOperand(i: 1)))
6204 return SignBits;
6205
6206 // TODO: Could probably figure something out with non-0 offsets.
6207 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6208 return std::max(a: SignBits, b: Op0SignBits);
6209 }
6210
6211 case AMDGPUISD::BFE_U32: {
6212 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6213 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
6214 }
6215
6216 case AMDGPUISD::CARRY:
6217 case AMDGPUISD::BORROW:
6218 return 31;
6219 case AMDGPUISD::BUFFER_LOAD_BYTE:
6220 return 25;
6221 case AMDGPUISD::BUFFER_LOAD_SHORT:
6222 return 17;
6223 case AMDGPUISD::BUFFER_LOAD_UBYTE:
6224 return 24;
6225 case AMDGPUISD::BUFFER_LOAD_USHORT:
6226 return 16;
6227 case AMDGPUISD::FP_TO_FP16:
6228 return 16;
6229 case AMDGPUISD::SMIN3:
6230 case AMDGPUISD::SMAX3:
6231 case AMDGPUISD::SMED3:
6232 case AMDGPUISD::UMIN3:
6233 case AMDGPUISD::UMAX3:
6234 case AMDGPUISD::UMED3: {
6235 unsigned Tmp2 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6236 if (Tmp2 == 1)
6237 return 1; // Early out.
6238
6239 unsigned Tmp1 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6240 if (Tmp1 == 1)
6241 return 1; // Early out.
6242
6243 unsigned Tmp0 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6244 if (Tmp0 == 1)
6245 return 1; // Early out.
6246
6247 return std::min(l: {Tmp0, Tmp1, Tmp2});
6248 }
6249 default:
6250 return 1;
6251 }
6252}
6253
6254unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
6255 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
6256 const MachineRegisterInfo &MRI, unsigned Depth) const {
6257 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
6258 if (!MI)
6259 return 1;
6260
6261 // TODO: Check range metadata on MMO.
6262 switch (MI->getOpcode()) {
6263 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
6264 return 25;
6265 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
6266 return 17;
6267 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
6268 return 24;
6269 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
6270 return 16;
6271 case AMDGPU::G_AMDGPU_SMED3:
6272 case AMDGPU::G_AMDGPU_UMED3: {
6273 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
6274 unsigned Tmp2 = Analysis.computeNumSignBits(R: Src2, DemandedElts, Depth: Depth + 1);
6275 if (Tmp2 == 1)
6276 return 1;
6277 unsigned Tmp1 = Analysis.computeNumSignBits(R: Src1, DemandedElts, Depth: Depth + 1);
6278 if (Tmp1 == 1)
6279 return 1;
6280 unsigned Tmp0 = Analysis.computeNumSignBits(R: Src0, DemandedElts, Depth: Depth + 1);
6281 if (Tmp0 == 1)
6282 return 1;
6283 return std::min(l: {Tmp0, Tmp1, Tmp2});
6284 }
6285 default:
6286 return 1;
6287 }
6288}
6289
6290bool AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode(
6291 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6292 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
6293 unsigned Opcode = Op.getOpcode();
6294 switch (Opcode) {
6295 case AMDGPUISD::BFE_I32:
6296 case AMDGPUISD::BFE_U32:
6297 return false;
6298 }
6299 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
6300 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
6301}
6302
6303bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
6304 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
6305 unsigned Depth) const {
6306 unsigned Opcode = Op.getOpcode();
6307 switch (Opcode) {
6308 case AMDGPUISD::FMIN_LEGACY:
6309 case AMDGPUISD::FMAX_LEGACY:
6310 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6311 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6312 case AMDGPUISD::FMUL_LEGACY:
6313 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
6314 if (SNaN)
6315 return true;
6316 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6317 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6318 }
6319 case AMDGPUISD::FMED3:
6320 case AMDGPUISD::FMIN3:
6321 case AMDGPUISD::FMAX3:
6322 case AMDGPUISD::FMINIMUM3:
6323 case AMDGPUISD::FMAXIMUM3:
6324 case AMDGPUISD::FMAD_FTZ: {
6325 if (SNaN)
6326 return true;
6327 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6328 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6329 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6330 }
6331 case AMDGPUISD::CVT_F32_UBYTE0:
6332 case AMDGPUISD::CVT_F32_UBYTE1:
6333 case AMDGPUISD::CVT_F32_UBYTE2:
6334 case AMDGPUISD::CVT_F32_UBYTE3:
6335 return true;
6336
6337 case AMDGPUISD::RCP:
6338 case AMDGPUISD::RSQ:
6339 case AMDGPUISD::RCP_LEGACY:
6340 case AMDGPUISD::RSQ_CLAMP: {
6341 if (SNaN)
6342 return true;
6343
6344 // TODO: Need is known positive check.
6345 return false;
6346 }
6347 case ISD::FLDEXP:
6348 case AMDGPUISD::FRACT: {
6349 if (SNaN)
6350 return true;
6351 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6352 }
6353 case AMDGPUISD::DIV_SCALE:
6354 case AMDGPUISD::DIV_FMAS:
6355 case AMDGPUISD::DIV_FIXUP:
6356 // TODO: Refine on operands.
6357 return SNaN;
6358 case AMDGPUISD::SIN_HW:
6359 case AMDGPUISD::COS_HW: {
6360 // TODO: Need check for infinity
6361 return SNaN;
6362 }
6363 case ISD::INTRINSIC_WO_CHAIN: {
6364 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
6365 // TODO: Handle more intrinsics
6366 switch (IntrinsicID) {
6367 case Intrinsic::amdgcn_cubeid:
6368 case Intrinsic::amdgcn_cvt_off_f32_i4:
6369 return true;
6370
6371 case Intrinsic::amdgcn_frexp_mant: {
6372 if (SNaN)
6373 return true;
6374 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6375 }
6376 case Intrinsic::amdgcn_cvt_pkrtz: {
6377 if (SNaN)
6378 return true;
6379 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6380 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6381 }
6382 case Intrinsic::amdgcn_rcp:
6383 case Intrinsic::amdgcn_rsq:
6384 case Intrinsic::amdgcn_rcp_legacy:
6385 case Intrinsic::amdgcn_rsq_legacy:
6386 case Intrinsic::amdgcn_rsq_clamp:
6387 case Intrinsic::amdgcn_tanh: {
6388 if (SNaN)
6389 return true;
6390
6391 // TODO: Need is known positive check.
6392 return false;
6393 }
6394 case Intrinsic::amdgcn_trig_preop:
6395 case Intrinsic::amdgcn_fdot2:
6396 // TODO: Refine on operand
6397 return SNaN;
6398 case Intrinsic::amdgcn_fma_legacy:
6399 if (SNaN)
6400 return true;
6401 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6402 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1) &&
6403 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 3), SNaN, Depth: Depth + 1);
6404 default:
6405 return false;
6406 }
6407 }
6408 default:
6409 return false;
6410 }
6411}
6412
6413bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
6414 Register N0, Register N1) const {
6415 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
6416}
6417