1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This is the parent TargetLowering class for hardware code gen
11/// targets.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUISelLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUMachineFunctionInfo.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUSelectionDAGInfo.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/Analysis.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/Support/AMDGPUAddrSpace.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/Support/KnownBits.h"
30#include "llvm/Target/TargetMachine.h"
31
32using namespace llvm;
33
34#define GET_CALLING_CONV_IMPL
35#include "AMDGPUGenCallingConv.inc"
36
37static cl::opt<bool> AMDGPUBypassSlowDiv(
38 "amdgpu-bypass-slow-div",
39 cl::desc("Skip 64-bit divide for dynamic 32-bit values"),
40 cl::init(Val: true));
41
42// Find a larger type to do a load / store of a vector with.
43EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
44 unsigned StoreSize = VT.getStoreSizeInBits();
45 if (StoreSize <= 32)
46 return EVT::getIntegerVT(Context&: Ctx, BitWidth: StoreSize);
47
48 if (StoreSize % 32 == 0)
49 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i32, NumElements: StoreSize / 32);
50
51 return VT;
52}
53
54unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
55 return DAG.computeKnownBits(Op).countMaxActiveBits();
56}
57
58unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
59 // In order for this to be a signed 24-bit value, bit 23, must
60 // be a sign bit.
61 return DAG.ComputeMaxSignificantBits(Op);
62}
63
64AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
65 const TargetSubtargetInfo &STI,
66 const AMDGPUSubtarget &AMDGPUSTI)
67 : TargetLowering(TM, STI), Subtarget(&AMDGPUSTI) {
68 // Always lower memset, memcpy, and memmove intrinsics to load/store
69 // instructions, rather then generating calls to memset, mempcy or memmove.
70 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U;
71 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U;
72 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U;
73
74 // Enable ganging up loads and stores in the memcpy DAG lowering.
75 MaxGluedStoresPerMemcpy = 16;
76
77 // Lower floating point store/load to integer store/load to reduce the number
78 // of patterns in tablegen.
79 setOperationAction(Op: ISD::LOAD, VT: MVT::f32, Action: Promote);
80 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
81
82 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Promote);
83 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
84
85 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f32, Action: Promote);
86 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
87
88 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f32, Action: Promote);
89 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
90
91 setOperationAction(Op: ISD::LOAD, VT: MVT::v5f32, Action: Promote);
92 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
93
94 setOperationAction(Op: ISD::LOAD, VT: MVT::v6f32, Action: Promote);
95 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
96
97 setOperationAction(Op: ISD::LOAD, VT: MVT::v7f32, Action: Promote);
98 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
99
100 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f32, Action: Promote);
101 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
102
103 setOperationAction(Op: ISD::LOAD, VT: MVT::v9f32, Action: Promote);
104 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
105
106 setOperationAction(Op: ISD::LOAD, VT: MVT::v10f32, Action: Promote);
107 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
108
109 setOperationAction(Op: ISD::LOAD, VT: MVT::v11f32, Action: Promote);
110 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
111
112 setOperationAction(Op: ISD::LOAD, VT: MVT::v12f32, Action: Promote);
113 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
114
115 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f32, Action: Promote);
116 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
117
118 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f32, Action: Promote);
119 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
120
121 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Promote);
122 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i64, DestVT: MVT::v2i32);
123
124 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
125 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
126
127 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Promote);
128 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f64, DestVT: MVT::v2i32);
129
130 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Promote);
131 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
132
133 setOperationAction(Op: ISD::LOAD, VT: MVT::v3i64, Action: Promote);
134 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
135
136 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i64, Action: Promote);
137 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
138
139 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f64, Action: Promote);
140 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
141
142 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f64, Action: Promote);
143 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
144
145 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i64, Action: Promote);
146 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
147
148 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f64, Action: Promote);
149 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
150
151 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i64, Action: Promote);
152 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
153
154 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f64, Action: Promote);
155 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
156
157 setOperationAction(Op: ISD::LOAD, VT: MVT::i128, Action: Promote);
158 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i128, DestVT: MVT::v4i32);
159
160 // There are no 64-bit extloads. These should be done as a 32-bit extload and
161 // an extension to 64-bit.
162 for (MVT VT : MVT::integer_valuetypes())
163 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64, MemVT: VT,
164 Action: Expand);
165
166 for (MVT VT : MVT::integer_valuetypes()) {
167 if (VT == MVT::i64)
168 continue;
169
170 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) {
171 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i1, Action: Promote);
172 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i8, Action: Legal);
173 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i16, Action: Legal);
174 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i32, Action: Expand);
175 }
176 }
177
178 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
179 for (auto MemVT :
180 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
181 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT,
182 Action: Expand);
183
184 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
185 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
186 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
187 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
188 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
189 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
190 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
191 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
192 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
193 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
194 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
195 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
196 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
197 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
198
199 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
200 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
201 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
202 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
203 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
204 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
205
206 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
207 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
208 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
209 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
210 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
211 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
212 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
213 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
214 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
215 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
216 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
218
219 setOperationAction(Op: ISD::STORE, VT: MVT::f32, Action: Promote);
220 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
221
222 setOperationAction(Op: ISD::STORE, VT: MVT::v2f32, Action: Promote);
223 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
224
225 setOperationAction(Op: ISD::STORE, VT: MVT::v3f32, Action: Promote);
226 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
227
228 setOperationAction(Op: ISD::STORE, VT: MVT::v4f32, Action: Promote);
229 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
230
231 setOperationAction(Op: ISD::STORE, VT: MVT::v5f32, Action: Promote);
232 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
233
234 setOperationAction(Op: ISD::STORE, VT: MVT::v6f32, Action: Promote);
235 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
236
237 setOperationAction(Op: ISD::STORE, VT: MVT::v7f32, Action: Promote);
238 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
239
240 setOperationAction(Op: ISD::STORE, VT: MVT::v8f32, Action: Promote);
241 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
242
243 setOperationAction(Op: ISD::STORE, VT: MVT::v9f32, Action: Promote);
244 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
245
246 setOperationAction(Op: ISD::STORE, VT: MVT::v10f32, Action: Promote);
247 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
248
249 setOperationAction(Op: ISD::STORE, VT: MVT::v11f32, Action: Promote);
250 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
251
252 setOperationAction(Op: ISD::STORE, VT: MVT::v12f32, Action: Promote);
253 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
254
255 setOperationAction(Op: ISD::STORE, VT: MVT::v16f32, Action: Promote);
256 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
257
258 setOperationAction(Op: ISD::STORE, VT: MVT::v32f32, Action: Promote);
259 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
260
261 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Promote);
262 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i64, DestVT: MVT::v2i32);
263
264 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
265 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
266
267 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Promote);
268 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f64, DestVT: MVT::v2i32);
269
270 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Promote);
271 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
272
273 setOperationAction(Op: ISD::STORE, VT: MVT::v3i64, Action: Promote);
274 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
275
276 setOperationAction(Op: ISD::STORE, VT: MVT::v3f64, Action: Promote);
277 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
278
279 setOperationAction(Op: ISD::STORE, VT: MVT::v4i64, Action: Promote);
280 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
281
282 setOperationAction(Op: ISD::STORE, VT: MVT::v4f64, Action: Promote);
283 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
284
285 setOperationAction(Op: ISD::STORE, VT: MVT::v8i64, Action: Promote);
286 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
287
288 setOperationAction(Op: ISD::STORE, VT: MVT::v8f64, Action: Promote);
289 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
290
291 setOperationAction(Op: ISD::STORE, VT: MVT::v16i64, Action: Promote);
292 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
293
294 setOperationAction(Op: ISD::STORE, VT: MVT::v16f64, Action: Promote);
295 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
296
297 setOperationAction(Op: ISD::STORE, VT: MVT::i128, Action: Promote);
298 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i128, DestVT: MVT::v4i32);
299
300 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i1, Action: Expand);
301 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i8, Action: Expand);
302 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
303 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i32, Action: Expand);
304
305 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i1, Action: Expand);
306 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i8, Action: Expand);
307 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i16, Action: Expand);
308 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i32, Action: Expand);
309
310 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
311 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
312 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
313 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
314 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
315 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
316 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
317 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
318 setTruncStoreAction(ValVT: MVT::v6f32, MemVT: MVT::v6f16, Action: Expand);
319 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
320 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
321 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
322 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
323 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
324 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
325
326 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
327 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
328 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
329
330 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
331 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
332 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
333
334 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i8, Action: Expand);
335
336 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
337 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
338 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i8, Action: Expand);
339 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i1, Action: Expand);
340 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
341 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
342 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
343
344 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i32, Action: Expand);
345 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i16, Action: Expand);
346 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
347 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
348 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
349
350 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i1, Action: Expand);
351 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i8, Action: Expand);
352 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i16, Action: Expand);
353
354 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i1, Action: Expand);
355 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i8, Action: Expand);
356 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i16, Action: Expand);
357
358 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i1, Action: Expand);
359 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i8, Action: Expand);
360 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i16, Action: Expand);
361
362 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
363 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
364 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
365
366 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
367 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
368 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
369 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i16, Action: Expand);
370 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
371 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
372 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i1, Action: Expand);
373
374 setOperationAction(Ops: ISD::Constant, VTs: {MVT::i32, MVT::i64}, Action: Legal);
375 setOperationAction(Ops: ISD::ConstantFP, VTs: {MVT::f32, MVT::f64}, Action: Legal);
376
377 setOperationAction(Ops: {ISD::BR_JT, ISD::BRIND}, VT: MVT::Other, Action: Expand);
378
379 // For R600, this is totally unsupported, just custom lower to produce an
380 // error.
381 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
382
383 // Library functions. These default to Expand, but we have instructions
384 // for them.
385 setOperationAction(
386 Ops: {ISD::FCEIL, ISD::FABS, ISD::FFLOOR, ISD::FROUNDEVEN, ISD::FTRUNC},
387 VTs: {MVT::f16, MVT::f32}, Action: Legal);
388 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM}, VT: MVT::f32, Action: Legal);
389
390 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Custom);
391 setOperationAction(Ops: ISD::FROUND, VTs: {MVT::f32, MVT::f64}, Action: Custom);
392 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND},
393 VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
394
395 setOperationAction(
396 Ops: {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10, ISD::FPOW},
397 VT: MVT::f32, Action: Custom);
398 setOperationAction(Ops: {ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, VT: MVT::f64, Action: Custom);
399
400 setOperationAction(Ops: ISD::FNEARBYINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
401
402 setOperationAction(Ops: ISD::FRINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
403
404 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VTs: {MVT::f16, MVT::f32, MVT::f64},
405 Action: Expand);
406
407 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
408 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
409 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Custom);
410
411 setOperationAction(Ops: {ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, VT: MVT::f16,
412 Action: Custom);
413
414 setOperationAction(Ops: ISD::FCANONICALIZE, VTs: {MVT::f32, MVT::f64}, Action: Legal);
415
416 // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
417 // scalarization code. Can be removed when IS_FPCLASS expand isn't called by
418 // default unless marked custom/legal.
419 setOperationAction(Ops: ISD::IS_FPCLASS,
420 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
421 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
422 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64,
423 MVT::v16f64},
424 Action: Custom);
425
426 // Expand to fneg + fadd.
427 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
428
429 setOperationAction(Ops: ISD::CONCAT_VECTORS,
430 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
431 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
432 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
433 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
434 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
435 Action: Custom);
436
437 setOperationAction(
438 Ops: ISD::EXTRACT_SUBVECTOR,
439 VTs: {MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32,
440 MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32,
441 MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32,
442 MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32,
443 MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
444 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
445 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64},
446 Action: Custom);
447
448 setOperationAction(Ops: {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP}, VT: MVT::f64,
449 Action: Expand);
450 setOperationAction(Ops: ISD::FP_TO_FP16, VTs: {MVT::f64, MVT::f32}, Action: Custom);
451
452 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
453 for (MVT VT : ScalarIntVTs) {
454 // These should use [SU]DIVREM, so set them to expand
455 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT,
456 Action: Expand);
457
458 // GPU does not have divrem function for signed or unsigned.
459 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT, Action: Custom);
460
461 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
462 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
463
464 setOperationAction(Ops: {ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Action: Expand);
465
466 setOperationAction(Ops: {ISD::ADDC, ISD::SUBC, ISD::ADDE, ISD::SUBE}, VT,
467 Action: Expand);
468 }
469
470 // The hardware supports 32-bit FSHR, but not FSHL.
471 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
472
473 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VTs: {MVT::i32, MVT::i64}, Action: Expand);
474
475 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT: MVT::i16, Action: Expand);
476
477 setOperationAction(Ops: {ISD::MUL, ISD::MULHU, ISD::MULHS}, VT: MVT::i64, Action: Expand);
478 setOperationAction(Ops: {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT,
479 ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
480 ISD::FP_TO_UINT_SAT},
481 VT: MVT::i64, Action: Custom);
482 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: Expand);
483
484 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i32,
485 Action: Legal);
486
487 setOperationAction(
488 Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON},
489 VT: MVT::i64, Action: Custom);
490
491 for (auto VT : {MVT::i8, MVT::i16})
492 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT, Action: Custom);
493
494 static const MVT::SimpleValueType VectorIntTypes[] = {
495 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
496 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
497
498 for (MVT VT : VectorIntTypes) {
499 // Expand the following operations for the current type by default.
500 // clang-format off
501 setOperationAction(Ops: {ISD::ADD, ISD::AND,
502 ISD::FP_TO_SINT, ISD::FP_TO_UINT,
503 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
504 ISD::MUL, ISD::MULHU,
505 ISD::MULHS, ISD::OR,
506 ISD::SHL, ISD::SRA,
507 ISD::SRL, ISD::ROTL,
508 ISD::ROTR, ISD::SUB,
509 ISD::SINT_TO_FP, ISD::UINT_TO_FP,
510 ISD::SDIV, ISD::UDIV,
511 ISD::SREM, ISD::UREM,
512 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
513 ISD::SDIVREM, ISD::UDIVREM,
514 ISD::SELECT, ISD::VSELECT,
515 ISD::SELECT_CC, ISD::XOR,
516 ISD::BSWAP, ISD::CTPOP,
517 ISD::CTTZ, ISD::CTLZ,
518 ISD::VECTOR_SHUFFLE, ISD::SETCC,
519 ISD::ADDRSPACECAST},
520 VT, Action: Expand);
521 // clang-format on
522 }
523
524 static const MVT::SimpleValueType FloatVectorTypes[] = {
525 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
526 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
527
528 for (MVT VT : FloatVectorTypes) {
529 setOperationAction(
530 Ops: {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM,
531 ISD::FADD, ISD::FCEIL, ISD::FCOS,
532 ISD::FDIV, ISD::FEXP2, ISD::FEXP,
533 ISD::FEXP10, ISD::FLOG2, ISD::FREM,
534 ISD::FLOG, ISD::FLOG10, ISD::FPOW,
535 ISD::FFLOOR, ISD::FTRUNC, ISD::FMUL,
536 ISD::FMA, ISD::FRINT, ISD::FNEARBYINT,
537 ISD::FSQRT, ISD::FSIN, ISD::FSUB,
538 ISD::FNEG, ISD::VSELECT, ISD::SELECT_CC,
539 ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, ISD::SETCC,
540 ISD::FCANONICALIZE, ISD::FROUNDEVEN},
541 VT, Action: Expand);
542 }
543
544 // This causes using an unrolled select operation rather than expansion with
545 // bit operations. This is in general better, but the alternative using BFI
546 // instructions may be better if the select sources are SGPRs.
547 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f32, Action: Promote);
548 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
549
550 setOperationAction(Op: ISD::SELECT, VT: MVT::v3f32, Action: Promote);
551 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
552
553 setOperationAction(Op: ISD::SELECT, VT: MVT::v4f32, Action: Promote);
554 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
555
556 setOperationAction(Op: ISD::SELECT, VT: MVT::v5f32, Action: Promote);
557 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
558
559 setOperationAction(Op: ISD::SELECT, VT: MVT::v6f32, Action: Promote);
560 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
561
562 setOperationAction(Op: ISD::SELECT, VT: MVT::v7f32, Action: Promote);
563 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
564
565 setOperationAction(Op: ISD::SELECT, VT: MVT::v9f32, Action: Promote);
566 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
567
568 setOperationAction(Op: ISD::SELECT, VT: MVT::v10f32, Action: Promote);
569 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
570
571 setOperationAction(Op: ISD::SELECT, VT: MVT::v11f32, Action: Promote);
572 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
573
574 setOperationAction(Op: ISD::SELECT, VT: MVT::v12f32, Action: Promote);
575 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
576
577 setSchedulingPreference(Sched::RegPressure);
578 setJumpIsExpensive(true);
579
580 setMinCmpXchgSizeInBits(32);
581 setSupportsUnalignedAtomics(false);
582
583 PredictableSelectIsExpensive = false;
584
585 // We want to find all load dependencies for long chains of stores to enable
586 // merging into very wide vectors. The problem is with vectors with > 4
587 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
588 // vectors are a legal type, even though we have to split the loads
589 // usually. When we can more precisely specify load legality per address
590 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
591 // smarter so that they can figure out what to do in 2 iterations without all
592 // N > 4 stores on the same chain.
593 GatherAllAliasesMaxDepth = 16;
594
595 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
596 // about these during lowering.
597 MaxStoresPerMemcpy = 0xffffffff;
598 MaxStoresPerMemmove = 0xffffffff;
599 MaxStoresPerMemset = 0xffffffff;
600
601 // The expansion for 64-bit division is enormous.
602 if (AMDGPUBypassSlowDiv)
603 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
604
605 setTargetDAGCombine({ISD::BITCAST, ISD::SHL,
606 ISD::SRA, ISD::SRL,
607 ISD::TRUNCATE, ISD::MUL,
608 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
609 ISD::MULHU, ISD::MULHS,
610 ISD::SELECT, ISD::SELECT_CC,
611 ISD::STORE, ISD::FADD,
612 ISD::FSUB, ISD::FNEG,
613 ISD::FABS, ISD::AssertZext,
614 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN});
615
616 setMaxAtomicSizeInBitsSupported(64);
617 setMaxDivRemBitWidthSupported(64);
618 setMaxLargeFPConvertBitWidthSupported(64);
619}
620
621//===----------------------------------------------------------------------===//
622// Target Information
623//===----------------------------------------------------------------------===//
624
625LLVM_READNONE
626static bool fnegFoldsIntoOpcode(unsigned Opc) {
627 switch (Opc) {
628 case ISD::FADD:
629 case ISD::FSUB:
630 case ISD::FMUL:
631 case ISD::FMA:
632 case ISD::FMAD:
633 case ISD::FMINNUM:
634 case ISD::FMAXNUM:
635 case ISD::FMINNUM_IEEE:
636 case ISD::FMAXNUM_IEEE:
637 case ISD::FMINIMUM:
638 case ISD::FMAXIMUM:
639 case ISD::FMINIMUMNUM:
640 case ISD::FMAXIMUMNUM:
641 case ISD::SELECT:
642 case ISD::FSIN:
643 case ISD::FTRUNC:
644 case ISD::FRINT:
645 case ISD::FNEARBYINT:
646 case ISD::FROUNDEVEN:
647 case ISD::FCANONICALIZE:
648 case AMDGPUISD::RCP:
649 case AMDGPUISD::RCP_LEGACY:
650 case AMDGPUISD::RCP_IFLAG:
651 case AMDGPUISD::SIN_HW:
652 case AMDGPUISD::FMUL_LEGACY:
653 case AMDGPUISD::FMIN_LEGACY:
654 case AMDGPUISD::FMAX_LEGACY:
655 case AMDGPUISD::FMED3:
656 // TODO: handle llvm.amdgcn.fma.legacy
657 return true;
658 case ISD::BITCAST:
659 llvm_unreachable("bitcast is special cased");
660 default:
661 return false;
662 }
663}
664
665static bool fnegFoldsIntoOp(const SDNode *N) {
666 unsigned Opc = N->getOpcode();
667 if (Opc == ISD::BITCAST) {
668 // TODO: Is there a benefit to checking the conditions performFNegCombine
669 // does? We don't for the other cases.
670 SDValue BCSrc = N->getOperand(Num: 0);
671 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
672 return BCSrc.getNumOperands() == 2 &&
673 BCSrc.getOperand(i: 1).getValueSizeInBits() == 32;
674 }
675
676 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32;
677 }
678
679 return fnegFoldsIntoOpcode(Opc);
680}
681
682/// \p returns true if the operation will definitely need to use a 64-bit
683/// encoding, and thus will use a VOP3 encoding regardless of the source
684/// modifiers.
685LLVM_READONLY
686static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
687 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) ||
688 VT == MVT::f64;
689}
690
691/// Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the
692/// type for ISD::SELECT.
693LLVM_READONLY
694static bool selectSupportsSourceMods(const SDNode *N) {
695 // TODO: Only applies if select will be vector
696 return N->getValueType(ResNo: 0) == MVT::f32;
697}
698
699// Most FP instructions support source modifiers, but this could be refined
700// slightly.
701LLVM_READONLY
702static bool hasSourceMods(const SDNode *N) {
703 if (isa<MemSDNode>(Val: N))
704 return false;
705
706 switch (N->getOpcode()) {
707 case ISD::CopyToReg:
708 case ISD::FDIV:
709 case ISD::FREM:
710 case ISD::INLINEASM:
711 case ISD::INLINEASM_BR:
712 case AMDGPUISD::DIV_SCALE:
713 case ISD::INTRINSIC_W_CHAIN:
714
715 // TODO: Should really be looking at the users of the bitcast. These are
716 // problematic because bitcasts are used to legalize all stores to integer
717 // types.
718 case ISD::BITCAST:
719 return false;
720 case ISD::INTRINSIC_WO_CHAIN: {
721 switch (N->getConstantOperandVal(Num: 0)) {
722 case Intrinsic::amdgcn_interp_p1:
723 case Intrinsic::amdgcn_interp_p2:
724 case Intrinsic::amdgcn_interp_mov:
725 case Intrinsic::amdgcn_interp_p1_f16:
726 case Intrinsic::amdgcn_interp_p2_f16:
727 return false;
728 default:
729 return true;
730 }
731 }
732 case ISD::SELECT:
733 return selectSupportsSourceMods(N);
734 default:
735 return true;
736 }
737}
738
739bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N,
740 unsigned CostThreshold) {
741 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
742 // it is truly free to use a source modifier in all cases. If there are
743 // multiple users but for each one will necessitate using VOP3, there will be
744 // a code size increase. Try to avoid increasing code size unless we know it
745 // will save on the instruction count.
746 unsigned NumMayIncreaseSize = 0;
747 MVT VT = N->getValueType(ResNo: 0).getScalarType().getSimpleVT();
748
749 assert(!N->use_empty());
750
751 // XXX - Should this limit number of uses to check?
752 for (const SDNode *U : N->users()) {
753 if (!hasSourceMods(N: U))
754 return false;
755
756 if (!opMustUseVOP3Encoding(N: U, VT)) {
757 if (++NumMayIncreaseSize > CostThreshold)
758 return false;
759 }
760 }
761
762 return true;
763}
764
765EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
766 ISD::NodeType ExtendKind) const {
767 assert(!VT.isVector() && "only scalar expected");
768
769 // Round to the next multiple of 32-bits.
770 unsigned Size = VT.getSizeInBits();
771 if (Size <= 32)
772 return MVT::i32;
773 return EVT::getIntegerVT(Context, BitWidth: 32 * ((Size + 31) / 32));
774}
775
776unsigned AMDGPUTargetLowering::getVectorIdxWidth(const DataLayout &) const {
777 return 32;
778}
779
780bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
781 return true;
782}
783
784// The backend supports 32 and 64 bit floating point immediates.
785// FIXME: Why are we reporting vectors of FP immediates as legal?
786bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
787 bool ForCodeSize) const {
788 return isTypeLegal(VT: VT.getScalarType());
789}
790
791// We don't want to shrink f64 / f32 constants.
792bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
793 EVT ScalarVT = VT.getScalarType();
794 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
795}
796
797bool AMDGPUTargetLowering::shouldReduceLoadWidth(
798 SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
799 std::optional<unsigned> ByteOffset) const {
800 // TODO: This may be worth removing. Check regression tests for diffs.
801 if (!TargetLoweringBase::shouldReduceLoadWidth(Load: N, ExtTy, NewVT, ByteOffset))
802 return false;
803
804 unsigned NewSize = NewVT.getStoreSizeInBits();
805
806 // If we are reducing to a 32-bit load or a smaller multi-dword load,
807 // this is always better.
808 if (NewSize >= 32)
809 return true;
810
811 EVT OldVT = N->getValueType(ResNo: 0);
812 unsigned OldSize = OldVT.getStoreSizeInBits();
813
814 MemSDNode *MN = cast<MemSDNode>(Val: N);
815 unsigned AS = MN->getAddressSpace();
816 // Do not shrink an aligned scalar load to sub-dword.
817 // Scalar engine cannot do sub-dword loads.
818 // Do not enable for gfx1250+ even though it has sub-dword loads because
819 // this will convert:
820 // i16 = trunc (zextload i16->i32)
821 // to:
822 // i16 = (load i16)
823 // This transformation will be reversed by LowerLOAD resulting in an infinite
824 // loop. Also, tablegen already has a pattern to match zextload i16->i32, but
825 // load i16 will not be matched since there is no instruction that does it.
826 if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) &&
827 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
828 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
829 (isa<LoadSDNode>(Val: N) && AS == AMDGPUAS::GLOBAL_ADDRESS &&
830 MN->isInvariant())) &&
831 AMDGPU::isUniformMMO(MMO: MN->getMemOperand()))
832 return false;
833
834 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
835 // extloads, so doing one requires using a buffer_load. In cases where we
836 // still couldn't use a scalar load, using the wider load shouldn't really
837 // hurt anything.
838
839 // If the old size already had to be an extload, there's no harm in continuing
840 // to reduce the width.
841 return (OldSize < 32);
842}
843
844bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
845 const SelectionDAG &DAG,
846 const MachineMemOperand &MMO) const {
847
848 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
849
850 if (LoadTy.getScalarType() == MVT::i32)
851 return false;
852
853 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
854 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
855
856 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
857 return false;
858
859 unsigned Fast = 0;
860 return allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
861 VT: CastTy, MMO, Fast: &Fast) &&
862 Fast;
863}
864
865// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
866// profitable with the expansion for 64-bit since it's generally good to
867// speculate things.
868bool AMDGPUTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
869 return true;
870}
871
872bool AMDGPUTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
873 return true;
874}
875
876bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const {
877 switch (N->getOpcode()) {
878 case ISD::EntryToken:
879 case ISD::TokenFactor:
880 return true;
881 case ISD::INTRINSIC_WO_CHAIN: {
882 unsigned IntrID = N->getConstantOperandVal(Num: 0);
883 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
884 }
885 case ISD::INTRINSIC_W_CHAIN: {
886 unsigned IntrID = N->getConstantOperandVal(Num: 1);
887 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
888 }
889 case ISD::LOAD:
890 if (cast<LoadSDNode>(Val: N)->getMemOperand()->getAddrSpace() ==
891 AMDGPUAS::CONSTANT_ADDRESS_32BIT)
892 return true;
893 return false;
894 case AMDGPUISD::SETCC: // ballot-style instruction
895 return true;
896 }
897 return false;
898}
899
900SDValue AMDGPUTargetLowering::getNegatedExpression(
901 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
902 NegatibleCost &Cost, unsigned Depth) const {
903
904 switch (Op.getOpcode()) {
905 case ISD::FMA:
906 case ISD::FMAD: {
907 // Negating a fma is not free if it has users without source mods.
908 if (!allUsesHaveSourceMods(N: Op.getNode()))
909 return SDValue();
910 break;
911 }
912 case AMDGPUISD::RCP: {
913 SDValue Src = Op.getOperand(i: 0);
914 EVT VT = Op.getValueType();
915 SDLoc SL(Op);
916
917 SDValue NegSrc = getNegatedExpression(Op: Src, DAG, LegalOperations,
918 ForCodeSize, Cost, Depth: Depth + 1);
919 if (NegSrc)
920 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: NegSrc, Flags: Op->getFlags());
921 return SDValue();
922 }
923 default:
924 break;
925 }
926
927 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps: LegalOperations,
928 OptForSize: ForCodeSize, Cost, Depth);
929}
930
931//===---------------------------------------------------------------------===//
932// Target Properties
933//===---------------------------------------------------------------------===//
934
935bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
936 assert(VT.isFloatingPoint());
937
938 // Packed operations do not have a fabs modifier.
939 // Report this based on the end legalized type.
940 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
941}
942
943bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
944 assert(VT.isFloatingPoint());
945 // Report this based on the end legalized type.
946 VT = VT.getScalarType();
947 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
948}
949
950bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
951 unsigned NumElem,
952 unsigned AS) const {
953 return true;
954}
955
956bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
957 // There are few operations which truly have vector input operands. Any vector
958 // operation is going to involve operations on each component, and a
959 // build_vector will be a copy per element, so it always makes sense to use a
960 // build_vector input in place of the extracted element to avoid a copy into a
961 // super register.
962 //
963 // We should probably only do this if all users are extracts only, but this
964 // should be the common case.
965 return true;
966}
967
968bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
969 // Truncate is just accessing a subregister.
970
971 unsigned SrcSize = Source.getSizeInBits();
972 unsigned DestSize = Dest.getSizeInBits();
973
974 return DestSize < SrcSize && DestSize % 32 == 0 ;
975}
976
977bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
978 // Truncate is just accessing a subregister.
979
980 unsigned SrcSize = Source->getScalarSizeInBits();
981 unsigned DestSize = Dest->getScalarSizeInBits();
982
983 if (DestSize== 16 && Subtarget->has16BitInsts())
984 return SrcSize >= 32;
985
986 return DestSize < SrcSize && DestSize % 32 == 0;
987}
988
989bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
990 unsigned SrcSize = Src->getScalarSizeInBits();
991 unsigned DestSize = Dest->getScalarSizeInBits();
992
993 if (SrcSize == 16 && Subtarget->has16BitInsts())
994 return DestSize >= 32;
995
996 return SrcSize == 32 && DestSize == 64;
997}
998
999bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
1000 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
1001 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
1002 // this will enable reducing 64-bit operations the 32-bit, which is always
1003 // good.
1004
1005 if (Src == MVT::i16)
1006 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1007
1008 return Src == MVT::i32 && Dest == MVT::i64;
1009}
1010
1011bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
1012 EVT DestVT) const {
1013 switch (N->getOpcode()) {
1014 case ISD::ABS:
1015 case ISD::ADD:
1016 case ISD::SUB:
1017 case ISD::SHL:
1018 case ISD::SRL:
1019 case ISD::SRA:
1020 case ISD::AND:
1021 case ISD::OR:
1022 case ISD::XOR:
1023 case ISD::MUL:
1024 case ISD::SETCC:
1025 case ISD::SELECT:
1026 case ISD::SMIN:
1027 case ISD::SMAX:
1028 case ISD::UMIN:
1029 case ISD::UMAX:
1030 case ISD::USUBSAT:
1031 case ISD::UADDSAT:
1032 if (isTypeLegal(VT: MVT::i16) &&
1033 (!DestVT.isVector() ||
1034 !isOperationLegal(Op: ISD::ADD, VT: MVT::v2i16))) { // Check if VOP3P
1035 // Don't narrow back down to i16 if promoted to i32 already.
1036 if (!N->isDivergent() && DestVT.isInteger() &&
1037 DestVT.getScalarSizeInBits() > 1 &&
1038 DestVT.getScalarSizeInBits() <= 16 &&
1039 SrcVT.getScalarSizeInBits() > 16) {
1040 return false;
1041 }
1042 }
1043 return true;
1044 default:
1045 break;
1046 }
1047
1048 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
1049 // limited number of native 64-bit operations. Shrinking an operation to fit
1050 // in a single 32-bit register should always be helpful. As currently used,
1051 // this is much less general than the name suggests, and is only used in
1052 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
1053 // not profitable, and may actually be harmful.
1054 if (isa<LoadSDNode>(Val: N))
1055 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
1056
1057 return true;
1058}
1059
1060bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
1061 const SDNode* N, CombineLevel Level) const {
1062 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
1063 N->getOpcode() == ISD::SRL) &&
1064 "Expected shift op");
1065
1066 SDValue ShiftLHS = N->getOperand(Num: 0);
1067 if (!ShiftLHS->hasOneUse())
1068 return false;
1069
1070 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
1071 !ShiftLHS.getOperand(i: 0)->hasOneUse())
1072 return false;
1073
1074 // Always commute pre-type legalization and right shifts.
1075 // We're looking for shl(or(x,y),z) patterns.
1076 if (Level < CombineLevel::AfterLegalizeTypes ||
1077 N->getOpcode() != ISD::SHL || N->getOperand(Num: 0).getOpcode() != ISD::OR)
1078 return true;
1079
1080 // If only user is a i32 right-shift, then don't destroy a BFE pattern.
1081 if (N->getValueType(ResNo: 0) == MVT::i32 && N->hasOneUse() &&
1082 (N->user_begin()->getOpcode() == ISD::SRA ||
1083 N->user_begin()->getOpcode() == ISD::SRL))
1084 return false;
1085
1086 // Don't destroy or(shl(load_zext(),c), load_zext()) patterns.
1087 auto IsShiftAndLoad = [](SDValue LHS, SDValue RHS) {
1088 if (LHS.getOpcode() != ISD::SHL)
1089 return false;
1090 auto *RHSLd = dyn_cast<LoadSDNode>(Val&: RHS);
1091 auto *LHS0 = dyn_cast<LoadSDNode>(Val: LHS.getOperand(i: 0));
1092 auto *LHS1 = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
1093 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD &&
1094 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1095 RHSLd->getExtensionType() == ISD::ZEXTLOAD;
1096 };
1097 SDValue LHS = N->getOperand(Num: 0).getOperand(i: 0);
1098 SDValue RHS = N->getOperand(Num: 0).getOperand(i: 1);
1099 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS));
1100}
1101
1102//===---------------------------------------------------------------------===//
1103// TargetLowering Callbacks
1104//===---------------------------------------------------------------------===//
1105
1106CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
1107 bool IsVarArg) {
1108 switch (CC) {
1109 case CallingConv::AMDGPU_VS:
1110 case CallingConv::AMDGPU_GS:
1111 case CallingConv::AMDGPU_PS:
1112 case CallingConv::AMDGPU_CS:
1113 case CallingConv::AMDGPU_HS:
1114 case CallingConv::AMDGPU_ES:
1115 case CallingConv::AMDGPU_LS:
1116 return CC_AMDGPU;
1117 case CallingConv::AMDGPU_CS_Chain:
1118 case CallingConv::AMDGPU_CS_ChainPreserve:
1119 return CC_AMDGPU_CS_CHAIN;
1120 case CallingConv::C:
1121 case CallingConv::Fast:
1122 case CallingConv::Cold:
1123 return CC_AMDGPU_Func;
1124 case CallingConv::AMDGPU_Gfx:
1125 case CallingConv::AMDGPU_Gfx_WholeWave:
1126 return CC_SI_Gfx;
1127 case CallingConv::AMDGPU_KERNEL:
1128 case CallingConv::SPIR_KERNEL:
1129 default:
1130 reportFatalUsageError(reason: "unsupported calling convention for call");
1131 }
1132}
1133
1134CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
1135 bool IsVarArg) {
1136 switch (CC) {
1137 case CallingConv::AMDGPU_KERNEL:
1138 case CallingConv::SPIR_KERNEL:
1139 llvm_unreachable("kernels should not be handled here");
1140 case CallingConv::AMDGPU_VS:
1141 case CallingConv::AMDGPU_GS:
1142 case CallingConv::AMDGPU_PS:
1143 case CallingConv::AMDGPU_CS:
1144 case CallingConv::AMDGPU_CS_Chain:
1145 case CallingConv::AMDGPU_CS_ChainPreserve:
1146 case CallingConv::AMDGPU_HS:
1147 case CallingConv::AMDGPU_ES:
1148 case CallingConv::AMDGPU_LS:
1149 return RetCC_SI_Shader;
1150 case CallingConv::AMDGPU_Gfx:
1151 case CallingConv::AMDGPU_Gfx_WholeWave:
1152 return RetCC_SI_Gfx;
1153 case CallingConv::C:
1154 case CallingConv::Fast:
1155 case CallingConv::Cold:
1156 return RetCC_AMDGPU_Func;
1157 default:
1158 reportFatalUsageError(reason: "unsupported calling convention");
1159 }
1160}
1161
1162/// The SelectionDAGBuilder will automatically promote function arguments
1163/// with illegal types. However, this does not work for the AMDGPU targets
1164/// since the function arguments are stored in memory as these illegal types.
1165/// In order to handle this properly we need to get the original types sizes
1166/// from the LLVM IR Function and fixup the ISD:InputArg values before
1167/// passing them to AnalyzeFormalArguments()
1168
1169/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
1170/// input values across multiple registers. Each item in the Ins array
1171/// represents a single value that will be stored in registers. Ins[x].VT is
1172/// the value type of the value that will be stored in the register, so
1173/// whatever SDNode we lower the argument to needs to be this type.
1174///
1175/// In order to correctly lower the arguments we need to know the size of each
1176/// argument. Since Ins[x].VT gives us the size of the register that will
1177/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
1178/// for the original function argument so that we can deduce the correct memory
1179/// type to use for Ins[x]. In most cases the correct memory type will be
1180/// Ins[x].ArgVT. However, this will not always be the case. If, for example,
1181/// we have a kernel argument of type v8i8, this argument will be split into
1182/// 8 parts and each part will be represented by its own item in the Ins array.
1183/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
1184/// the argument before it was split. From this, we deduce that the memory type
1185/// for each individual part is i8. We pass the memory type as LocVT to the
1186/// calling convention analysis function and the register type (Ins[x].VT) as
1187/// the ValVT.
1188void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
1189 CCState &State,
1190 const SmallVectorImpl<ISD::InputArg> &Ins) const {
1191 const MachineFunction &MF = State.getMachineFunction();
1192 const Function &Fn = MF.getFunction();
1193 LLVMContext &Ctx = Fn.getContext();
1194 const unsigned ExplicitOffset = Subtarget->getExplicitKernelArgOffset();
1195 CallingConv::ID CC = Fn.getCallingConv();
1196
1197 Align MaxAlign = Align(1);
1198 uint64_t ExplicitArgOffset = 0;
1199 const DataLayout &DL = Fn.getDataLayout();
1200
1201 unsigned InIndex = 0;
1202
1203 for (const Argument &Arg : Fn.args()) {
1204 const bool IsByRef = Arg.hasByRefAttr();
1205 Type *BaseArgTy = Arg.getType();
1206 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1207 Align Alignment = DL.getValueOrABITypeAlignment(
1208 Alignment: IsByRef ? Arg.getParamAlign() : std::nullopt, Ty: MemArgTy);
1209 MaxAlign = std::max(a: Alignment, b: MaxAlign);
1210 uint64_t AllocSize = DL.getTypeAllocSize(Ty: MemArgTy);
1211
1212 uint64_t ArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + ExplicitOffset;
1213 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + AllocSize;
1214
1215 // We're basically throwing away everything passed into us and starting over
1216 // to get accurate in-memory offsets. The "PartOffset" is completely useless
1217 // to us as computed in Ins.
1218 //
1219 // We also need to figure out what type legalization is trying to do to get
1220 // the correct memory offsets.
1221
1222 SmallVector<EVT, 16> ValueVTs;
1223 SmallVector<uint64_t, 16> Offsets;
1224 ComputeValueVTs(TLI: *this, DL, Ty: BaseArgTy, ValueVTs, /*MemVTs=*/nullptr,
1225 FixedOffsets: &Offsets, StartingOffset: ArgOffset);
1226
1227 for (unsigned Value = 0, NumValues = ValueVTs.size();
1228 Value != NumValues; ++Value) {
1229 uint64_t BasePartOffset = Offsets[Value];
1230
1231 EVT ArgVT = ValueVTs[Value];
1232 EVT MemVT = ArgVT;
1233 MVT RegisterVT = getRegisterTypeForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1234 unsigned NumRegs = getNumRegistersForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1235
1236 if (NumRegs == 1) {
1237 // This argument is not split, so the IR type is the memory type.
1238 if (ArgVT.isExtended()) {
1239 // We have an extended type, like i24, so we should just use the
1240 // register type.
1241 MemVT = RegisterVT;
1242 } else {
1243 MemVT = ArgVT;
1244 }
1245 } else if (ArgVT.isVector() && RegisterVT.isVector() &&
1246 ArgVT.getScalarType() == RegisterVT.getScalarType()) {
1247 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
1248 // We have a vector value which has been split into a vector with
1249 // the same scalar type, but fewer elements. This should handle
1250 // all the floating-point vector types.
1251 MemVT = RegisterVT;
1252 } else if (ArgVT.isVector() &&
1253 ArgVT.getVectorNumElements() == NumRegs) {
1254 // This arg has been split so that each element is stored in a separate
1255 // register.
1256 MemVT = ArgVT.getScalarType();
1257 } else if (ArgVT.isExtended()) {
1258 // We have an extended type, like i65.
1259 MemVT = RegisterVT;
1260 } else {
1261 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
1262 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0);
1263 if (RegisterVT.isInteger()) {
1264 MemVT = EVT::getIntegerVT(Context&: State.getContext(), BitWidth: MemoryBits);
1265 } else if (RegisterVT.isVector()) {
1266 assert(!RegisterVT.getScalarType().isFloatingPoint());
1267 unsigned NumElements = RegisterVT.getVectorNumElements();
1268 assert(MemoryBits % NumElements == 0);
1269 // This vector type has been split into another vector type with
1270 // a different elements size.
1271 EVT ScalarVT = EVT::getIntegerVT(Context&: State.getContext(),
1272 BitWidth: MemoryBits / NumElements);
1273 MemVT = EVT::getVectorVT(Context&: State.getContext(), VT: ScalarVT, NumElements);
1274 } else {
1275 llvm_unreachable("cannot deduce memory type.");
1276 }
1277 }
1278
1279 // Convert one element vectors to scalar.
1280 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
1281 MemVT = MemVT.getScalarType();
1282
1283 // Round up vec3/vec5 argument.
1284 if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
1285 MemVT = MemVT.getPow2VectorType(Context&: State.getContext());
1286 } else if (!MemVT.isSimple() && !MemVT.isVector()) {
1287 MemVT = MemVT.getRoundIntegerType(Context&: State.getContext());
1288 }
1289
1290 unsigned PartOffset = 0;
1291 for (unsigned i = 0; i != NumRegs; ++i) {
1292 State.addLoc(V: CCValAssign::getCustomMem(ValNo: InIndex++, ValVT: RegisterVT,
1293 Offset: BasePartOffset + PartOffset,
1294 LocVT: MemVT.getSimpleVT(),
1295 HTP: CCValAssign::Full));
1296 PartOffset += MemVT.getStoreSize();
1297 }
1298 }
1299 }
1300}
1301
1302SDValue AMDGPUTargetLowering::LowerReturn(
1303 SDValue Chain, CallingConv::ID CallConv,
1304 bool isVarArg,
1305 const SmallVectorImpl<ISD::OutputArg> &Outs,
1306 const SmallVectorImpl<SDValue> &OutVals,
1307 const SDLoc &DL, SelectionDAG &DAG) const {
1308 // FIXME: Fails for r600 tests
1309 //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
1310 // "wave terminate should not have return values");
1311 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM, DL, VT: MVT::Other, Operand: Chain);
1312}
1313
1314//===---------------------------------------------------------------------===//
1315// Target specific lowering
1316//===---------------------------------------------------------------------===//
1317
1318/// Selects the correct CCAssignFn for a given CallingConvention value.
1319CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1320 bool IsVarArg) {
1321 return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1322}
1323
1324CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1325 bool IsVarArg) {
1326 return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1327}
1328
1329SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1330 SelectionDAG &DAG,
1331 MachineFrameInfo &MFI,
1332 int ClobberedFI) const {
1333 SmallVector<SDValue, 8> ArgChains;
1334 int64_t FirstByte = MFI.getObjectOffset(ObjectIdx: ClobberedFI);
1335 int64_t LastByte = FirstByte + MFI.getObjectSize(ObjectIdx: ClobberedFI) - 1;
1336
1337 // Include the original chain at the beginning of the list. When this is
1338 // used by target LowerCall hooks, this helps legalize find the
1339 // CALLSEQ_BEGIN node.
1340 ArgChains.push_back(Elt: Chain);
1341
1342 // Add a chain value for each stack argument corresponding
1343 for (SDNode *U : DAG.getEntryNode().getNode()->users()) {
1344 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U)) {
1345 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr())) {
1346 if (FI->getIndex() < 0) {
1347 int64_t InFirstByte = MFI.getObjectOffset(ObjectIdx: FI->getIndex());
1348 int64_t InLastByte = InFirstByte;
1349 InLastByte += MFI.getObjectSize(ObjectIdx: FI->getIndex()) - 1;
1350
1351 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1352 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1353 ArgChains.push_back(Elt: SDValue(L, 1));
1354 }
1355 }
1356 }
1357 }
1358
1359 // Build a tokenfactor for all the chains.
1360 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
1361}
1362
1363SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1364 SmallVectorImpl<SDValue> &InVals,
1365 StringRef Reason) const {
1366 SDValue Callee = CLI.Callee;
1367 SelectionDAG &DAG = CLI.DAG;
1368
1369 const Function &Fn = DAG.getMachineFunction().getFunction();
1370
1371 StringRef FuncName("<unknown>");
1372
1373 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Val&: Callee))
1374 FuncName = G->getSymbol();
1375 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
1376 FuncName = G->getGlobal()->getName();
1377
1378 DAG.getContext()->diagnose(
1379 DI: DiagnosticInfoUnsupported(Fn, Reason + FuncName, CLI.DL.getDebugLoc()));
1380
1381 if (!CLI.IsTailCall) {
1382 for (ISD::InputArg &Arg : CLI.Ins)
1383 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
1384 }
1385
1386 // FIXME: Hack because R600 doesn't handle callseq pseudos yet.
1387 if (getTargetMachine().getTargetTriple().getArch() == Triple::r600)
1388 return CLI.Chain;
1389
1390 SDValue Chain = DAG.getCALLSEQ_START(Chain: CLI.Chain, InSize: 0, OutSize: 0, DL: CLI.DL);
1391 return DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, /*InGlue=*/Glue: SDValue(), DL: CLI.DL);
1392}
1393
1394SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1395 SmallVectorImpl<SDValue> &InVals) const {
1396 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported call to function ");
1397}
1398
1399SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1400 SelectionDAG &DAG) const {
1401 const Function &Fn = DAG.getMachineFunction().getFunction();
1402
1403 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1404 Fn, "unsupported dynamic alloca", SDLoc(Op).getDebugLoc()));
1405 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()), Op.getOperand(i: 0)};
1406 return DAG.getMergeValues(Ops, dl: SDLoc());
1407}
1408
1409SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1410 SelectionDAG &DAG) const {
1411 switch (Op.getOpcode()) {
1412 default:
1413 Op->print(OS&: errs(), G: &DAG);
1414 llvm_unreachable("Custom lowering code for this "
1415 "instruction is not implemented yet!");
1416 break;
1417 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1418 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1419 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1420 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1421 case ISD::SDIVREM:
1422 return LowerSDIVREM(Op, DAG);
1423 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1424 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1425 case ISD::FRINT: return LowerFRINT(Op, DAG);
1426 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1427 case ISD::FROUNDEVEN:
1428 return LowerFROUNDEVEN(Op, DAG);
1429 case ISD::FROUND: return LowerFROUND(Op, DAG);
1430 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1431 case ISD::FLOG2:
1432 return LowerFLOG2(Op, DAG);
1433 case ISD::FLOG:
1434 case ISD::FLOG10:
1435 return LowerFLOGCommon(Op, DAG);
1436 case ISD::FEXP:
1437 case ISD::FEXP10:
1438 return lowerFEXP(Op, DAG);
1439 case ISD::FEXP2:
1440 return lowerFEXP2(Op, DAG);
1441 case ISD::FPOW:
1442 return lowerFPOW(Op, DAG);
1443 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1444 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1445 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1446 case ISD::FP_TO_SINT:
1447 case ISD::FP_TO_UINT:
1448 return LowerFP_TO_INT(Op, DAG);
1449 case ISD::FP_TO_SINT_SAT:
1450 case ISD::FP_TO_UINT_SAT:
1451 return LowerFP_TO_INT_SAT(Op, DAG);
1452 case ISD::CTTZ:
1453 case ISD::CTTZ_ZERO_POISON:
1454 case ISD::CTLZ:
1455 case ISD::CTLZ_ZERO_POISON:
1456 return LowerCTLZ_CTTZ(Op, DAG);
1457 case ISD::CTLS:
1458 return LowerCTLS(Op, DAG);
1459 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1460 }
1461 return Op;
1462}
1463
1464void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
1465 SmallVectorImpl<SDValue> &Results,
1466 SelectionDAG &DAG) const {
1467 switch (N->getOpcode()) {
1468 case ISD::SIGN_EXTEND_INREG:
1469 // Different parts of legalization seem to interpret which type of
1470 // sign_extend_inreg is the one to check for custom lowering. The extended
1471 // from type is what really matters, but some places check for custom
1472 // lowering of the result type. This results in trying to use
1473 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1474 // nothing here and let the illegal result integer be handled normally.
1475 return;
1476 case ISD::FLOG2:
1477 if (SDValue Lowered = LowerFLOG2(Op: SDValue(N, 0), DAG))
1478 Results.push_back(Elt: Lowered);
1479 return;
1480 case ISD::FLOG:
1481 case ISD::FLOG10:
1482 if (SDValue Lowered = LowerFLOGCommon(Op: SDValue(N, 0), DAG))
1483 Results.push_back(Elt: Lowered);
1484 return;
1485 case ISD::FEXP2:
1486 if (SDValue Lowered = lowerFEXP2(Op: SDValue(N, 0), DAG))
1487 Results.push_back(Elt: Lowered);
1488 return;
1489 case ISD::FEXP:
1490 case ISD::FEXP10:
1491 if (SDValue Lowered = lowerFEXP(Op: SDValue(N, 0), DAG))
1492 Results.push_back(Elt: Lowered);
1493 return;
1494 case ISD::CTLZ:
1495 case ISD::CTLZ_ZERO_POISON:
1496 if (auto Lowered = lowerCTLZResults(Op: SDValue(N, 0u), DAG))
1497 Results.push_back(Elt: Lowered);
1498 return;
1499 default:
1500 return;
1501 }
1502}
1503
1504SDValue AMDGPUTargetLowering::LowerBlockAddress(SDValue Op,
1505 SelectionDAG &DAG) const {
1506 BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val&: Op);
1507 SDLoc SL(Op);
1508 EVT VT = Op.getValueType();
1509 return DAG.getTargetBlockAddress(BA: BA->getBlockAddress(), VT, Offset: BA->getOffset(),
1510 TargetFlags: BA->getTargetFlags());
1511}
1512
1513SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
1514 SDValue Op,
1515 SelectionDAG &DAG) const {
1516
1517 const DataLayout &DL = DAG.getDataLayout();
1518 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Val&: Op);
1519 const GlobalValue *GV = G->getGlobal();
1520
1521 if (G->getAddressSpace() == AMDGPUAS::BARRIER) {
1522 const GlobalVariable *GVar = cast<GlobalVariable>(Val: GV);
1523
1524 if (!AMDGPU::isNamedBarrier(GV: *GVar)) {
1525 const Function &Fn = DAG.getMachineFunction().getFunction();
1526 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1527 Fn, "unsupported use of BARRIER address space",
1528 SDLoc(Op).getDebugLoc(), DS_Error));
1529 return DAG.getPOISON(VT: Op.getValueType());
1530 }
1531
1532 unsigned Offset = MFI->allocateBarrierGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1533 return DAG.getConstant(Val: Offset, DL: SDLoc(Op), VT: Op.getValueType());
1534 }
1535
1536 if (!MFI->isModuleEntryFunction()) {
1537 if (std::optional<uint32_t> Address =
1538 AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(
1539 GV: *GV, AS: AMDGPUAS::LOCAL_ADDRESS)) {
1540 return DAG.getConstant(Val: *Address, DL: SDLoc(Op), VT: Op.getValueType());
1541 }
1542 }
1543
1544 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1545 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
1546 if (!MFI->isModuleEntryFunction() &&
1547 GV->getName() != "llvm.amdgcn.module.lds") {
1548 SDLoc DL(Op);
1549 const Function &Fn = DAG.getMachineFunction().getFunction();
1550 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1551 Fn, "local memory global used by non-kernel function",
1552 DL.getDebugLoc(), DS_Warning));
1553
1554 // We currently don't have a way to correctly allocate LDS objects that
1555 // aren't directly associated with a kernel. We do force inlining of
1556 // functions that use local objects. However, if these dead functions are
1557 // not eliminated, we don't want a compile time error. Just emit a warning
1558 // and a trap, since there should be no callable path here.
1559 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL, VT: MVT::Other, Operand: DAG.getEntryNode());
1560 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
1561 N1: Trap, N2: DAG.getRoot());
1562 DAG.setRoot(OutputChain);
1563 return DAG.getPOISON(VT: Op.getValueType());
1564 }
1565
1566 // TODO: We could emit code to handle the initialization somewhere.
1567 // We ignore the initializer for now and legalize it to allow selection.
1568 // The initializer will anyway get errored out during assembly emission.
1569 unsigned Offset = MFI->allocateLDSGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1570 // A constant byte offset (e.g. from a GEP into an array of named barriers)
1571 // folds directly into the allocated LDS address.
1572 return DAG.getConstant(Val: Offset + G->getOffset(), DL: SDLoc(Op),
1573 VT: Op.getValueType());
1574 }
1575 return SDValue();
1576}
1577
1578SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1579 SelectionDAG &DAG) const {
1580 SmallVector<SDValue, 8> Args;
1581 SDLoc SL(Op);
1582
1583 EVT VT = Op.getValueType();
1584 if (VT.getVectorElementType().getSizeInBits() < 32) {
1585 unsigned OpBitSize = Op.getOperand(i: 0).getValueType().getSizeInBits();
1586 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1587 unsigned NewNumElt = OpBitSize / 32;
1588 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1589 : EVT::getVectorVT(Context&: *DAG.getContext(),
1590 VT: MVT::i32, NumElements: NewNumElt);
1591 for (const SDUse &U : Op->ops()) {
1592 SDValue In = U.get();
1593 SDValue NewIn = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewEltVT, Operand: In);
1594 if (NewNumElt > 1)
1595 DAG.ExtractVectorElements(Op: NewIn, Args);
1596 else
1597 Args.push_back(Elt: NewIn);
1598 }
1599
1600 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
1601 NumElements: NewNumElt * Op.getNumOperands());
1602 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1603 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: BV);
1604 }
1605 }
1606
1607 for (const SDUse &U : Op->ops())
1608 DAG.ExtractVectorElements(Op: U.get(), Args);
1609
1610 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1611}
1612
1613SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1614 SelectionDAG &DAG) const {
1615 SDLoc SL(Op);
1616 SmallVector<SDValue, 8> Args;
1617 unsigned Start = Op.getConstantOperandVal(i: 1);
1618 EVT VT = Op.getValueType();
1619 EVT SrcVT = Op.getOperand(i: 0).getValueType();
1620
1621 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
1622 unsigned NumElt = VT.getVectorNumElements();
1623 unsigned NumSrcElt = SrcVT.getVectorNumElements();
1624 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types");
1625
1626 // Extract 32-bit registers at a time.
1627 EVT NewSrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumSrcElt / 2);
1628 EVT NewVT = NumElt == 2
1629 ? MVT::i32
1630 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElt / 2);
1631 SDValue Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewSrcVT, Operand: Op.getOperand(i: 0));
1632
1633 DAG.ExtractVectorElements(Op: Tmp, Args, Start: Start / 2, Count: NumElt / 2);
1634 if (NumElt == 2)
1635 Tmp = Args[0];
1636 else
1637 Tmp = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1638
1639 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Tmp);
1640 }
1641
1642 DAG.ExtractVectorElements(Op: Op.getOperand(i: 0), Args, Start,
1643 Count: VT.getVectorNumElements());
1644
1645 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1646}
1647
1648// TODO: Handle fabs too
1649static SDValue peekFNeg(SDValue Val) {
1650 if (Val.getOpcode() == ISD::FNEG)
1651 return Val.getOperand(i: 0);
1652
1653 return Val;
1654}
1655
1656// SelectionDAG twin of AMDGPUCombinerHelper::canIgnoreLegacyMinMaxTies.
1657static bool canIgnoreLegacyMinMaxTies(const SelectionDAG &DAG,
1658 SDNodeFlags Flags, SDValue LHS,
1659 SDValue RHS) {
1660 return Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
1661 DAG.isKnownNeverLogicalZero(Op: RHS);
1662}
1663
1664SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl(
1665 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1666 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1667 SelectionDAG &DAG = DCI.DAG;
1668 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
1669 assert(CCOpcode != ISD::SETCC_INVALID && "Invalid setcc condcode!");
1670
1671 switch (CCOpcode) {
1672 case ISD::SETOLE:
1673 case ISD::SETOLT:
1674 case ISD::SETLE:
1675 case ISD::SETLT:
1676 case ISD::SETOGE:
1677 case ISD::SETOGT:
1678 case ISD::SETGE:
1679 case ISD::SETGT:
1680 // Only do this after legalization to avoid interfering with other combines
1681 // which might occur.
1682 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1683 !DCI.isCalledByLegalizer())
1684 return SDValue();
1685 break;
1686 default:
1687 break;
1688 }
1689
1690 // Canonicalize so the select returns the compare's LHS on a true predicate.
1691 if (LHS != True)
1692 CCOpcode = ISD::getSetCCInverse(Operation: CCOpcode, Type: VT);
1693
1694 unsigned Opc;
1695 bool Swap; // Emit (rhs, lhs) instead of (lhs, rhs).
1696 switch (CCOpcode) {
1697 case ISD::SETOLT:
1698 case ISD::SETLT:
1699 case ISD::SETOLE:
1700 Opc = AMDGPUISD::FMIN_LEGACY;
1701 Swap = false;
1702 break;
1703 case ISD::SETULE:
1704 case ISD::SETLE:
1705 case ISD::SETULT:
1706 Opc = AMDGPUISD::FMIN_LEGACY;
1707 Swap = true;
1708 break;
1709 case ISD::SETOGE:
1710 case ISD::SETGE:
1711 case ISD::SETOGT:
1712 Opc = AMDGPUISD::FMAX_LEGACY;
1713 Swap = false;
1714 break;
1715 case ISD::SETUGT:
1716 case ISD::SETGT:
1717 case ISD::SETUGE:
1718 Opc = AMDGPUISD::FMAX_LEGACY;
1719 Swap = true;
1720 break;
1721 default:
1722 return SDValue();
1723 }
1724
1725 // For these predicates the NaN-correct operand order is the signed zero
1726 // tie-incorrect one, so the fold needs the tie to be unobservable.
1727 if ((CCOpcode == ISD::SETOLE || CCOpcode == ISD::SETULT ||
1728 CCOpcode == ISD::SETOGT || CCOpcode == ISD::SETUGE) &&
1729 !canIgnoreLegacyMinMaxTies(DAG, Flags, LHS, RHS))
1730 return SDValue();
1731
1732 if (Swap)
1733 std::swap(a&: LHS, b&: RHS);
1734 return DAG.getNode(Opcode: Opc, DL, VT, N1: LHS, N2: RHS, Flags);
1735}
1736
1737/// Generate Min/Max node
1738SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(
1739 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1740 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1741 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
1742 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, Flags,
1743 DCI);
1744
1745 SelectionDAG &DAG = DCI.DAG;
1746
1747 // If we can't directly match this, try to see if we can fold an fneg to
1748 // match.
1749
1750 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
1751 ConstantFPSDNode *CFalse = dyn_cast<ConstantFPSDNode>(Val&: False);
1752 SDValue NegTrue = peekFNeg(Val: True);
1753
1754 // Undo the combine foldFreeOpFromSelect does if it helps us match the
1755 // fmin/fmax.
1756 //
1757 // select (fcmp olt (lhs, K)), (fneg lhs), -K
1758 // -> fneg (fmin_legacy lhs, K)
1759 //
1760 // TODO: Use getNegatedExpression
1761 if (LHS == NegTrue && CFalse && CRHS) {
1762 APFloat NegRHS = neg(X: CRHS->getValueAPF());
1763 if (NegRHS == CFalse->getValueAPF()) {
1764 SDValue Combined = combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True: NegTrue,
1765 False, CC, Flags, DCI);
1766 if (Combined)
1767 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Combined);
1768 return SDValue();
1769 }
1770 }
1771
1772 return SDValue();
1773}
1774
1775std::pair<SDValue, SDValue>
1776AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1777 SDLoc SL(Op);
1778
1779 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1780
1781 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1782 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1783
1784 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1785 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1786
1787 return std::pair(Lo, Hi);
1788}
1789
1790SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1791 SDLoc SL(Op);
1792
1793 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1794 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1795 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1796}
1797
1798// Split a vector type into two parts. The first part is a power of two vector.
1799// The second part is whatever is left over, and is a scalar if it would
1800// otherwise be a 1-vector.
1801std::pair<EVT, EVT>
1802AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1803 EVT LoVT, HiVT;
1804 EVT EltVT = VT.getVectorElementType();
1805 unsigned NumElts = VT.getVectorNumElements();
1806 unsigned LoNumElts = PowerOf2Ceil(A: (NumElts + 1) / 2);
1807 LoVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: LoNumElts);
1808 HiVT = NumElts - LoNumElts == 1
1809 ? EltVT
1810 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts - LoNumElts);
1811 return std::pair(LoVT, HiVT);
1812}
1813
1814// Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1815// scalar.
1816std::pair<SDValue, SDValue>
1817AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1818 const EVT &LoVT, const EVT &HiVT,
1819 SelectionDAG &DAG) const {
1820 EVT VT = N.getValueType();
1821 assert(LoVT.getVectorNumElements() +
1822 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
1823 VT.getVectorNumElements() &&
1824 "More vector elements requested than available!");
1825 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N,
1826 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1827
1828 unsigned LoNumElts = LoVT.getVectorNumElements();
1829
1830 if (HiVT.isVector()) {
1831 unsigned HiNumElts = HiVT.getVectorNumElements();
1832 if ((VT.getVectorNumElements() % HiNumElts) == 0) {
1833 // Avoid creating an extract_subvector with an index that isn't a multiple
1834 // of the result type.
1835 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
1836 N2: DAG.getConstant(Val: LoNumElts, DL, VT: MVT::i32));
1837 return {Lo, Hi};
1838 }
1839
1840 SmallVector<SDValue, 8> Elts;
1841 DAG.ExtractVectorElements(Op: N, Args&: Elts, /*Start=*/LoNumElts,
1842 /*Count=*/HiNumElts);
1843 SDValue Hi = DAG.getBuildVector(VT: HiVT, DL, Ops: Elts);
1844 return {Lo, Hi};
1845 }
1846
1847 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: HiVT, N1: N,
1848 N2: DAG.getVectorIdxConstant(Val: LoNumElts, DL));
1849 return {Lo, Hi};
1850}
1851
1852SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1853 SelectionDAG &DAG) const {
1854 LoadSDNode *Load = cast<LoadSDNode>(Val: Op);
1855 EVT VT = Op.getValueType();
1856 SDLoc SL(Op);
1857
1858
1859 // If this is a 2 element vector, we really want to scalarize and not create
1860 // weird 1 element vectors.
1861 if (VT.getVectorNumElements() == 2) {
1862 SDValue Ops[2];
1863 std::tie(args&: Ops[0], args&: Ops[1]) = scalarizeVectorLoad(LD: Load, DAG);
1864 return DAG.getMergeValues(Ops, dl: SL);
1865 }
1866
1867 SDValue BasePtr = Load->getBasePtr();
1868 EVT MemVT = Load->getMemoryVT();
1869
1870 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1871
1872 EVT LoVT, HiVT;
1873 EVT LoMemVT, HiMemVT;
1874 SDValue Lo, Hi;
1875
1876 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1877 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1878 std::tie(args&: Lo, args&: Hi) = splitVector(N: Op, DL: SL, LoVT, HiVT, DAG);
1879
1880 unsigned Size = LoMemVT.getStoreSize();
1881 Align BaseAlign = Load->getAlign();
1882 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1883
1884 SDValue LoLoad = DAG.getExtLoad(
1885 ExtType: Load->getExtensionType(), dl: SL, VT: LoVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1886 MemVT: LoMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1887 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Size));
1888 SDValue HiLoad = DAG.getExtLoad(
1889 ExtType: Load->getExtensionType(), dl: SL, VT: HiVT, Chain: Load->getChain(), Ptr: HiPtr,
1890 PtrInfo: SrcValue.getWithOffset(O: LoMemVT.getStoreSize()), MemVT: HiMemVT, Alignment: HiAlign,
1891 MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1892
1893 SDValue Join;
1894 if (LoVT == HiVT) {
1895 // This is the case that the vector is power of two so was evenly split.
1896 Join = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, N1: LoLoad, N2: HiLoad);
1897 } else {
1898 Join = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SL, VT, N1: DAG.getPOISON(VT), N2: LoLoad,
1899 N3: DAG.getVectorIdxConstant(Val: 0, DL: SL));
1900 Join = DAG.getNode(
1901 Opcode: HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, DL: SL,
1902 VT, N1: Join, N2: HiLoad,
1903 N3: DAG.getVectorIdxConstant(Val: LoVT.getVectorNumElements(), DL: SL));
1904 }
1905
1906 SDValue Ops[] = {Join, DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
1907 N1: LoLoad.getValue(R: 1), N2: HiLoad.getValue(R: 1))};
1908
1909 return DAG.getMergeValues(Ops, dl: SL);
1910}
1911
1912SDValue AMDGPUTargetLowering::WidenOrSplitVectorLoad(SDValue Op,
1913 SelectionDAG &DAG) const {
1914 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
1915 EVT VT = Op.getValueType();
1916 SDValue BasePtr = Load->getBasePtr();
1917 EVT MemVT = Load->getMemoryVT();
1918 SDLoc SL(Op);
1919 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1920 Align BaseAlign = Load->getAlign();
1921 unsigned NumElements = MemVT.getVectorNumElements();
1922
1923 // Widen from vec3 to vec4 when the load is at least 8-byte aligned
1924 // or 16-byte fully dereferenceable. Otherwise, split the vector load.
1925 if (NumElements != 3 ||
1926 (BaseAlign < Align(8) &&
1927 !SrcValue.isDereferenceable(Size: 16, C&: *DAG.getContext(), DL: DAG.getDataLayout())))
1928 return SplitVectorLoad(Op, DAG);
1929
1930 assert(NumElements == 3);
1931
1932 EVT WideVT =
1933 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
1934 EVT WideMemVT =
1935 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(), NumElements: 4);
1936 SDValue WideLoad = DAG.getExtLoad(
1937 ExtType: Load->getExtensionType(), dl: SL, VT: WideVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1938 MemVT: WideMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags());
1939 return DAG.getMergeValues(
1940 Ops: {DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT, N1: WideLoad,
1941 N2: DAG.getVectorIdxConstant(Val: 0, DL: SL)),
1942 WideLoad.getValue(R: 1)},
1943 dl: SL);
1944}
1945
1946SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1947 SelectionDAG &DAG) const {
1948 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
1949 SDValue Val = Store->getValue();
1950 EVT VT = Val.getValueType();
1951
1952 // If this is a 2 element vector, we really want to scalarize and not create
1953 // weird 1 element vectors.
1954 if (VT.getVectorNumElements() == 2)
1955 return scalarizeVectorStore(ST: Store, DAG);
1956
1957 EVT MemVT = Store->getMemoryVT();
1958 SDValue Chain = Store->getChain();
1959 SDValue BasePtr = Store->getBasePtr();
1960 SDLoc SL(Op);
1961
1962 EVT LoVT, HiVT;
1963 EVT LoMemVT, HiMemVT;
1964 SDValue Lo, Hi;
1965
1966 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1967 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1968 std::tie(args&: Lo, args&: Hi) = splitVector(N: Val, DL: SL, LoVT, HiVT, DAG);
1969
1970 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: LoMemVT.getStoreSize());
1971
1972 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
1973 Align BaseAlign = Store->getAlign();
1974 unsigned Size = LoMemVT.getStoreSize();
1975 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1976
1977 SDValue LoStore =
1978 DAG.getTruncStore(Chain, dl: SL, Val: Lo, Ptr: BasePtr, PtrInfo: SrcValue, SVT: LoMemVT, Alignment: BaseAlign,
1979 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
1980 SDValue HiStore = DAG.getTruncStore(
1981 Chain, dl: SL, Val: Hi, Ptr: HiPtr, PtrInfo: SrcValue.getWithOffset(O: Size), SVT: HiMemVT, Alignment: HiAlign,
1982 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
1983
1984 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: LoStore, N2: HiStore);
1985}
1986
1987// This is a shortcut for integer division because we have fast i32<->f32
1988// conversions, and fast f32 reciprocal instructions.
1989SDValue AMDGPUTargetLowering::LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG,
1990 bool Sign) const {
1991 SDLoc DL(Op);
1992 EVT VT = Op.getValueType();
1993 assert(VT == MVT::i32 && "LowerDIVREMToFloat expects an i32");
1994
1995 SDValue LHS = Op.getOperand(i: 0);
1996 SDValue RHS = Op.getOperand(i: 1);
1997 MVT IntVT = MVT::i32;
1998 MVT FltVT = MVT::f32;
1999
2000 unsigned LHSSignBits;
2001 unsigned RHSSignBits;
2002 if (Sign) {
2003 LHSSignBits = DAG.ComputeNumSignBits(Op: LHS);
2004 RHSSignBits = DAG.ComputeNumSignBits(Op: RHS);
2005 if (LHSSignBits < 9 || RHSSignBits < 9)
2006 return SDValue();
2007 } else {
2008 KnownBits LHSKnown = DAG.computeKnownBits(Op: LHS);
2009 KnownBits RHSKnown = DAG.computeKnownBits(Op: RHS);
2010
2011 LHSSignBits = LHSKnown.countMinLeadingZeros();
2012 RHSSignBits = RHSKnown.countMinLeadingZeros();
2013 }
2014
2015 unsigned BitSize = VT.getSizeInBits();
2016 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
2017 unsigned DivBits = BitSize - SignBits;
2018 if (Sign)
2019 ++DivBits;
2020
2021 // In order to avoid problems due to 1 ulp accuracy issues with v_rcp_f32,
2022 // limit LowerDIVREMToFloat to:
2023 // [-0x400000,0x3FFFFF] for Sign
2024 // [ 0x000000,0x3FFFFF] for !Sign
2025 // This matches what is done in expandDivRemToFloatImpl.
2026 if (DivBits > (Sign ? 23 : 22))
2027 return SDValue();
2028
2029 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
2030 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
2031
2032 // int ia = (int)LHS;
2033 SDValue ia = LHS;
2034
2035 // int ib, (int)RHS;
2036 SDValue ib = RHS;
2037
2038 // The calculation:
2039 // fq = fa*recip(fb)
2040 // may be too small due to the 1ulp accuracy in the recip
2041 // operation and rounding issues. Since fq is truncated to produce
2042 // an integer value it may be too small by one. This is
2043 // dealt with by incrementing fa by 1ulp:
2044 // fq = (fa+1ulp)*recip(fb)
2045 // This will increase fa's magnitude by at most 0.5
2046 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
2047 // Thus, this method is safe since fa must be incremented by at least 1.0
2048 // for the quotient to increase by one.
2049 SDValue fa = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ia);
2050 SDValue faAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: fa);
2051 SDValue faIncremented = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: faAsInt,
2052 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
2053 fa = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FltVT, Operand: faIncremented);
2054
2055 // float fb = (float)ib;
2056 SDValue fb = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ib);
2057
2058 SDValue fq = DAG.getNode(Opcode: ISD::FMUL, DL, VT: FltVT,
2059 N1: fa, N2: DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: FltVT, Operand: fb));
2060
2061 // fq = trunc(fq);
2062 fq = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: FltVT, Operand: fq);
2063
2064 // int iq = (int)fq;
2065 SDValue Div = DAG.getNode(Opcode: ToInt, DL, VT: IntVT, Operand: fq);
2066
2067 // Rem needs compensation, it's easier to recompute it
2068 SDValue Rem = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Div, N2: RHS);
2069 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Rem);
2070
2071 return DAG.getMergeValues(Ops: { Div, Rem }, dl: DL);
2072}
2073
2074void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
2075 SelectionDAG &DAG,
2076 SmallVectorImpl<SDValue> &Results) const {
2077 SDLoc DL(Op);
2078 EVT VT = Op.getValueType();
2079
2080 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
2081
2082 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2083
2084 SDValue One = DAG.getConstant(Val: 1, DL, VT: HalfVT);
2085 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HalfVT);
2086
2087 //HiLo split
2088 SDValue LHS_Lo, LHS_Hi;
2089 SDValue LHS = Op.getOperand(i: 0);
2090 std::tie(args&: LHS_Lo, args&: LHS_Hi) = DAG.SplitScalar(N: LHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2091
2092 SDValue RHS_Lo, RHS_Hi;
2093 SDValue RHS = Op.getOperand(i: 1);
2094 std::tie(args&: RHS_Lo, args&: RHS_Hi) = DAG.SplitScalar(N: RHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2095
2096 if (DAG.MaskedValueIsZero(Op: RHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32)) &&
2097 DAG.MaskedValueIsZero(Op: LHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32))) {
2098
2099 SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2100 N1: LHS_Lo, N2: RHS_Lo);
2101
2102 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 0), Zero});
2103 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 1), Zero});
2104
2105 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV));
2106 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM));
2107 return;
2108 }
2109
2110 if (isTypeLegal(VT: MVT::i64)) {
2111 // The algorithm here is based on ideas from "Software Integer Division",
2112 // Tom Rodeheffer, August 2008.
2113
2114 MachineFunction &MF = DAG.getMachineFunction();
2115 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2116
2117 // Compute denominator reciprocal.
2118 unsigned FMAD =
2119 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA
2120 : MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign()
2121 ? (unsigned)ISD::FMAD
2122 : (unsigned)AMDGPUISD::FMAD_FTZ;
2123
2124 SDValue Cvt_Lo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Lo);
2125 SDValue Cvt_Hi = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Hi);
2126 SDValue Mad1 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Cvt_Hi,
2127 N2: DAG.getConstantFP(Val: APInt(32, 0x4f800000).bitsToFloat(), DL, VT: MVT::f32),
2128 N3: Cvt_Lo);
2129 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: MVT::f32, Operand: Mad1);
2130 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Rcp,
2131 N2: DAG.getConstantFP(Val: APInt(32, 0x5f7ffffc).bitsToFloat(), DL, VT: MVT::f32));
2132 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Mul1,
2133 N2: DAG.getConstantFP(Val: APInt(32, 0x2f800000).bitsToFloat(), DL, VT: MVT::f32));
2134 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: MVT::f32, Operand: Mul2);
2135 SDValue Mad2 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Trunc,
2136 N2: DAG.getConstantFP(Val: APInt(32, 0xcf800000).bitsToFloat(), DL, VT: MVT::f32),
2137 N3: Mul1);
2138 SDValue Rcp_Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Mad2);
2139 SDValue Rcp_Hi = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Trunc);
2140 SDValue Rcp64 = DAG.getBitcast(VT,
2141 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Rcp_Lo, Rcp_Hi}));
2142
2143 SDValue Zero64 = DAG.getConstant(Val: 0, DL, VT);
2144 SDValue One64 = DAG.getConstant(Val: 1, DL, VT);
2145 SDValue Zero1 = DAG.getConstant(Val: 0, DL, VT: MVT::i1);
2146 SDVTList HalfCarryVT = DAG.getVTList(VT1: HalfVT, VT2: MVT::i1);
2147
2148 // First round of UNR (Unsigned integer Newton-Raphson).
2149 SDValue Neg_RHS = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero64, N2: RHS);
2150 SDValue Mullo1 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Rcp64);
2151 SDValue Mulhi1 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Rcp64, N2: Mullo1);
2152 SDValue Mulhi1_Lo, Mulhi1_Hi;
2153 std::tie(args&: Mulhi1_Lo, args&: Mulhi1_Hi) =
2154 DAG.SplitScalar(N: Mulhi1, DL, LoVT: HalfVT, HiVT: HalfVT);
2155 SDValue Add1_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Lo,
2156 N2: Mulhi1_Lo, N3: Zero1);
2157 SDValue Add1_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Hi,
2158 N2: Mulhi1_Hi, N3: Add1_Lo.getValue(R: 1));
2159 SDValue Add1 = DAG.getBitcast(VT,
2160 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add1_Lo, Add1_Hi}));
2161
2162 // Second round of UNR.
2163 SDValue Mullo2 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Add1);
2164 SDValue Mulhi2 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Add1, N2: Mullo2);
2165 SDValue Mulhi2_Lo, Mulhi2_Hi;
2166 std::tie(args&: Mulhi2_Lo, args&: Mulhi2_Hi) =
2167 DAG.SplitScalar(N: Mulhi2, DL, LoVT: HalfVT, HiVT: HalfVT);
2168 SDValue Add2_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Lo,
2169 N2: Mulhi2_Lo, N3: Zero1);
2170 SDValue Add2_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Hi,
2171 N2: Mulhi2_Hi, N3: Add2_Lo.getValue(R: 1));
2172 SDValue Add2 = DAG.getBitcast(VT,
2173 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add2_Lo, Add2_Hi}));
2174
2175 SDValue Mulhi3 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: LHS, N2: Add2);
2176
2177 SDValue Mul3 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: RHS, N2: Mulhi3);
2178
2179 SDValue Mul3_Lo, Mul3_Hi;
2180 std::tie(args&: Mul3_Lo, args&: Mul3_Hi) = DAG.SplitScalar(N: Mul3, DL, LoVT: HalfVT, HiVT: HalfVT);
2181 SDValue Sub1_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Lo,
2182 N2: Mul3_Lo, N3: Zero1);
2183 SDValue Sub1_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Hi,
2184 N2: Mul3_Hi, N3: Sub1_Lo.getValue(R: 1));
2185 SDValue Sub1_Mi = DAG.getNode(Opcode: ISD::SUB, DL, VT: HalfVT, N1: LHS_Hi, N2: Mul3_Hi);
2186 SDValue Sub1 = DAG.getBitcast(VT,
2187 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub1_Lo, Sub1_Hi}));
2188
2189 SDValue MinusOne = DAG.getConstant(Val: 0xffffffffu, DL, VT: HalfVT);
2190 SDValue C1 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2191 Cond: ISD::SETUGE);
2192 SDValue C2 = DAG.getSelectCC(DL, LHS: Sub1_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2193 Cond: ISD::SETUGE);
2194 SDValue C3 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: C2, False: C1, Cond: ISD::SETEQ);
2195
2196 // TODO: Here and below portions of the code can be enclosed into if/endif.
2197 // Currently control flow is unconditional and we have 4 selects after
2198 // potential endif to substitute PHIs.
2199
2200 // if C3 != 0 ...
2201 SDValue Sub2_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Lo,
2202 N2: RHS_Lo, N3: Zero1);
2203 SDValue Sub2_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Mi,
2204 N2: RHS_Hi, N3: Sub1_Lo.getValue(R: 1));
2205 SDValue Sub2_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2206 N2: Zero, N3: Sub2_Lo.getValue(R: 1));
2207 SDValue Sub2 = DAG.getBitcast(VT,
2208 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub2_Lo, Sub2_Hi}));
2209
2210 SDValue Add3 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mulhi3, N2: One64);
2211
2212 SDValue C4 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2213 Cond: ISD::SETUGE);
2214 SDValue C5 = DAG.getSelectCC(DL, LHS: Sub2_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2215 Cond: ISD::SETUGE);
2216 SDValue C6 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: C5, False: C4, Cond: ISD::SETEQ);
2217
2218 // if (C6 != 0)
2219 SDValue Add4 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Add3, N2: One64);
2220
2221 SDValue Sub3_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Lo,
2222 N2: RHS_Lo, N3: Zero1);
2223 SDValue Sub3_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2224 N2: RHS_Hi, N3: Sub2_Lo.getValue(R: 1));
2225 SDValue Sub3_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub3_Mi,
2226 N2: Zero, N3: Sub3_Lo.getValue(R: 1));
2227 SDValue Sub3 = DAG.getBitcast(VT,
2228 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub3_Lo, Sub3_Hi}));
2229
2230 // endif C6
2231 // endif C3
2232
2233 SDValue Sel1 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Add4, False: Add3, Cond: ISD::SETNE);
2234 SDValue Div = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel1, False: Mulhi3, Cond: ISD::SETNE);
2235
2236 SDValue Sel2 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Sub3, False: Sub2, Cond: ISD::SETNE);
2237 SDValue Rem = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel2, False: Sub1, Cond: ISD::SETNE);
2238
2239 Results.push_back(Elt: Div);
2240 Results.push_back(Elt: Rem);
2241
2242 return;
2243 }
2244
2245 // r600 expandion.
2246 // Get Speculative values
2247 SDValue DIV_Part = DAG.getNode(Opcode: ISD::UDIV, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2248 SDValue REM_Part = DAG.getNode(Opcode: ISD::UREM, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2249
2250 SDValue REM_Lo = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: REM_Part, False: LHS_Hi, Cond: ISD::SETEQ);
2251 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {REM_Lo, Zero});
2252 REM = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM);
2253
2254 SDValue DIV_Hi = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: DIV_Part, False: Zero, Cond: ISD::SETEQ);
2255 SDValue DIV_Lo = Zero;
2256
2257 const unsigned halfBitWidth = HalfVT.getSizeInBits();
2258
2259 for (unsigned i = 0; i < halfBitWidth; ++i) {
2260 const unsigned bitPos = halfBitWidth - i - 1;
2261 SDValue POS = DAG.getConstant(Val: bitPos, DL, VT: HalfVT);
2262 // Get value of high bit
2263 SDValue HBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: HalfVT, N1: LHS_Lo, N2: POS);
2264 HBit = DAG.getNode(Opcode: ISD::AND, DL, VT: HalfVT, N1: HBit, N2: One);
2265 HBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: HBit);
2266
2267 // Shift
2268 REM = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: REM, N2: DAG.getConstant(Val: 1, DL, VT));
2269 // Add LHS high bit
2270 REM = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: REM, N2: HBit);
2271
2272 SDValue BIT = DAG.getConstant(Val: 1ULL << bitPos, DL, VT: HalfVT);
2273 SDValue realBIT = DAG.getSelectCC(DL, LHS: REM, RHS, True: BIT, False: Zero, Cond: ISD::SETUGE);
2274
2275 DIV_Lo = DAG.getNode(Opcode: ISD::OR, DL, VT: HalfVT, N1: DIV_Lo, N2: realBIT);
2276
2277 // Update REM
2278 SDValue REM_sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: REM, N2: RHS);
2279 REM = DAG.getSelectCC(DL, LHS: REM, RHS, True: REM_sub, False: REM, Cond: ISD::SETUGE);
2280 }
2281
2282 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {DIV_Lo, DIV_Hi});
2283 DIV = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV);
2284 Results.push_back(Elt: DIV);
2285 Results.push_back(Elt: REM);
2286}
2287
2288SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
2289 SelectionDAG &DAG) const {
2290 SDLoc DL(Op);
2291 EVT VT = Op.getValueType();
2292
2293 if (VT == MVT::i64) {
2294 SmallVector<SDValue, 2> Results;
2295 LowerUDIVREM64(Op, DAG, Results);
2296 return DAG.getMergeValues(Ops: Results, dl: DL);
2297 }
2298
2299 if (VT == MVT::i32) {
2300 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: false))
2301 return Res;
2302 }
2303
2304 SDValue X = Op.getOperand(i: 0);
2305 SDValue Y = Op.getOperand(i: 1);
2306
2307 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
2308 // algorithm used here.
2309
2310 // Initial estimate of inv(y).
2311 SDValue Z = DAG.getNode(Opcode: AMDGPUISD::URECIP, DL, VT, Operand: Y);
2312
2313 // One round of UNR.
2314 SDValue NegY = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Y);
2315 SDValue NegYZ = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: NegY, N2: Z);
2316 Z = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Z,
2317 N2: DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Z, N2: NegYZ));
2318
2319 // Quotient/remainder estimate.
2320 SDValue Q = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: X, N2: Z);
2321 SDValue R =
2322 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: X, N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Q, N2: Y));
2323
2324 // First quotient/remainder refinement.
2325 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2326 SDValue One = DAG.getConstant(Val: 1, DL, VT);
2327 SDValue Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2328 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2329 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2330 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2331 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2332
2333 // Second quotient/remainder refinement.
2334 Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2335 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2336 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2337 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2338 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2339
2340 return DAG.getMergeValues(Ops: {Q, R}, dl: DL);
2341}
2342
2343SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
2344 SelectionDAG &DAG) const {
2345 SDLoc DL(Op);
2346 EVT VT = Op.getValueType();
2347
2348 SDValue LHS = Op.getOperand(i: 0);
2349 SDValue RHS = Op.getOperand(i: 1);
2350
2351 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
2352 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2353
2354 if (VT == MVT::i32) {
2355 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: true))
2356 return Res;
2357 }
2358
2359 // LHS must have > 33 sign-bits to ensure that LHS != -2147483648
2360 // Otherwise 32-bit division cannot be used safely.
2361 // -2147483648/1 and -2147483648/-1 are not equal,
2362 // but they produce the same lower 32-bit result.
2363 if (VT == MVT::i64 && DAG.ComputeNumSignBits(Op: LHS) > 33 &&
2364 DAG.ComputeNumSignBits(Op: RHS) > 32) {
2365 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2366
2367 //HiLo split
2368 SDValue LHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: LHS, N2: Zero);
2369 SDValue RHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: RHS, N2: Zero);
2370 SDValue DIVREM = DAG.getNode(Opcode: ISD::SDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2371 N1: LHS_Lo, N2: RHS_Lo);
2372 SDValue Res[2] = {
2373 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 0)),
2374 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 1))
2375 };
2376 return DAG.getMergeValues(Ops: Res, dl: DL);
2377 }
2378
2379 SDValue LHSign = DAG.getSelectCC(DL, LHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2380 SDValue RHSign = DAG.getSelectCC(DL, LHS: RHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2381 SDValue DSign = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHSign, N2: RHSign);
2382 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2383
2384 LHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: LHSign);
2385 RHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: RHS, N2: RHSign);
2386
2387 LHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHS, N2: LHSign);
2388 RHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: RHS, N2: RHSign);
2389
2390 SDValue Div = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
2391 SDValue Rem = Div.getValue(R: 1);
2392
2393 Div = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Div, N2: DSign);
2394 Rem = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Rem, N2: RSign);
2395
2396 Div = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Div, N2: DSign);
2397 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Rem, N2: RSign);
2398
2399 SDValue Res[2] = {
2400 Div,
2401 Rem
2402 };
2403 return DAG.getMergeValues(Ops: Res, dl: DL);
2404}
2405
2406SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2407 SDLoc SL(Op);
2408 SDValue Src = Op.getOperand(i: 0);
2409
2410 // result = trunc(src)
2411 // if (src > 0.0 && src != result)
2412 // result += 1.0
2413
2414 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2415
2416 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2417 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
2418
2419 EVT SetCCVT =
2420 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2421
2422 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOGT);
2423 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2424 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2425
2426 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: One, N3: Zero);
2427 // TODO: Should this propagate fast-math-flags?
2428 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2429}
2430
2431static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2432 SelectionDAG &DAG) {
2433 const unsigned FractBits = 52;
2434 const unsigned ExpBits = 11;
2435
2436 SDValue ExpPart = DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32,
2437 N1: Hi,
2438 N2: DAG.getConstant(Val: FractBits - 32, DL: SL, VT: MVT::i32),
2439 N3: DAG.getConstant(Val: ExpBits, DL: SL, VT: MVT::i32));
2440 SDValue Exp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ExpPart,
2441 N2: DAG.getConstant(Val: 1023, DL: SL, VT: MVT::i32));
2442
2443 return Exp;
2444}
2445
2446SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2447 SDLoc SL(Op);
2448 SDValue Src = Op.getOperand(i: 0);
2449
2450 assert(Op.getValueType() == MVT::f64);
2451
2452 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
2453
2454 // Extract the upper half, since this is where we will find the sign and
2455 // exponent.
2456 SDValue Hi = getHiHalf64(Op: Src, DAG);
2457
2458 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2459
2460 const unsigned FractBits = 52;
2461
2462 // Extract the sign bit.
2463 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, DL: SL, VT: MVT::i32);
2464 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Hi, N2: SignBitMask);
2465
2466 // Extend back to 64-bits.
2467 SDValue SignBit64 = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Zero, SignBit});
2468 SignBit64 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: SignBit64);
2469
2470 SDValue BcInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Src);
2471 const SDValue FractMask
2472 = DAG.getConstant(Val: (UINT64_C(1) << FractBits) - 1, DL: SL, VT: MVT::i64);
2473
2474 SDValue Shr = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: FractMask, N2: Exp);
2475 SDValue Not = DAG.getNOT(DL: SL, Val: Shr, VT: MVT::i64);
2476 SDValue Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i64, N1: BcInt, N2: Not);
2477
2478 EVT SetCCVT =
2479 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i32);
2480
2481 const SDValue FiftyOne = DAG.getConstant(Val: FractBits - 1, DL: SL, VT: MVT::i32);
2482
2483 SDValue ExpLt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: Zero, Cond: ISD::SETLT);
2484 SDValue ExpGt51 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: FiftyOne, Cond: ISD::SETGT);
2485
2486 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpLt0, N2: SignBit64, N3: Tmp0);
2487 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpGt51, N2: BcInt, N3: Tmp1);
2488
2489 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f64, Operand: Tmp2);
2490}
2491
2492SDValue AMDGPUTargetLowering::LowerFROUNDEVEN(SDValue Op,
2493 SelectionDAG &DAG) const {
2494 SDLoc SL(Op);
2495 SDValue Src = Op.getOperand(i: 0);
2496
2497 assert(Op.getValueType() == MVT::f64);
2498
2499 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2500 SDValue C1 = DAG.getConstantFP(Val: C1Val, DL: SL, VT: MVT::f64);
2501 SDValue CopySign = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MVT::f64, N1: C1, N2: Src);
2502
2503 // TODO: Should this propagate fast-math-flags?
2504
2505 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Src, N2: CopySign);
2506 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT: MVT::f64, N1: Tmp1, N2: CopySign);
2507
2508 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f64, Operand: Src);
2509
2510 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2511 SDValue C2 = DAG.getConstantFP(Val: C2Val, DL: SL, VT: MVT::f64);
2512
2513 EVT SetCCVT =
2514 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2515 SDValue Cond = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Fabs, RHS: C2, Cond: ISD::SETOGT);
2516
2517 return DAG.getSelect(DL: SL, VT: MVT::f64, Cond, LHS: Src, RHS: Tmp2);
2518}
2519
2520SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op,
2521 SelectionDAG &DAG) const {
2522 // FNEARBYINT and FRINT are the same, except in their handling of FP
2523 // exceptions. Those aren't really meaningful for us, and OpenCL only has
2524 // rint, so just treat them as equivalent.
2525 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT: Op.getValueType(),
2526 Operand: Op.getOperand(i: 0));
2527}
2528
2529SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2530 auto VT = Op.getValueType();
2531 auto Arg = Op.getOperand(i: 0u);
2532 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT, Operand: Arg);
2533}
2534
2535// XXX - May require not supporting f32 denormals?
2536
2537// Don't handle v2f16. The extra instructions to scalarize and repack around the
2538// compare and vselect end up producing worse code than scalarizing the whole
2539// operation.
2540SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2541 SDLoc SL(Op);
2542 SDValue X = Op.getOperand(i: 0);
2543 EVT VT = Op.getValueType();
2544
2545 SDValue T = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: X);
2546
2547 // TODO: Should this propagate fast-math-flags?
2548
2549 SDValue Diff = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: T);
2550
2551 SDValue AbsDiff = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Diff);
2552
2553 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2554 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2555
2556 EVT SetCCVT =
2557 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2558
2559 const SDValue Half = DAG.getConstantFP(Val: 0.5, DL: SL, VT);
2560 SDValue Cmp = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsDiff, RHS: Half, Cond: ISD::SETOGE);
2561 SDValue OneOrZeroFP = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cmp, N2: One, N3: Zero);
2562
2563 SDValue SignedOffset = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: OneOrZeroFP, N2: X);
2564 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: T, N2: SignedOffset);
2565}
2566
2567SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2568 SDLoc SL(Op);
2569 SDValue Src = Op.getOperand(i: 0);
2570
2571 // result = trunc(src);
2572 // if (src < 0.0 && src != result)
2573 // result += -1.0.
2574
2575 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2576
2577 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2578 const SDValue NegOne = DAG.getConstantFP(Val: -1.0, DL: SL, VT: MVT::f64);
2579
2580 EVT SetCCVT =
2581 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2582
2583 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOLT);
2584 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2585 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2586
2587 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: NegOne, N3: Zero);
2588 // TODO: Should this propagate fast-math-flags?
2589 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2590}
2591
2592/// Return true if it's known that \p Src can never be an f32 denormal value.
2593static bool valueIsKnownNeverF32Denorm(SDValue Src) {
2594 switch (Src.getOpcode()) {
2595 case ISD::FP_EXTEND:
2596 return Src.getOperand(i: 0).getValueType() == MVT::f16;
2597 case ISD::FP16_TO_FP:
2598 case ISD::FFREXP:
2599 case ISD::FSQRT:
2600 case AMDGPUISD::LOG:
2601 case AMDGPUISD::EXP:
2602 return true;
2603 case ISD::INTRINSIC_WO_CHAIN: {
2604 unsigned IntrinsicID = Src.getConstantOperandVal(i: 0);
2605 switch (IntrinsicID) {
2606 case Intrinsic::amdgcn_frexp_mant:
2607 case Intrinsic::amdgcn_log:
2608 case Intrinsic::amdgcn_log_clamp:
2609 case Intrinsic::amdgcn_exp2:
2610 case Intrinsic::amdgcn_sqrt:
2611 return true;
2612 default:
2613 return false;
2614 }
2615 }
2616 default:
2617 return false;
2618 }
2619
2620 llvm_unreachable("covered opcode switch");
2621}
2622
2623bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG,
2624 SDNodeFlags Flags) {
2625 return Flags.hasApproximateFuncs();
2626}
2627
2628bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG,
2629 SDValue Src,
2630 SDNodeFlags Flags) {
2631 return !valueIsKnownNeverF32Denorm(Src) &&
2632 DAG.getMachineFunction()
2633 .getDenormalMode(FPType: APFloat::IEEEsingle())
2634 .Input != DenormalMode::PreserveSign;
2635}
2636
2637SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src,
2638 SDNodeFlags Flags) const {
2639 SDLoc SL(Src);
2640 EVT VT = Src.getValueType();
2641 const fltSemantics &Semantics = VT.getFltSemantics();
2642 SDValue Inf = DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL: SL, VT);
2643
2644 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Src, Flags);
2645 SDValue IsFinite = DAG.getSetCC(
2646 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Fabs,
2647 RHS: Inf, Cond: ISD::SETOLT);
2648 return IsFinite;
2649}
2650
2651/// If denormal handling is required return the scaled input to FLOG2, and the
2652/// check for denormal range. Otherwise, return null values.
2653std::pair<SDValue, SDValue>
2654AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL,
2655 SDValue Src, SDNodeFlags Flags) const {
2656 if (!needsDenormHandlingF32(DAG, Src, Flags))
2657 return {};
2658
2659 MVT VT = MVT::f32;
2660 const fltSemantics &Semantics = APFloat::IEEEsingle();
2661 SDValue SmallestNormal =
2662 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2663
2664 SDValue IsLtSmallestNormal = DAG.getSetCC(
2665 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2666 RHS: SmallestNormal, Cond: ISD::SETOLT);
2667
2668 SDValue Scale32 = DAG.getConstantFP(Val: 0x1.0p+32, DL: SL, VT);
2669 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2670 SDValue ScaleFactor =
2671 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: Scale32, N3: One, Flags);
2672
2673 SDValue ScaledInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Src, N2: ScaleFactor, Flags);
2674 return {ScaledInput, IsLtSmallestNormal};
2675}
2676
2677SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const {
2678 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
2679 // If we have to handle denormals, scale up the input and adjust the result.
2680
2681 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
2682 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
2683
2684 SDLoc SL(Op);
2685 EVT VT = Op.getValueType();
2686 SDValue Src = Op.getOperand(i: 0);
2687 SDNodeFlags Flags = Op->getFlags();
2688
2689 if (VT == MVT::f16) {
2690 // Nothing in half is a denormal when promoted to f32.
2691 assert(!isTypeLegal(VT));
2692 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
2693 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
2694 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
2695 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
2696 }
2697
2698 auto [ScaledInput, IsLtSmallestNormal] =
2699 getScaledLogInput(DAG, SL, Src, Flags);
2700 if (!ScaledInput)
2701 return DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: Src, Flags);
2702
2703 SDValue Log2 = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2704
2705 SDValue ThirtyTwo = DAG.getConstantFP(Val: 32.0, DL: SL, VT);
2706 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2707 SDValue ResultOffset =
2708 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: ThirtyTwo, N3: Zero);
2709 return DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: Log2, N2: ResultOffset, Flags);
2710}
2711
2712static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2713 SDValue Y, SDValue C, SDNodeFlags Flags = SDNodeFlags()) {
2714 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Y, Flags);
2715 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: C, Flags);
2716}
2717
2718SDValue AMDGPUTargetLowering::LowerFLOGCommon(SDValue Op,
2719 SelectionDAG &DAG) const {
2720 SDValue X = Op.getOperand(i: 0);
2721 EVT VT = Op.getValueType();
2722 SDNodeFlags Flags = Op->getFlags();
2723 SDLoc DL(Op);
2724 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10;
2725 assert(IsLog10 || Op.getOpcode() == ISD::FLOG);
2726
2727 if (VT == MVT::f16 || Flags.hasApproximateFuncs()) {
2728 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
2729 // depending on !fpmath metadata.
2730
2731 bool PromoteToF32 = VT == MVT::f16 && (!Flags.hasApproximateFuncs() ||
2732 !isTypeLegal(VT: MVT::f16));
2733
2734 if (PromoteToF32) {
2735 // Log and multiply in f32 is always good enough for f16.
2736 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2737 }
2738
2739 SDValue Lowered = LowerFLOGUnsafe(Op: X, SL: DL, DAG, IsLog10, Flags);
2740 if (PromoteToF32) {
2741 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Lowered,
2742 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32), Flags);
2743 }
2744
2745 return Lowered;
2746 }
2747
2748 SDValue ScaledInput, IsScaled;
2749 if (VT == MVT::f16)
2750 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2751 else {
2752 std::tie(args&: ScaledInput, args&: IsScaled) = getScaledLogInput(DAG, SL: DL, Src: X, Flags);
2753 if (ScaledInput)
2754 X = ScaledInput;
2755 }
2756
2757 SDValue Y = DAG.getNode(Opcode: AMDGPUISD::LOG, DL, VT, Operand: X, Flags);
2758
2759 SDValue R;
2760 if (Subtarget->hasFastFMAF32()) {
2761 // c+cc are ln(2)/ln(10) to more than 49 bits
2762 const float c_log10 = 0x1.344134p-2f;
2763 const float cc_log10 = 0x1.09f79ep-26f;
2764
2765 // c + cc is ln(2) to more than 49 bits
2766 const float c_log = 0x1.62e42ep-1f;
2767 const float cc_log = 0x1.efa39ep-25f;
2768
2769 SDValue C = DAG.getConstantFP(Val: IsLog10 ? c_log10 : c_log, DL, VT);
2770 SDValue CC = DAG.getConstantFP(Val: IsLog10 ? cc_log10 : cc_log, DL, VT);
2771 // This adds correction terms for which contraction may lead to an increase
2772 // in the error of the approximation, so disable it.
2773 Flags.setAllowContract(false);
2774 R = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Y, N2: C, Flags);
2775 SDValue NegR = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: R, Flags);
2776 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: C, N3: NegR, Flags);
2777 SDValue FMA1 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: CC, N3: FMA0, Flags);
2778 R = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: R, N2: FMA1, Flags);
2779 } else {
2780 // ch+ct is ln(2)/ln(10) to more than 36 bits
2781 const float ch_log10 = 0x1.344000p-2f;
2782 const float ct_log10 = 0x1.3509f6p-18f;
2783
2784 // ch + ct is ln(2) to more than 36 bits
2785 const float ch_log = 0x1.62e000p-1f;
2786 const float ct_log = 0x1.0bfbe8p-15f;
2787
2788 SDValue CH = DAG.getConstantFP(Val: IsLog10 ? ch_log10 : ch_log, DL, VT);
2789 SDValue CT = DAG.getConstantFP(Val: IsLog10 ? ct_log10 : ct_log, DL, VT);
2790
2791 SDValue YAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Y);
2792 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL, VT: MVT::i32);
2793 SDValue YHInt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: YAsInt, N2: MaskConst);
2794 SDValue YH = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: YHInt);
2795 SDValue YT = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: YH, Flags);
2796 // This adds correction terms for which contraction may lead to an increase
2797 // in the error of the approximation, so disable it.
2798 Flags.setAllowContract(false);
2799 SDValue YTCT = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: YT, N2: CT, Flags);
2800 SDValue Mad0 = getMad(DAG, SL: DL, VT, X: YH, Y: CT, C: YTCT, Flags);
2801 SDValue Mad1 = getMad(DAG, SL: DL, VT, X: YT, Y: CH, C: Mad0, Flags);
2802 R = getMad(DAG, SL: DL, VT, X: YH, Y: CH, C: Mad1);
2803 }
2804
2805 const bool IsFiniteOnly = Flags.hasNoNaNs() && Flags.hasNoInfs();
2806
2807 // TODO: Check if known finite from source value.
2808 if (!IsFiniteOnly) {
2809 SDValue IsFinite = getIsFinite(DAG, Src: Y, Flags);
2810 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsFinite, N2: R, N3: Y, Flags);
2811 }
2812
2813 if (IsScaled) {
2814 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
2815 SDValue ShiftK =
2816 DAG.getConstantFP(Val: IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2817 SDValue Shift =
2818 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsScaled, N2: ShiftK, N3: Zero, Flags);
2819 R = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: R, N2: Shift, Flags);
2820 }
2821
2822 return R;
2823}
2824
2825// Do f32 fast math expansion for flog2 or flog10. This is accurate enough for a
2826// promote f16 operation.
2827SDValue AMDGPUTargetLowering::LowerFLOGUnsafe(SDValue Src, const SDLoc &SL,
2828 SelectionDAG &DAG, bool IsLog10,
2829 SDNodeFlags Flags) const {
2830 EVT VT = Src.getValueType();
2831 unsigned LogOp =
2832 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2;
2833
2834 double Log2BaseInverted =
2835 IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2836
2837 if (VT == MVT::f32) {
2838 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags);
2839 if (ScaledInput) {
2840 SDValue LogSrc = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2841 SDValue ScaledResultOffset =
2842 DAG.getConstantFP(Val: -32.0 * Log2BaseInverted, DL: SL, VT);
2843
2844 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL: SL, VT);
2845
2846 SDValue ResultOffset = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsScaled,
2847 N2: ScaledResultOffset, N3: Zero, Flags);
2848
2849 SDValue Log2Inv = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2850
2851 if (Subtarget->hasFastFMAF32())
2852 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: LogSrc, N2: Log2Inv, N3: ResultOffset,
2853 Flags);
2854 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LogSrc, N2: Log2Inv, Flags);
2855 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: ResultOffset);
2856 }
2857 }
2858
2859 SDValue Log2Operand = DAG.getNode(Opcode: LogOp, DL: SL, VT, Operand: Src, Flags);
2860 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2861
2862 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Log2Operand, N2: Log2BaseInvertedOperand,
2863 Flags);
2864}
2865
2866// This expansion gives a result slightly better than 1ulp.
2867SDValue AMDGPUTargetLowering::lowerFEXPF64(SDValue Op,
2868 SelectionDAG &DAG) const {
2869 SDLoc DL(Op);
2870 SDValue X = Op.getOperand(i: 0);
2871
2872 // TODO: Check if reassoc is safe. There is an output change in exp2 and
2873 // exp10, which slightly increases ulp.
2874 SDNodeFlags Flags = Op->getFlags() & ~SDNodeFlags::AllowReassociation;
2875
2876 SDValue DN, F, T;
2877
2878 if (Op.getOpcode() == ISD::FEXP2) {
2879 // dn = rint(x)
2880 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: X, Flags);
2881 // f = x - dn
2882 F = DAG.getNode(Opcode: ISD::FSUB, DL, VT: MVT::f64, N1: X, N2: DN, Flags);
2883 // t = f*C1 + f*C2
2884 SDValue C1 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2885 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2886 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C2, Flags);
2887 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C1, N3: Mul2, Flags);
2888 } else if (Op.getOpcode() == ISD::FEXP10) {
2889 // dn = rint(x * C1)
2890 SDValue C1 = DAG.getConstantFP(Val: 0x1.a934f0979a371p+1, DL, VT: MVT::f64);
2891 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2892 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2893
2894 // f = FMA(-dn, C2, FMA(-dn, C3, x))
2895 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2896 SDValue C2 = DAG.getConstantFP(Val: -0x1.9dc1da994fd21p-59, DL, VT: MVT::f64);
2897 SDValue C3 = DAG.getConstantFP(Val: 0x1.34413509f79ffp-2, DL, VT: MVT::f64);
2898 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2899 F = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2900
2901 // t = FMA(f, C4, f*C5)
2902 SDValue C4 = DAG.getConstantFP(Val: 0x1.26bb1bbb55516p+1, DL, VT: MVT::f64);
2903 SDValue C5 = DAG.getConstantFP(Val: -0x1.f48ad494ea3e9p-53, DL, VT: MVT::f64);
2904 SDValue MulF = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C5, Flags);
2905 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C4, N3: MulF, Flags);
2906 } else { // ISD::FEXP
2907 // dn = rint(x * C1)
2908 SDValue C1 = DAG.getConstantFP(Val: 0x1.71547652b82fep+0, DL, VT: MVT::f64);
2909 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2910 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2911
2912 // t = FMA(-dn, C2, FMA(-dn, C3, x))
2913 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2914 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2915 SDValue C3 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2916 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2917 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2918 }
2919
2920 // Polynomial expansion for p
2921 SDValue P = DAG.getConstantFP(Val: 0x1.ade156a5dcb37p-26, DL, VT: MVT::f64);
2922 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2923 N3: DAG.getConstantFP(Val: 0x1.28af3fca7ab0cp-22, DL, VT: MVT::f64),
2924 Flags);
2925 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2926 N3: DAG.getConstantFP(Val: 0x1.71dee623fde64p-19, DL, VT: MVT::f64),
2927 Flags);
2928 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2929 N3: DAG.getConstantFP(Val: 0x1.a01997c89e6b0p-16, DL, VT: MVT::f64),
2930 Flags);
2931 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2932 N3: DAG.getConstantFP(Val: 0x1.a01a014761f6ep-13, DL, VT: MVT::f64),
2933 Flags);
2934 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2935 N3: DAG.getConstantFP(Val: 0x1.6c16c1852b7b0p-10, DL, VT: MVT::f64),
2936 Flags);
2937 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2938 N3: DAG.getConstantFP(Val: 0x1.1111111122322p-7, DL, VT: MVT::f64), Flags);
2939 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2940 N3: DAG.getConstantFP(Val: 0x1.55555555502a1p-5, DL, VT: MVT::f64), Flags);
2941 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2942 N3: DAG.getConstantFP(Val: 0x1.5555555555511p-3, DL, VT: MVT::f64), Flags);
2943 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2944 N3: DAG.getConstantFP(Val: 0x1.000000000000bp-1, DL, VT: MVT::f64), Flags);
2945
2946 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT: MVT::f64);
2947
2948 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
2949 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
2950
2951 // z = ldexp(p, (int)dn)
2952 SDValue DNInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: MVT::i32, Operand: DN);
2953 SDValue Z = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: P, N2: DNInt, Flags);
2954
2955 // Overflow/underflow guards
2956 SDValue CondHi = DAG.getSetCC(
2957 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: 1024.0, DL, VT: MVT::f64), Cond: ISD::SETULE);
2958
2959 if (!Flags.hasNoInfs()) {
2960 SDValue PInf = DAG.getConstantFP(Val: std::numeric_limits<double>::infinity(),
2961 DL, VT: MVT::f64);
2962 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondHi, LHS: Z, RHS: PInf, Flags);
2963 }
2964
2965 SDValue CondLo = DAG.getSetCC(
2966 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: -1075.0, DL, VT: MVT::f64), Cond: ISD::SETUGE);
2967 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
2968 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondLo, LHS: Z, RHS: Zero, Flags);
2969
2970 return Z;
2971}
2972
2973SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const {
2974 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
2975 // If we have to handle denormals, scale up the input and adjust the result.
2976
2977 EVT VT = Op.getValueType();
2978 if (VT == MVT::f64)
2979 return lowerFEXPF64(Op, DAG);
2980
2981 SDLoc SL(Op);
2982 SDValue Src = Op.getOperand(i: 0);
2983 SDNodeFlags Flags = Op->getFlags();
2984
2985 if (VT == MVT::f16) {
2986 // Nothing in half is a denormal when promoted to f32.
2987 assert(!isTypeLegal(MVT::f16));
2988 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
2989 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
2990 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
2991 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
2992 }
2993
2994 assert(VT == MVT::f32);
2995
2996 if (!needsDenormHandlingF32(DAG, Src, Flags))
2997 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Src, Flags);
2998
2999 // bool needs_scaling = x < -0x1.f80000p+6f;
3000 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3001
3002 // -nextafter(128.0, -1)
3003 SDValue RangeCheckConst = DAG.getConstantFP(Val: -0x1.f80000p+6f, DL: SL, VT);
3004
3005 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3006
3007 SDValue NeedsScaling =
3008 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: RangeCheckConst, Cond: ISD::SETOLT);
3009
3010 SDValue SixtyFour = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3011 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3012
3013 SDValue AddOffset =
3014 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: SixtyFour, N3: Zero);
3015
3016 SDValue AddInput = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Src, N2: AddOffset, Flags);
3017 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: AddInput, Flags);
3018
3019 SDValue TwoExpNeg64 = DAG.getConstantFP(Val: 0x1.0p-64f, DL: SL, VT);
3020 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
3021 SDValue ResultScale =
3022 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: TwoExpNeg64, N3: One);
3023
3024 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScale, Flags);
3025}
3026
3027SDValue AMDGPUTargetLowering::lowerFEXPUnsafeImpl(SDValue X, const SDLoc &SL,
3028 SelectionDAG &DAG,
3029 SDNodeFlags Flags,
3030 bool IsExp10) const {
3031 // exp(x) -> exp2(M_LOG2E_F * x);
3032 // exp10(x) -> exp2(log2(10) * x);
3033 EVT VT = X.getValueType();
3034 SDValue Const =
3035 DAG.getConstantFP(Val: IsExp10 ? 0x1.a934f0p+1f : numbers::log2e, DL: SL, VT);
3036
3037 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Const, Flags);
3038 return DAG.getNode(Opcode: VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
3039 : (unsigned)ISD::FEXP2,
3040 DL: SL, VT, Operand: Mul, Flags);
3041}
3042
3043SDValue AMDGPUTargetLowering::lowerFEXPUnsafe(SDValue X, const SDLoc &SL,
3044 SelectionDAG &DAG,
3045 SDNodeFlags Flags) const {
3046 EVT VT = X.getValueType();
3047 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags))
3048 return lowerFEXPUnsafeImpl(X, SL, DAG, Flags, /*IsExp10=*/false);
3049
3050 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3051
3052 SDValue Threshold = DAG.getConstantFP(Val: -0x1.5d58a0p+6f, DL: SL, VT);
3053 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3054
3055 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3056
3057 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3058
3059 SDValue AdjustedX =
3060 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3061
3062 const SDValue Log2E = DAG.getConstantFP(Val: numbers::log2e, DL: SL, VT);
3063 SDValue ExpInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: Log2E, Flags);
3064
3065 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: ExpInput, Flags);
3066
3067 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.969d48p-93f, DL: SL, VT);
3068 SDValue AdjustedResult =
3069 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScaleFactor, Flags);
3070
3071 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: Exp2,
3072 Flags);
3073}
3074
3075/// Emit approx-funcs appropriate lowering for exp10. inf/nan should still be
3076/// handled correctly.
3077SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL,
3078 SelectionDAG &DAG,
3079 SDNodeFlags Flags) const {
3080 const EVT VT = X.getValueType();
3081
3082 const unsigned Exp2Op = VT == MVT::f32 ? static_cast<unsigned>(AMDGPUISD::EXP)
3083 : static_cast<unsigned>(ISD::FEXP2);
3084
3085 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags)) {
3086 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
3087 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3088 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3089
3090 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K0, Flags);
3091 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3092 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K1, Flags);
3093 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3094 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1);
3095 }
3096
3097 // bool s = x < -0x1.2f7030p+5f;
3098 // x += s ? 0x1.0p+5f : 0.0f;
3099 // exp10 = exp2(x * 0x1.a92000p+1f) *
3100 // exp2(x * 0x1.4f0978p-11f) *
3101 // (s ? 0x1.9f623ep-107f : 1.0f);
3102
3103 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3104
3105 SDValue Threshold = DAG.getConstantFP(Val: -0x1.2f7030p+5f, DL: SL, VT);
3106 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3107
3108 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+5f, DL: SL, VT);
3109 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3110 SDValue AdjustedX =
3111 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3112
3113 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3114 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3115
3116 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K0, Flags);
3117 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3118 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K1, Flags);
3119 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3120
3121 SDValue MulExps = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1, Flags);
3122
3123 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.9f623ep-107f, DL: SL, VT);
3124 SDValue AdjustedResult =
3125 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: MulExps, N2: ResultScaleFactor, Flags);
3126
3127 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: MulExps,
3128 Flags);
3129}
3130
3131SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
3132 EVT VT = Op.getValueType();
3133
3134 if (VT == MVT::f64)
3135 return lowerFEXPF64(Op, DAG);
3136
3137 SDLoc SL(Op);
3138 SDValue X = Op.getOperand(i: 0);
3139 SDNodeFlags Flags = Op->getFlags();
3140 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10;
3141
3142 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
3143 // library behavior. Also, is known-not-daz source sufficient?
3144 if (allowApproxFunc(DAG, Flags)) { // TODO: Does this really require fast?
3145 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags)
3146 : lowerFEXPUnsafe(X, SL, DAG, Flags);
3147 }
3148
3149 if (VT.getScalarType() == MVT::f16) {
3150 if (VT.isVector())
3151 return SDValue();
3152
3153 // Nothing in half is a denormal when promoted to f32.
3154 //
3155 // exp(f16 x) ->
3156 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
3157 //
3158 // exp10(f16 x) ->
3159 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
3160 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: X, Flags);
3161 SDValue Lowered = lowerFEXPUnsafeImpl(X: Ext, SL, DAG, Flags, IsExp10);
3162 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Lowered,
3163 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3164 }
3165
3166 assert(VT == MVT::f32);
3167
3168 // Algorithm:
3169 //
3170 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
3171 //
3172 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
3173 // n = 64*m + j, 0 <= j < 64
3174 //
3175 // e^x = 2^((64*m + j + f)/64)
3176 // = (2^m) * (2^(j/64)) * 2^(f/64)
3177 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
3178 //
3179 // f = x*(64/ln(2)) - n
3180 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
3181 //
3182 // e^x = (2^m) * (2^(j/64)) * e^r
3183 //
3184 // (2^(j/64)) is precomputed
3185 //
3186 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3187 // e^r = 1 + q
3188 //
3189 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3190 //
3191 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
3192 SDNodeFlags FlagsNoContract = Flags;
3193 FlagsNoContract.setAllowContract(false);
3194
3195 SDValue PH, PL;
3196 if (Subtarget->hasFastFMAF32()) {
3197 const float c_exp = numbers::log2ef;
3198 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
3199 const float c_exp10 = 0x1.a934f0p+1f;
3200 const float cc_exp10 = 0x1.2f346ep-24f;
3201
3202 SDValue C = DAG.getConstantFP(Val: IsExp10 ? c_exp10 : c_exp, DL: SL, VT);
3203 SDValue CC = DAG.getConstantFP(Val: IsExp10 ? cc_exp10 : cc_exp, DL: SL, VT);
3204
3205 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: C, Flags);
3206 SDValue NegPH = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: PH, Flags);
3207 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: C, N3: NegPH, Flags);
3208 PL = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: CC, N3: FMA0, Flags);
3209 } else {
3210 const float ch_exp = 0x1.714000p+0f;
3211 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
3212
3213 const float ch_exp10 = 0x1.a92000p+1f;
3214 const float cl_exp10 = 0x1.4f0978p-11f;
3215
3216 SDValue CH = DAG.getConstantFP(Val: IsExp10 ? ch_exp10 : ch_exp, DL: SL, VT);
3217 SDValue CL = DAG.getConstantFP(Val: IsExp10 ? cl_exp10 : cl_exp, DL: SL, VT);
3218
3219 SDValue XAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: X);
3220 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL: SL, VT: MVT::i32);
3221 SDValue XHAsInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: XAsInt, N2: MaskConst);
3222 SDValue XH = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: XHAsInt);
3223 SDValue XL = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: XH, Flags);
3224
3225 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XH, N2: CH, Flags);
3226
3227 SDValue XLCL = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XL, N2: CL, Flags);
3228 SDValue Mad0 = getMad(DAG, SL, VT, X: XL, Y: CH, C: XLCL, Flags);
3229 PL = getMad(DAG, SL, VT, X: XH, Y: CL, C: Mad0, Flags);
3230 }
3231
3232 SDValue E = DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SL, VT, Operand: PH, Flags);
3233
3234 // It is unsafe to contract this fsub into the PH multiply.
3235 SDValue PHSubE = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: PH, N2: E, Flags: FlagsNoContract);
3236
3237 SDValue A = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: PHSubE, N2: PL, Flags);
3238 SDValue IntE = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: SL, VT: MVT::i32, Operand: E);
3239 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: A, Flags);
3240
3241 SDValue R = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: Exp2, N2: IntE, Flags);
3242
3243 SDValue UnderflowCheckConst =
3244 DAG.getConstantFP(Val: IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, DL: SL, VT);
3245
3246 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3247 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3248 SDValue Underflow =
3249 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: UnderflowCheckConst, Cond: ISD::SETOLT);
3250
3251 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Underflow, N2: Zero, N3: R);
3252
3253 if (!Flags.hasNoInfs()) {
3254 SDValue OverflowCheckConst =
3255 DAG.getConstantFP(Val: IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, DL: SL, VT);
3256 SDValue Overflow =
3257 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: OverflowCheckConst, Cond: ISD::SETOGT);
3258 SDValue Inf =
3259 DAG.getConstantFP(Val: APFloat::getInf(Sem: APFloat::IEEEsingle()), DL: SL, VT);
3260 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Overflow, N2: Inf, N3: R);
3261 }
3262
3263 return R;
3264}
3265
3266// No pow instruction or libcall to fall back on. fmul_legacy returns 0 for a
3267// zero operand even against an infinity or a NaN, so pow(x, 0) and pow(1, y)
3268// fall out as exp2(0) = 1.
3269SDValue AMDGPUTargetLowering::lowerFPOW(SDValue Op, SelectionDAG &DAG) const {
3270 EVT VT = Op.getValueType();
3271 assert(VT == MVT::f32);
3272
3273 SDLoc SL(Op);
3274 SDValue X = Op.getOperand(i: 0);
3275 SDValue Y = Op.getOperand(i: 1);
3276 SDNodeFlags Flags = Op->getFlags();
3277
3278 // log2(0) is -inf, which exp2 turns back into a finite result, so the core
3279 // goes infinite for inputs a ninf fpow still asserts about, like pow(0, 2).
3280 SDNodeFlags CoreFlags = Flags;
3281 CoreFlags.setNoInfs(false);
3282
3283 // Fast expansion: ignores denormals, NaN for a negative base.
3284 if (allowApproxFunc(DAG, Flags)) {
3285 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: X, Flags: CoreFlags);
3286 SDValue Mul =
3287 DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL: SL, VT, N1: Y, N2: Log, Flags: CoreFlags);
3288 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: Mul, Flags: CoreFlags);
3289 }
3290
3291 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: X, Flags);
3292 SDValue Log = DAG.getNode(Opcode: ISD::FLOG2, DL: SL, VT, Operand: Abs, Flags: CoreFlags);
3293 SDValue Mul = DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL: SL, VT, N1: Y, N2: Log, Flags: CoreFlags);
3294 SDValue R = DAG.getNode(Opcode: ISD::FEXP2, DL: SL, VT, Operand: Mul, Flags: CoreFlags);
3295
3296 // A base that is never negative needs neither the sign fixup nor the NaN.
3297 if (DAG.computeKnownFPClass(Op: X, InterestedClasses: fcNegative).signBitIsZeroOrNaN())
3298 return R;
3299
3300 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3301
3302 // Infinities count as integers, and every f32 >= 2^24 in magnitude is even.
3303 SDValue YTrunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: Y);
3304 SDValue YIsInt = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: YTrunc, RHS: Y, Cond: ISD::SETOEQ);
3305 SDValue YHalf =
3306 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Y, N2: DAG.getConstantFP(Val: 0.5, DL: SL, VT));
3307 SDValue YHalfTrunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: YHalf);
3308 SDValue YIsOdd =
3309 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: YIsInt,
3310 N2: DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: YHalfTrunc, RHS: YHalf, Cond: ISD::SETONE));
3311
3312 // pow(-x, odd y) = -pow(x, y). Selecting the copysign lets even y fold it.
3313 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: YIsOdd,
3314 N2: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: R, N2: X), N3: R);
3315
3316 if (Flags.hasNoNaNs())
3317 return R;
3318
3319 // A negative finite base to a non-integral power is NaN. -inf is excluded:
3320 // the core already gives pow(+inf, y). So are subnormals when flushed.
3321 FPClassTest NegFiniteMask = fcNegNormal;
3322 if (!DAG.getMachineFunction()
3323 .getDenormalMode(FPType: APFloat::IEEEsingle())
3324 .inputsAreZero())
3325 NegFiniteMask |= fcNegSubnormal;
3326 SDValue XNegFinite =
3327 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: SL, VT: SetCCVT, N1: X,
3328 N2: DAG.getTargetConstant(Val: NegFiniteMask, DL: SL, VT: MVT::i32));
3329 // Not a SETONE compare: pow(-1, NaN) needs the NaN-true behavior of !SETOEQ.
3330 SDValue NegNonInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: XNegFinite,
3331 N2: DAG.getNOT(DL: SL, Val: YIsInt, VT: SetCCVT));
3332 SDValue NaN =
3333 DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()), DL: SL, VT);
3334 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NegNonInt, N2: NaN, N3: R);
3335}
3336
3337static bool isCtlzOpc(unsigned Opc) {
3338 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
3339}
3340
3341static bool isCttzOpc(unsigned Opc) {
3342 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_POISON;
3343}
3344
3345SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op,
3346 SelectionDAG &DAG) const {
3347 auto SL = SDLoc(Op);
3348 auto Opc = Op.getOpcode();
3349 auto Arg = Op.getOperand(i: 0u);
3350 auto ResultVT = Op.getValueType();
3351
3352 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3353 return {};
3354
3355 assert(isCtlzOpc(Opc));
3356 assert(ResultVT == Arg.getValueType());
3357
3358 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3359 SDValue NumExtBits = DAG.getConstant(Val: 32u - NumBits, DL: SL, VT: MVT::i32);
3360 SDValue NewOp;
3361
3362 if (Opc == ISD::CTLZ_ZERO_POISON) {
3363 NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3364 NewOp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3365 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3366 } else {
3367 NewOp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3368 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3369 NewOp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3370 }
3371
3372 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ResultVT, Operand: NewOp);
3373}
3374
3375SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
3376 SDLoc SL(Op);
3377 SDValue Src = Op.getOperand(i: 0);
3378
3379 assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode()));
3380 bool Ctlz = isCtlzOpc(Opc: Op.getOpcode());
3381 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;
3382
3383 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_POISON ||
3384 Op.getOpcode() == ISD::CTTZ_ZERO_POISON;
3385 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3386
3387 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3388 // (ctlz hi:lo) -> (umin (ffbh src), 32)
3389 // (cttz hi:lo) -> (umin (ffbl src), 32)
3390 // (ctlz_zero_poison src) -> (ffbh src)
3391 // (cttz_zero_poison src) -> (ffbl src)
3392
3393 // 64-bit scalar version produce 32-bit result
3394 // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64)
3395 // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64)
3396 // (ctlz_zero_poison src) -> (S_FLBIT_I32_B64 src)
3397 // (cttz_zero_poison src) -> (S_FF1_I32_B64 src)
3398 SDValue NewOpr = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Src);
3399 if (!ZeroUndef) {
3400 const SDValue ConstVal = DAG.getConstant(
3401 Val: Op.getValueType().getScalarSizeInBits(), DL: SL, VT: MVT::i32);
3402 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: ConstVal);
3403 }
3404 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: Src.getValueType(), Operand: NewOpr);
3405 }
3406
3407 SDValue Lo, Hi;
3408 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3409
3410 SDValue OprLo = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Lo);
3411 SDValue OprHi = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Hi);
3412
3413 // (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64)
3414 // (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64)
3415 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
3416 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
3417
3418 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT;
3419 const SDValue Const32 = DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32);
3420 if (Ctlz)
3421 OprLo = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprLo, N2: Const32);
3422 else
3423 OprHi = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprHi, N2: Const32);
3424
3425 SDValue NewOpr;
3426 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: OprLo, N2: OprHi);
3427 if (!ZeroUndef) {
3428 const SDValue Const64 = DAG.getConstant(Val: 64, DL: SL, VT: MVT::i32);
3429 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: Const64);
3430 }
3431
3432 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: NewOpr);
3433}
3434
3435SDValue AMDGPUTargetLowering::LowerCTLS(SDValue Op, SelectionDAG &DAG) const {
3436 SDLoc SL(Op);
3437 SDValue Src = Op.getOperand(i: 0);
3438 assert(Src.getValueType() == MVT::i32 && "LowerCTLS only supports i32");
3439 SDValue Ffbh = DAG.getNode(
3440 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3441 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Src);
3442 SDValue Clamped = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: Ffbh,
3443 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3444 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: Clamped,
3445 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i32));
3446}
3447
3448SDValue AMDGPUTargetLowering::LowerINT_TO_FP16(SDValue Op, SelectionDAG &DAG,
3449 EVT FP16Ty) const {
3450 assert(FP16Ty == MVT::f16 || FP16Ty == MVT::bf16);
3451 SDLoc SL(Op);
3452 SDValue Src = Op.getOperand(i: 0);
3453 SDValue ToF32 = DAG.getNode(Opcode: Op.getOpcode(), DL: SL, VT: MVT::f32, Operand: Src);
3454 SDValue FPRoundFlag = DAG.getIntPtrConstant(Val: 0, DL: SL, /*isTarget=*/true);
3455 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: FP16Ty, N1: ToF32, N2: FPRoundFlag);
3456}
3457
3458SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
3459 bool Signed) const {
3460 // The regular method converting a 64-bit integer to float roughly consists of
3461 // 2 steps: normalization and rounding. In fact, after normalization, the
3462 // conversion from a 64-bit integer to a float is essentially the same as the
3463 // one from a 32-bit integer. The only difference is that it has more
3464 // trailing bits to be rounded. To leverage the native 32-bit conversion, a
3465 // 64-bit integer could be preprocessed and fit into a 32-bit integer then
3466 // converted into the correct float number. The basic steps for the unsigned
3467 // conversion are illustrated in the following pseudo code:
3468 //
3469 // f32 uitofp(i64 u) {
3470 // i32 hi, lo = split(u);
3471 // // Only count the leading zeros in hi as we have native support of the
3472 // // conversion from i32 to f32. If hi is all 0s, the conversion is
3473 // // reduced to a 32-bit one automatically.
3474 // i32 shamt = clz(hi); // Return 32 if hi is all 0s.
3475 // u <<= shamt;
3476 // hi, lo = split(u);
3477 // hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo.
3478 // // convert it as a 32-bit integer and scale the result back.
3479 // return uitofp(hi) * 2^(32 - shamt);
3480 // }
3481 //
3482 // The signed one follows the same principle but uses 'ffbh_i32' to count its
3483 // sign bits instead. If 'ffbh_i32' is not available, its absolute value is
3484 // converted instead followed by negation based its sign bit.
3485
3486 SDLoc SL(Op);
3487 SDValue Src = Op.getOperand(i: 0);
3488
3489 SDValue Lo, Hi;
3490 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3491 SDValue Sign;
3492 SDValue ShAmt;
3493 if (Signed && Subtarget->isGCN()) {
3494 // We also need to consider the sign bit in Lo if Hi has just sign bits,
3495 // i.e. Hi is 0 or -1. However, that only needs to take the MSB into
3496 // account. That is, the maximal shift is
3497 // - 32 if Lo and Hi have opposite signs;
3498 // - 33 if Lo and Hi have the same sign.
3499 //
3500 // Or, MaxShAmt = 33 + OppositeSign, where
3501 //
3502 // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is
3503 // - -1 if Lo and Hi have opposite signs; and
3504 // - 0 otherwise.
3505 //
3506 // All in all, ShAmt is calculated as
3507 //
3508 // umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1.
3509 //
3510 // or
3511 //
3512 // umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31).
3513 //
3514 // to reduce the critical path.
3515 SDValue OppositeSign = DAG.getNode(
3516 Opcode: ISD::SRA, DL: SL, VT: MVT::i32, N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: Lo, N2: Hi),
3517 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3518 SDValue MaxShAmt =
3519 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3520 N2: OppositeSign);
3521 // Count the leading sign bits.
3522 ShAmt = DAG.getNode(
3523 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3524 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Hi);
3525 // Different from unsigned conversion, the shift should be one bit less to
3526 // preserve the sign bit.
3527 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ShAmt,
3528 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
3529 ShAmt = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: ShAmt, N2: MaxShAmt);
3530 } else {
3531 if (Signed) {
3532 // Without 'ffbh_i32', only leading zeros could be counted. Take the
3533 // absolute value first.
3534 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: Src,
3535 N2: DAG.getConstant(Val: 63, DL: SL, VT: MVT::i64));
3536 SDValue Abs =
3537 DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64,
3538 N1: DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: Src, N2: Sign), N2: Sign);
3539 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Abs, DAG);
3540 }
3541 // Count the leading zeros.
3542 ShAmt = DAG.getNode(Opcode: ISD::CTLZ, DL: SL, VT: MVT::i32, Operand: Hi);
3543 // The shift amount for signed integers is [0, 32].
3544 }
3545 // Normalize the given 64-bit integer.
3546 SDValue Norm = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i64, N1: Src, N2: ShAmt);
3547 // Split it again.
3548 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Norm, DAG);
3549 // Calculate the adjust bit for rounding.
3550 // (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo)
3551 SDValue Adjust = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32,
3552 N1: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32), N2: Lo);
3553 // Get the 32-bit normalized integer.
3554 Norm = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Hi, N2: Adjust);
3555 // Convert the normalized 32-bit integer into f32.
3556
3557 bool UseLDEXP = isOperationLegal(Op: ISD::FLDEXP, VT: MVT::f32);
3558 unsigned Opc = Signed && UseLDEXP ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3559 SDValue FVal = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::f32, Operand: Norm);
3560
3561 // Finally, need to scale back the converted floating number as the original
3562 // 64-bit integer is converted as a 32-bit one.
3563 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3564 N2: ShAmt);
3565 // On GCN, use LDEXP directly.
3566 if (UseLDEXP)
3567 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f32, N1: FVal, N2: ShAmt);
3568
3569 // Otherwise, align 'ShAmt' to the exponent part and add it into the exponent
3570 // part directly to emulate the multiplication of 2^ShAmt. That 8-bit
3571 // exponent is enough to avoid overflowing into the sign bit.
3572 SDValue Exp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ShAmt,
3573 N2: DAG.getConstant(Val: 23, DL: SL, VT: MVT::i32));
3574 SDValue IVal =
3575 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32,
3576 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: FVal), N2: Exp);
3577 if (Signed) {
3578 // Set the sign bit.
3579 Sign = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32,
3580 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Sign),
3581 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3582 IVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: IVal, N2: Sign);
3583 }
3584 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: IVal);
3585}
3586
3587SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
3588 bool Signed) const {
3589 SDLoc SL(Op);
3590 SDValue Src = Op.getOperand(i: 0);
3591
3592 SDValue Lo, Hi;
3593 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3594
3595 SDValue CvtHi = DAG.getNode(Opcode: Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
3596 DL: SL, VT: MVT::f64, Operand: Hi);
3597
3598 SDValue CvtLo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: SL, VT: MVT::f64, Operand: Lo);
3599
3600 SDValue LdExp = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f64, N1: CvtHi,
3601 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3602 // TODO: Should this propagate fast-math-flags?
3603 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: LdExp, N2: CvtLo);
3604}
3605
3606SDValue AMDGPUTargetLowering::lowerINT_TO_FPImpl(SDValue Op, SelectionDAG &DAG,
3607 bool Signed) const {
3608 EVT DestVT = Op.getValueType();
3609 SDValue Src = Op.getOperand(i: 0);
3610 EVT SrcVT = Src.getValueType();
3611 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3612 unsigned CvtOpc = Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3613
3614 if (SrcVT == MVT::i16) {
3615 if (DestVT == MVT::f16)
3616 return Op;
3617
3618 SDLoc DL(Op);
3619 SDValue Ext = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i32, Operand: Src);
3620 return DAG.getNode(Opcode: CvtOpc, DL, VT: DestVT, Operand: Ext);
3621 }
3622
3623 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3624 return LowerINT_TO_FP16(Op, DAG, FP16Ty: DestVT);
3625
3626 if (SrcVT != MVT::i64)
3627 return Op;
3628
3629 if (DestVT == MVT::f32)
3630 return LowerINT_TO_FP32(Op, DAG, Signed);
3631
3632 assert(DestVT == MVT::f64);
3633 return LowerINT_TO_FP64(Op, DAG, Signed);
3634}
3635
3636SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
3637 SelectionDAG &DAG) const {
3638 return lowerINT_TO_FPImpl(Op, DAG, Signed: false);
3639}
3640
3641SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
3642 SelectionDAG &DAG) const {
3643 return lowerINT_TO_FPImpl(Op, DAG, Signed: true);
3644}
3645
3646SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
3647 bool Signed) const {
3648 SDLoc SL(Op);
3649
3650 SDValue Src = Op.getOperand(i: 0);
3651 EVT SrcVT = Src.getValueType();
3652
3653 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3654
3655 // The basic idea of converting a floating point number into a pair of 32-bit
3656 // integers is illustrated as follows:
3657 //
3658 // tf := trunc(val);
3659 // hif := floor(tf * 2^-32);
3660 // lof := tf - hif * 2^32; // lof is always positive due to floor.
3661 // hi := fptoi(hif);
3662 // lo := fptoi(lof);
3663 //
3664 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: SrcVT, Operand: Src);
3665 SDValue Sign;
3666 if (Signed && SrcVT == MVT::f32) {
3667 // However, a 32-bit floating point number has only 23 bits mantissa and
3668 // it's not enough to hold all the significant bits of `lof` if val is
3669 // negative. To avoid the loss of precision, We need to take the absolute
3670 // value after truncating and flip the result back based on the original
3671 // signedness.
3672 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i32,
3673 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Trunc),
3674 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3675 Trunc = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: SrcVT, Operand: Trunc);
3676 }
3677
3678 SDValue K0, K1;
3679 if (SrcVT == MVT::f64) {
3680 K0 = DAG.getConstantFP(
3681 Val: llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), DL: SL,
3682 VT: SrcVT);
3683 K1 = DAG.getConstantFP(
3684 Val: llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), DL: SL,
3685 VT: SrcVT);
3686 } else {
3687 K0 = DAG.getConstantFP(
3688 Val: llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), DL: SL, VT: SrcVT);
3689 K1 = DAG.getConstantFP(
3690 Val: llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), DL: SL, VT: SrcVT);
3691 }
3692 // TODO: Should this propagate fast-math-flags?
3693 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: SrcVT, N1: Trunc, N2: K0);
3694
3695 SDValue FloorMul = DAG.getNode(Opcode: ISD::FFLOOR, DL: SL, VT: SrcVT, Operand: Mul);
3696
3697 SDValue Fma = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: SrcVT, N1: FloorMul, N2: K1, N3: Trunc);
3698
3699 SDValue Hi = DAG.getNode(Opcode: (Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
3700 : ISD::FP_TO_UINT,
3701 DL: SL, VT: MVT::i32, Operand: FloorMul);
3702 SDValue Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SL, VT: MVT::i32, Operand: Fma);
3703
3704 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3705 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Lo, Hi}));
3706
3707 if (Signed && SrcVT == MVT::f32) {
3708 assert(Sign);
3709 // Flip the result based on the signedness, which is either all 0s or 1s.
3710 Sign = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3711 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Sign, Sign}));
3712 // r := xor(r, sign) - sign;
3713 Result =
3714 DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i64,
3715 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64, N1: Result, N2: Sign), N2: Sign);
3716 }
3717
3718 return Result;
3719}
3720
3721SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
3722 SDLoc DL(Op);
3723 SDValue N0 = Op.getOperand(i: 0);
3724
3725 // Convert to target node to get known bits
3726 if (N0.getValueType() == MVT::f32)
3727 return DAG.getNode(Opcode: AMDGPUISD::FP_TO_FP16, DL, VT: Op.getValueType(), Operand: N0);
3728
3729 if (Op->getFlags().hasApproximateFuncs()) {
3730 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
3731 return SDValue();
3732 }
3733
3734 return LowerF64ToF16Safe(Src: N0, DL, DAG);
3735}
3736
3737// return node in i32
3738SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
3739 SelectionDAG &DAG) const {
3740 assert(Src.getSimpleValueType() == MVT::f64);
3741
3742 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
3743 // TODO: We can generate better code for True16.
3744 const unsigned ExpMask = 0x7ff;
3745 const unsigned ExpBiasf64 = 1023;
3746 const unsigned ExpBiasf16 = 15;
3747 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3748 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
3749 SDValue U = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Src);
3750 SDValue UH = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: U,
3751 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3752 UH = DAG.getZExtOrTrunc(Op: UH, DL, VT: MVT::i32);
3753 U = DAG.getZExtOrTrunc(Op: U, DL, VT: MVT::i32);
3754 SDValue E = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3755 N2: DAG.getConstant(Val: 20, DL, VT: MVT::i64));
3756 E = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: E,
3757 N2: DAG.getConstant(Val: ExpMask, DL, VT: MVT::i32));
3758 // Subtract the fp64 exponent bias (1023) to get the real exponent and
3759 // add the f16 bias (15) to get the biased exponent for the f16 format.
3760 E = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: E,
3761 N2: DAG.getConstant(Val: -ExpBiasf64 + ExpBiasf16, DL, VT: MVT::i32));
3762
3763 SDValue M = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3764 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32));
3765 M = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M,
3766 N2: DAG.getConstant(Val: 0xffe, DL, VT: MVT::i32));
3767
3768 SDValue MaskedSig = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: UH,
3769 N2: DAG.getConstant(Val: 0x1ff, DL, VT: MVT::i32));
3770 MaskedSig = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: MaskedSig, N2: U);
3771
3772 SDValue Lo40Set = DAG.getSelectCC(DL, LHS: MaskedSig, RHS: Zero, True: Zero, False: One, Cond: ISD::SETEQ);
3773 M = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M, N2: Lo40Set);
3774
3775 // (M != 0 ? 0x0200 : 0) | 0x7c00;
3776 SDValue I = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
3777 N1: DAG.getSelectCC(DL, LHS: M, RHS: Zero, True: DAG.getConstant(Val: 0x0200, DL, VT: MVT::i32),
3778 False: Zero, Cond: ISD::SETNE), N2: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32));
3779
3780 // N = M | (E << 12);
3781 SDValue N = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3782 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: E,
3783 N2: DAG.getConstant(Val: 12, DL, VT: MVT::i32)));
3784
3785 // B = clamp(1-E, 0, 13);
3786 SDValue OneSubExp = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
3787 N1: One, N2: E);
3788 SDValue B = DAG.getNode(Opcode: ISD::SMAX, DL, VT: MVT::i32, N1: OneSubExp, N2: Zero);
3789 B = DAG.getNode(Opcode: ISD::SMIN, DL, VT: MVT::i32, N1: B,
3790 N2: DAG.getConstant(Val: 13, DL, VT: MVT::i32));
3791
3792 SDValue SigSetHigh = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3793 N2: DAG.getConstant(Val: 0x1000, DL, VT: MVT::i32));
3794
3795 SDValue D = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SigSetHigh, N2: B);
3796 SDValue D0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: D, N2: B);
3797 SDValue D1 = DAG.getSelectCC(DL, LHS: D0, RHS: SigSetHigh, True: One, False: Zero, Cond: ISD::SETNE);
3798 D = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: D, N2: D1);
3799
3800 SDValue V = DAG.getSelectCC(DL, LHS: E, RHS: One, True: D, False: N, Cond: ISD::SETLT);
3801 SDValue VLow3 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: V,
3802 N2: DAG.getConstant(Val: 0x7, DL, VT: MVT::i32));
3803 V = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: V,
3804 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3805 SDValue V0 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 3, DL, VT: MVT::i32),
3806 True: One, False: Zero, Cond: ISD::SETEQ);
3807 SDValue V1 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 5, DL, VT: MVT::i32),
3808 True: One, False: Zero, Cond: ISD::SETGT);
3809 V1 = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: V0, N2: V1);
3810 V = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: V, N2: V1);
3811
3812 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 30, DL, VT: MVT::i32),
3813 True: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32), False: V, Cond: ISD::SETGT);
3814 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 1039, DL, VT: MVT::i32),
3815 True: I, False: V, Cond: ISD::SETEQ);
3816
3817 // Extract the sign bit.
3818 SDValue Sign = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3819 N2: DAG.getConstant(Val: 16, DL, VT: MVT::i32));
3820 Sign = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Sign,
3821 N2: DAG.getConstant(Val: 0x8000, DL, VT: MVT::i32));
3822
3823 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Sign, N2: V);
3824}
3825
3826SDValue AMDGPUTargetLowering::LowerFP_TO_INT(const SDValue Op,
3827 SelectionDAG &DAG) const {
3828 SDValue Src = Op.getOperand(i: 0);
3829 unsigned OpOpcode = Op.getOpcode();
3830 EVT SrcVT = Src.getValueType();
3831 EVT DestVT = Op.getValueType();
3832
3833 // Will be selected natively
3834 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3835 return Op;
3836
3837 if (SrcVT == MVT::bf16 || (SrcVT == MVT::f16 && DestVT == MVT::i32)) {
3838 SDLoc DL(Op);
3839 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3840 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DestVT, Operand: PromotedSrc);
3841 }
3842
3843 // Promote i16 to i32
3844 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3845 SDLoc DL(Op);
3846
3847 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3848 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToInt32);
3849 }
3850
3851 if (DestVT != MVT::i64)
3852 return Op;
3853
3854 if (SrcVT == MVT::f16 ||
3855 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
3856 SDLoc DL(Op);
3857
3858 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3859 unsigned Ext =
3860 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3861 return DAG.getNode(Opcode: Ext, DL, VT: MVT::i64, Operand: FpToInt32);
3862 }
3863
3864 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3865 return LowerFP_TO_INT64(Op, DAG, Signed: OpOpcode == ISD::FP_TO_SINT);
3866
3867 return SDValue();
3868}
3869
3870SDValue AMDGPUTargetLowering::LowerFP_TO_INT_SAT(const SDValue Op,
3871 SelectionDAG &DAG) const {
3872 SDValue Src = Op.getOperand(i: 0);
3873 unsigned OpOpcode = Op.getOpcode();
3874 EVT SrcVT = Src.getValueType();
3875 EVT DstVT = Op.getValueType();
3876 SDValue SatVTOp = Op.getNode()->getOperand(Num: 1);
3877 EVT SatVT = cast<VTSDNode>(Val&: SatVTOp)->getVT();
3878 SDLoc DL(Op);
3879
3880 uint64_t DstWidth = DstVT.getScalarSizeInBits();
3881 uint64_t SatWidth = SatVT.getScalarSizeInBits();
3882 assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
3883
3884 // Scalar cases will be selected natively to v_cvt_/s_cvt_ instructions.
3885 // v2f32 -> v2i16 will be selected natively to v_cvt_pk_[iu]16_f32.
3886 if (SatWidth == DstWidth) {
3887 if ((DstVT == MVT::i32 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
3888 (DstVT == MVT::i16 && (SrcVT == MVT::f16 || SrcVT == MVT::f32)) ||
3889 (DstVT == MVT::v2i16 && SrcVT == MVT::v2f32))
3890 return Op;
3891 }
3892
3893 // Vectors can only be selected natively.
3894 if (DstVT.isVector())
3895 return SDValue();
3896
3897 // Perform all saturation at selected width (i16 or i32) and truncate
3898 if (SatWidth < DstWidth && SatWidth <= 32) {
3899 // For f16 conversion with sub-i16 saturation perform saturation
3900 // at i16, if available in the target. This removes the need for extra f16
3901 // to f32 conversion. For all the others use i32.
3902 MVT ResultVT =
3903 Subtarget->has16BitInsts() && SrcVT == MVT::f16 && SatWidth < 16
3904 ? MVT::i16
3905 : MVT::i32;
3906
3907 const SDValue ResultVTOp = DAG.getValueType(ResultVT);
3908 const uint64_t ResultWidth = ResultVT.getScalarSizeInBits();
3909
3910 // First, convert input float into selected integer (i16 or i32)
3911 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: ResultVT, N1: Src, N2: ResultVTOp);
3912 SDValue IntSatVal;
3913
3914 // Then, clamp at the saturation width using either i16 or i32 instructions
3915 if (OpOpcode == ISD::FP_TO_SINT_SAT) {
3916 SDValue MinConst = DAG.getConstant(
3917 Val: APInt::getSignedMaxValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3918 SDValue MaxConst = DAG.getConstant(
3919 Val: APInt::getSignedMinValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3920 SDValue MinVal = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3921 IntSatVal = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ResultVT, N1: MinVal, N2: MaxConst);
3922 } else {
3923 SDValue MinConst = DAG.getConstant(
3924 Val: APInt::getMaxValue(numBits: SatWidth).zext(width: ResultWidth), DL, VT: ResultVT);
3925 IntSatVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3926 }
3927
3928 // Finally, after saturating at i16 or i32 fit into the destination type
3929 return DAG.getExtOrTrunc(IsSigned: OpOpcode == ISD::FP_TO_SINT_SAT, Op: IntSatVal, DL,
3930 VT: DstVT);
3931 }
3932
3933 // SatWidth == DstWidth or SatWidth > 32
3934
3935 // Saturate at i32 for i64 dst and f16/bf16 src (will invoke f16 promotion
3936 // below)
3937 if (DstVT == MVT::i64 &&
3938 (SrcVT == MVT::f16 || SrcVT == MVT::bf16 ||
3939 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP))) {
3940 const SDValue Int32VTOp = DAG.getValueType(MVT::i32);
3941 return DAG.getNode(Opcode: OpOpcode, DL, VT: DstVT, N1: Src, N2: Int32VTOp);
3942 }
3943
3944 // Promote f16/bf16 src to f32 for i32 conversion
3945 if (DstVT == MVT::i32 && (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
3946 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3947 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DstVT, N1: PromotedSrc, N2: SatVTOp);
3948 }
3949
3950 // For DstWidth < 16, promote i1 and i8 dst to i16 (if legal) with sub-i16
3951 // saturation. For DstWidth == 16, promote i16 dst to i32 with sub-i32
3952 // saturation; this covers i16.f32 and i16.f64
3953 if (DstWidth < 32) {
3954 // Note: this triggers SatWidth < DstWidth above to generate saturated
3955 // truncate by requesting MVT::i16/i32 destination with SatWidth < 16/32.
3956 MVT PromoteVT =
3957 (DstWidth < 16 && Subtarget->has16BitInsts()) ? MVT::i16 : MVT::i32;
3958 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: PromoteVT, N1: Src, N2: SatVTOp);
3959 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: FpToInt);
3960 }
3961
3962 // TODO: can we implement i64 dst for f32/f64?
3963
3964 return SDValue();
3965}
3966
3967SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
3968 SelectionDAG &DAG) const {
3969 EVT ExtraVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3970 MVT VT = Op.getSimpleValueType();
3971 MVT ScalarVT = VT.getScalarType();
3972
3973 assert(VT.isVector());
3974
3975 SDValue Src = Op.getOperand(i: 0);
3976 SDLoc DL(Op);
3977
3978 // TODO: Don't scalarize on Evergreen?
3979 unsigned NElts = VT.getVectorNumElements();
3980 SmallVector<SDValue, 8> Args;
3981 DAG.ExtractVectorElements(Op: Src, Args, Start: 0, Count: NElts);
3982
3983 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
3984 for (unsigned I = 0; I < NElts; ++I)
3985 Args[I] = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: ScalarVT, N1: Args[I], N2: VTOp);
3986
3987 return DAG.getBuildVector(VT, DL, Ops: Args);
3988}
3989
3990//===----------------------------------------------------------------------===//
3991// Custom DAG optimizations
3992//===----------------------------------------------------------------------===//
3993
3994static bool isU24(SDValue Op, SelectionDAG &DAG) {
3995 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
3996}
3997
3998static bool isI24(SDValue Op, SelectionDAG &DAG) {
3999 EVT VT = Op.getValueType();
4000 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
4001 // as unsigned 24-bit values.
4002 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24;
4003}
4004
4005static SDValue simplifyMul24(SDNode *Node24,
4006 TargetLowering::DAGCombinerInfo &DCI) {
4007 SelectionDAG &DAG = DCI.DAG;
4008 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4009 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
4010
4011 SDValue LHS = IsIntrin ? Node24->getOperand(Num: 1) : Node24->getOperand(Num: 0);
4012 SDValue RHS = IsIntrin ? Node24->getOperand(Num: 2) : Node24->getOperand(Num: 1);
4013 unsigned NewOpcode = Node24->getOpcode();
4014 if (IsIntrin) {
4015 unsigned IID = Node24->getConstantOperandVal(Num: 0);
4016 switch (IID) {
4017 case Intrinsic::amdgcn_mul_i24:
4018 NewOpcode = AMDGPUISD::MUL_I24;
4019 break;
4020 case Intrinsic::amdgcn_mul_u24:
4021 NewOpcode = AMDGPUISD::MUL_U24;
4022 break;
4023 case Intrinsic::amdgcn_mulhi_i24:
4024 NewOpcode = AMDGPUISD::MULHI_I24;
4025 break;
4026 case Intrinsic::amdgcn_mulhi_u24:
4027 NewOpcode = AMDGPUISD::MULHI_U24;
4028 break;
4029 default:
4030 llvm_unreachable("Expected 24-bit mul intrinsic");
4031 }
4032 }
4033
4034 APInt Demanded = APInt::getLowBitsSet(numBits: LHS.getValueSizeInBits(), loBitsSet: 24);
4035
4036 if (isNullConstant(V: LHS) || isNullConstant(V: RHS))
4037 return DAG.getConstant(Val: 0, DL: SDLoc(Node24), VT: Node24->getValueType(ResNo: 0));
4038
4039 // First try SimplifyDemandedBits which can simplify the nodes used by our
4040 // operands if this node is the only user.
4041 if (LHS.hasOneUse() && TLI.SimplifyDemandedBits(Op: LHS, DemandedBits: Demanded, DCI))
4042 return SDValue(Node24, 0);
4043 if (RHS.hasOneUse() && TLI.SimplifyDemandedBits(Op: RHS, DemandedBits: Demanded, DCI))
4044 return SDValue(Node24, 0);
4045
4046 // Then try SimplifyMultipleUseDemandedBits which allows the operands to have
4047 // other uses, but will only perform simplifications that involve bypassing
4048 // some nodes for this user.
4049 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(Op: LHS, DemandedBits: Demanded, DAG);
4050 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(Op: RHS, DemandedBits: Demanded, DAG);
4051 if (DemandedLHS || DemandedRHS)
4052 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(Node24), VTList: Node24->getVTList(),
4053 N1: DemandedLHS ? DemandedLHS : LHS,
4054 N2: DemandedRHS ? DemandedRHS : RHS);
4055
4056 return SDValue();
4057}
4058
4059template <typename IntTy>
4060static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
4061 uint32_t Width, const SDLoc &DL) {
4062 if (Width + Offset < 32) {
4063 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
4064 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
4065 if constexpr (std::is_signed_v<IntTy>) {
4066 return DAG.getSignedConstant(Val: Result, DL, VT: MVT::i32);
4067 } else {
4068 return DAG.getConstant(Result, DL, MVT::i32);
4069 }
4070 }
4071
4072 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
4073}
4074
4075static bool hasVolatileUser(SDNode *Val) {
4076 for (SDNode *U : Val->users()) {
4077 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: U)) {
4078 if (M->isVolatile())
4079 return true;
4080 }
4081 }
4082
4083 return false;
4084}
4085
4086bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
4087 // i32 vectors are the canonical memory type.
4088 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
4089 return false;
4090
4091 if (!VT.isByteSized())
4092 return false;
4093
4094 unsigned Size = VT.getStoreSize();
4095
4096 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
4097 return false;
4098
4099 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
4100 return false;
4101
4102 return true;
4103}
4104
4105// Replace load of an illegal type with a bitcast from a load of a friendlier
4106// type.
4107SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
4108 DAGCombinerInfo &DCI) const {
4109 if (!DCI.isBeforeLegalize())
4110 return SDValue();
4111
4112 LoadSDNode *LN = cast<LoadSDNode>(Val: N);
4113 if (!LN->isSimple() || !ISD::isNormalLoad(N: LN) || hasVolatileUser(Val: LN))
4114 return SDValue();
4115
4116 SDLoc SL(N);
4117 SelectionDAG &DAG = DCI.DAG;
4118 EVT VT = LN->getMemoryVT();
4119
4120 unsigned Size = VT.getStoreSize();
4121 Align Alignment = LN->getAlign();
4122 if (Alignment < Size && isTypeLegal(VT)) {
4123 unsigned IsFast;
4124 unsigned AS = LN->getAddressSpace();
4125
4126 // Expand unaligned loads earlier than legalization. Due to visitation order
4127 // problems during legalization, the emitted instructions to pack and unpack
4128 // the bytes again are not eliminated in the case of an unaligned copy.
4129 if (!allowsMisalignedMemoryAccesses(
4130 VT, AddrSpace: AS, Alignment, Flags: LN->getMemOperand()->getFlags(), &IsFast)) {
4131 if (VT.isVector())
4132 return SplitVectorLoad(Op: SDValue(LN, 0), DAG);
4133
4134 SDValue Ops[2];
4135 std::tie(args&: Ops[0], args&: Ops[1]) = expandUnalignedLoad(LD: LN, DAG);
4136
4137 return DAG.getMergeValues(Ops, dl: SDLoc(N));
4138 }
4139
4140 if (!IsFast)
4141 return SDValue();
4142 }
4143
4144 if (!shouldCombineMemoryType(VT))
4145 return SDValue();
4146
4147 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4148
4149 SDValue NewLoad
4150 = DAG.getLoad(VT: NewVT, dl: SL, Chain: LN->getChain(),
4151 Ptr: LN->getBasePtr(), MMO: LN->getMemOperand());
4152
4153 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewLoad);
4154 DCI.CombineTo(N, Res0: BC, Res1: NewLoad.getValue(R: 1));
4155 return SDValue(N, 0);
4156}
4157
4158// Replace store of an illegal type with a store of a bitcast to a friendlier
4159// type.
4160SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
4161 DAGCombinerInfo &DCI) const {
4162 if (!DCI.isBeforeLegalize())
4163 return SDValue();
4164
4165 StoreSDNode *SN = cast<StoreSDNode>(Val: N);
4166 if (!SN->isSimple() || !ISD::isNormalStore(N: SN))
4167 return SDValue();
4168
4169 EVT VT = SN->getMemoryVT();
4170 unsigned Size = VT.getStoreSize();
4171
4172 SDLoc SL(N);
4173 SelectionDAG &DAG = DCI.DAG;
4174 Align Alignment = SN->getAlign();
4175 if (Alignment < Size && isTypeLegal(VT)) {
4176 unsigned IsFast;
4177 unsigned AS = SN->getAddressSpace();
4178
4179 // Expand unaligned stores earlier than legalization. Due to visitation
4180 // order problems during legalization, the emitted instructions to pack and
4181 // unpack the bytes again are not eliminated in the case of an unaligned
4182 // copy.
4183 if (!allowsMisalignedMemoryAccesses(
4184 VT, AddrSpace: AS, Alignment, Flags: SN->getMemOperand()->getFlags(), &IsFast)) {
4185 if (VT.isVector())
4186 return SplitVectorStore(Op: SDValue(SN, 0), DAG);
4187
4188 return expandUnalignedStore(ST: SN, DAG);
4189 }
4190
4191 if (!IsFast)
4192 return SDValue();
4193 }
4194
4195 if (!shouldCombineMemoryType(VT))
4196 return SDValue();
4197
4198 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4199 SDValue Val = SN->getValue();
4200
4201 // DCI.AddToWorklist(Val.getNode());
4202
4203 bool OtherUses = !Val.hasOneUse();
4204 SDValue CastVal = DAG.getBitcast(VT: NewVT, V: Val);
4205 if (OtherUses) {
4206 SDValue CastBack = DAG.getBitcast(VT, V: CastVal);
4207 DAG.ReplaceAllUsesOfValueWith(From: Val, To: CastBack);
4208 }
4209
4210 return DAG.getStore(Chain: SN->getChain(), dl: SL, Val: CastVal,
4211 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
4212}
4213
4214// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
4215// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
4216// issues.
4217SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
4218 DAGCombinerInfo &DCI) const {
4219 SelectionDAG &DAG = DCI.DAG;
4220 SDValue N0 = N->getOperand(Num: 0);
4221
4222 // (vt2 (assertzext (truncate vt0:x), vt1)) ->
4223 // (vt2 (truncate (assertzext vt0:x, vt1)))
4224 if (N0.getOpcode() == ISD::TRUNCATE) {
4225 SDValue N1 = N->getOperand(Num: 1);
4226 EVT ExtVT = cast<VTSDNode>(Val&: N1)->getVT();
4227 SDLoc SL(N);
4228
4229 SDValue Src = N0.getOperand(i: 0);
4230 EVT SrcVT = Src.getValueType();
4231 if (SrcVT.bitsGE(VT: ExtVT)) {
4232 SDValue NewInReg = DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: SrcVT, N1: Src, N2: N1);
4233 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: N->getValueType(ResNo: 0), Operand: NewInReg);
4234 }
4235 }
4236
4237 return SDValue();
4238}
4239
4240SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
4241 SDNode *N, DAGCombinerInfo &DCI) const {
4242 unsigned IID = N->getConstantOperandVal(Num: 0);
4243 switch (IID) {
4244 case Intrinsic::amdgcn_mul_i24:
4245 case Intrinsic::amdgcn_mul_u24:
4246 case Intrinsic::amdgcn_mulhi_i24:
4247 case Intrinsic::amdgcn_mulhi_u24:
4248 return simplifyMul24(Node24: N, DCI);
4249 case Intrinsic::amdgcn_fract:
4250 case Intrinsic::amdgcn_rsq:
4251 case Intrinsic::amdgcn_rcp_legacy:
4252 case Intrinsic::amdgcn_rsq_legacy:
4253 case Intrinsic::amdgcn_rsq_clamp:
4254 case Intrinsic::amdgcn_tanh:
4255 case Intrinsic::amdgcn_prng_b32: {
4256 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
4257 SDValue Src = N->getOperand(Num: 1);
4258 return Src.isUndef() ? Src : SDValue();
4259 }
4260 case Intrinsic::amdgcn_frexp_exp: {
4261 // frexp_exp (fneg x) -> frexp_exp x
4262 // frexp_exp (fabs x) -> frexp_exp x
4263 // frexp_exp (fneg (fabs x)) -> frexp_exp x
4264 SDValue Src = N->getOperand(Num: 1);
4265 SDValue PeekSign = peekFPSignOps(Val: Src);
4266 if (PeekSign == Src)
4267 return SDValue();
4268 return SDValue(DCI.DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: PeekSign),
4269 0);
4270 }
4271 default:
4272 return SDValue();
4273 }
4274}
4275
4276/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
4277/// binary operation \p Opc to it with the corresponding constant operands.
4278SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
4279 DAGCombinerInfo &DCI, const SDLoc &SL,
4280 unsigned Opc, SDValue LHS,
4281 uint32_t ValLo, uint32_t ValHi) const {
4282 SelectionDAG &DAG = DCI.DAG;
4283 SDValue Lo, Hi;
4284 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: LHS, DAG);
4285
4286 SDValue LoRHS = DAG.getConstant(Val: ValLo, DL: SL, VT: MVT::i32);
4287 SDValue HiRHS = DAG.getConstant(Val: ValHi, DL: SL, VT: MVT::i32);
4288
4289 SDValue LoAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Lo, N2: LoRHS);
4290 SDValue HiAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Hi, N2: HiRHS);
4291
4292 // Re-visit the ands. It's possible we eliminated one of them and it could
4293 // simplify the vector.
4294 DCI.AddToWorklist(N: Lo.getNode());
4295 DCI.AddToWorklist(N: Hi.getNode());
4296
4297 SDValue Vec = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoAnd, HiAnd});
4298 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
4299}
4300
4301SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
4302 DAGCombinerInfo &DCI) const {
4303 EVT VT = N->getValueType(ResNo: 0);
4304 SDValue LHS = N->getOperand(Num: 0);
4305 SDValue RHS = N->getOperand(Num: 1);
4306 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4307 SDLoc SL(N);
4308 SelectionDAG &DAG = DCI.DAG;
4309
4310 unsigned RHSVal;
4311 if (CRHS) {
4312 RHSVal = CRHS->getZExtValue();
4313 if (!RHSVal)
4314 return LHS;
4315
4316 switch (LHS->getOpcode()) {
4317 default:
4318 break;
4319 case ISD::ZERO_EXTEND:
4320 case ISD::SIGN_EXTEND:
4321 case ISD::ANY_EXTEND: {
4322 SDValue X = LHS->getOperand(Num: 0);
4323
4324 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
4325 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16)) {
4326 // Prefer build_vector as the canonical form if packed types are legal.
4327 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
4328 SDValue Vec = DAG.getBuildVector(
4329 VT: MVT::v2i16, DL: SL,
4330 Ops: {DAG.getConstant(Val: 0, DL: SL, VT: MVT::i16), LHS->getOperand(Num: 0)});
4331 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Vec);
4332 }
4333
4334 // shl (ext x) => zext (shl x), if shift does not overflow int
4335 if (VT != MVT::i64)
4336 break;
4337 KnownBits Known = DAG.computeKnownBits(Op: X);
4338 unsigned LZ = Known.countMinLeadingZeros();
4339 if (LZ < RHSVal)
4340 break;
4341 EVT XVT = X.getValueType();
4342 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: XVT, N1: X, N2: SDValue(CRHS, 0));
4343 return DAG.getZExtOrTrunc(Op: Shl, DL: SL, VT);
4344 }
4345 }
4346 }
4347
4348 if (VT.getScalarType() != MVT::i64)
4349 return SDValue();
4350
4351 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4352 // common case, splitting this into a move and a 32-bit shift is faster and
4353 // the same code size.
4354 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4355
4356 EVT ElementType = VT.getScalarType();
4357 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4358 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4359
4360 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4361 return SDValue();
4362 SDValue ShiftAmt;
4363
4364 if (CRHS) {
4365 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4366 VT: TargetType);
4367 } else {
4368 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4369 const SDValue ShiftMask =
4370 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4371 // This AND instruction will clamp out of bounds shift values.
4372 // It will also be removed during later instruction selection.
4373 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4374 }
4375
4376 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: LHS);
4377 SDValue NewShift =
4378 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: TargetType, N1: Lo, N2: ShiftAmt, Flags: N->getFlags());
4379
4380 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4381 SDValue Vec;
4382
4383 if (VT.isVector()) {
4384 EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4385 unsigned NElts = TargetType.getVectorNumElements();
4386 SmallVector<SDValue, 8> HiOps;
4387 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4388
4389 DAG.ExtractVectorElements(Op: NewShift, Args&: HiOps, Start: 0, Count: NElts);
4390 for (unsigned I = 0; I != NElts; ++I)
4391 HiAndLoOps[2 * I + 1] = HiOps[I];
4392 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4393 } else {
4394 EVT ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4395 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {Zero, NewShift});
4396 }
4397 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4398}
4399
4400SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
4401 DAGCombinerInfo &DCI) const {
4402 SDValue RHS = N->getOperand(Num: 1);
4403 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4404 EVT VT = N->getValueType(ResNo: 0);
4405 SDValue LHS = N->getOperand(Num: 0);
4406 SelectionDAG &DAG = DCI.DAG;
4407 SDLoc SL(N);
4408
4409 if (VT.getScalarType() != MVT::i64)
4410 return SDValue();
4411
4412 // For C >= 32
4413 // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
4414
4415 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4416 // common case, splitting this into a move and a 32-bit shift is faster and
4417 // the same code size.
4418 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4419
4420 EVT ElementType = VT.getScalarType();
4421 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4422 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4423
4424 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4425 return SDValue();
4426
4427 SDValue ShiftFullAmt =
4428 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4429 SDValue ShiftAmt;
4430 if (CRHS) {
4431 unsigned RHSVal = CRHS->getZExtValue();
4432 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4433 VT: TargetType);
4434 } else if (Known.getMinValue().getZExtValue() ==
4435 (ElementType.getSizeInBits() - 1)) {
4436 ShiftAmt = ShiftFullAmt;
4437 } else {
4438 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4439 const SDValue ShiftMask =
4440 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4441 // This AND instruction will clamp out of bounds shift values.
4442 // It will also be removed during later instruction selection.
4443 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4444 }
4445
4446 EVT ConcatType;
4447 SDValue Hi;
4448 SDLoc LHSSL(LHS);
4449 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4450 if (VT.isVector()) {
4451 unsigned NElts = TargetType.getVectorNumElements();
4452 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4453 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4454 SmallVector<SDValue, 8> HiOps(NElts);
4455 SmallVector<SDValue, 16> HiAndLoOps;
4456
4457 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, Start: 0, Count: NElts * 2);
4458 for (unsigned I = 0; I != NElts; ++I) {
4459 HiOps[I] = HiAndLoOps[2 * I + 1];
4460 }
4461 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4462 } else {
4463 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4464 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4465 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4466 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4467 }
4468
4469 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
4470 SDValue NewShift, HiShift;
4471 if (KnownLHS.isNegative()) {
4472 HiShift = DAG.getAllOnesConstant(DL: SL, VT: TargetType);
4473 NewShift =
4474 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4475 } else if (CRHS &&
4476 CRHS->getZExtValue() == (ElementType.getSizeInBits() - 1)) {
4477 NewShift = HiShift =
4478 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4479 } else {
4480 Hi = DAG.getFreeze(V: Hi);
4481 HiShift = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftFullAmt);
4482 NewShift =
4483 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4484 }
4485
4486 SDValue Vec;
4487 if (VT.isVector()) {
4488 unsigned NElts = TargetType.getVectorNumElements();
4489 SmallVector<SDValue, 8> HiOps;
4490 SmallVector<SDValue, 8> LoOps;
4491 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
4492
4493 DAG.ExtractVectorElements(Op: HiShift, Args&: HiOps, Start: 0, Count: NElts);
4494 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4495 for (unsigned I = 0; I != NElts; ++I) {
4496 HiAndLoOps[2 * I + 1] = HiOps[I];
4497 HiAndLoOps[2 * I] = LoOps[I];
4498 }
4499 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4500 } else {
4501 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, HiShift});
4502 }
4503 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4504}
4505
4506SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
4507 DAGCombinerInfo &DCI) const {
4508 SDValue RHS = N->getOperand(Num: 1);
4509 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4510 EVT VT = N->getValueType(ResNo: 0);
4511 SDValue LHS = N->getOperand(Num: 0);
4512 SelectionDAG &DAG = DCI.DAG;
4513 SDLoc SL(N);
4514 unsigned RHSVal;
4515
4516 if (CRHS) {
4517 RHSVal = CRHS->getZExtValue();
4518
4519 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
4520 // this improves the ability to match BFE patterns in isel.
4521 if (LHS.getOpcode() == ISD::AND) {
4522 if (auto *Mask = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
4523 unsigned MaskIdx, MaskLen;
4524 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4525 MaskIdx == RHSVal) {
4526 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT,
4527 N1: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 0),
4528 N2: N->getOperand(Num: 1)),
4529 N2: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 1),
4530 N2: N->getOperand(Num: 1)));
4531 }
4532 }
4533 }
4534 }
4535
4536 if (VT.getScalarType() != MVT::i64)
4537 return SDValue();
4538
4539 // for C >= 32
4540 // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
4541
4542 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4543 // common case, splitting this into a move and a 32-bit shift is faster and
4544 // the same code size.
4545 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4546
4547 EVT ElementType = VT.getScalarType();
4548 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4549 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4550
4551 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4552 return SDValue();
4553
4554 SDValue ShiftAmt;
4555 if (CRHS) {
4556 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4557 VT: TargetType);
4558 } else {
4559 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4560 const SDValue ShiftMask =
4561 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4562 // This AND instruction will clamp out of bounds shift values.
4563 // It will also be removed during later instruction selection.
4564 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4565 }
4566
4567 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4568 EVT ConcatType;
4569 SDValue Hi;
4570 SDLoc LHSSL(LHS);
4571 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4572 if (VT.isVector()) {
4573 unsigned NElts = TargetType.getVectorNumElements();
4574 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4575 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4576 SmallVector<SDValue, 8> HiOps(NElts);
4577 SmallVector<SDValue, 16> HiAndLoOps;
4578
4579 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, /*Start=*/0, Count: NElts * 2);
4580 for (unsigned I = 0; I != NElts; ++I)
4581 HiOps[I] = HiAndLoOps[2 * I + 1];
4582 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4583 } else {
4584 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4585 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4586 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4587 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4588 }
4589
4590 SDValue NewShift =
4591 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4592
4593 SDValue Vec;
4594 if (VT.isVector()) {
4595 unsigned NElts = TargetType.getVectorNumElements();
4596 SmallVector<SDValue, 8> LoOps;
4597 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4598
4599 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4600 for (unsigned I = 0; I != NElts; ++I)
4601 HiAndLoOps[2 * I] = LoOps[I];
4602 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4603 } else {
4604 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, Zero});
4605 }
4606 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4607}
4608
4609SDValue AMDGPUTargetLowering::performTruncateCombine(
4610 SDNode *N, DAGCombinerInfo &DCI) const {
4611 SDLoc SL(N);
4612 SelectionDAG &DAG = DCI.DAG;
4613 EVT VT = N->getValueType(ResNo: 0);
4614 SDValue Src = N->getOperand(Num: 0);
4615
4616 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
4617 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
4618 SDValue Vec = Src.getOperand(i: 0);
4619 if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
4620 SDValue Elt0 = Vec.getOperand(i: 0);
4621 EVT EltVT = Elt0.getValueType();
4622 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
4623 if (EltVT.isFloatingPoint()) {
4624 Elt0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL,
4625 VT: EltVT.changeTypeToInteger(), Operand: Elt0);
4626 }
4627
4628 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Elt0);
4629 }
4630 }
4631 }
4632
4633 // Equivalent of above for accessing the high element of a vector as an
4634 // integer operation.
4635 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
4636 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
4637 if (auto *K = isConstOrConstSplat(N: Src.getOperand(i: 1))) {
4638 SDValue BV = stripBitcast(Val: Src.getOperand(i: 0));
4639 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
4640 EVT SrcEltVT = BV.getOperand(i: 0).getValueType();
4641 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
4642 unsigned BitIndex = K->getZExtValue();
4643 unsigned PartIndex = BitIndex / SrcEltSize;
4644
4645 if (PartIndex * SrcEltSize == BitIndex &&
4646 PartIndex < BV.getNumOperands()) {
4647 if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
4648 SDValue SrcElt =
4649 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcEltVT.changeTypeToInteger(),
4650 Operand: BV.getOperand(i: PartIndex));
4651 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: SrcElt);
4652 }
4653 }
4654 }
4655 }
4656 }
4657
4658 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
4659 //
4660 // i16 (trunc (srl i64:x, K)), K <= 16 ->
4661 // i16 (trunc (srl (i32 (trunc x), K)))
4662 if (VT.getScalarSizeInBits() < 32) {
4663 EVT SrcVT = Src.getValueType();
4664 if (SrcVT.getScalarSizeInBits() > 32 &&
4665 (Src.getOpcode() == ISD::SRL ||
4666 Src.getOpcode() == ISD::SRA ||
4667 Src.getOpcode() == ISD::SHL)) {
4668 SDValue Amt = Src.getOperand(i: 1);
4669 KnownBits Known = DAG.computeKnownBits(Op: Amt);
4670
4671 // - For left shifts, do the transform as long as the shift
4672 // amount is still legal for i32, so when ShiftAmt < 32 (<= 31)
4673 // - For right shift, do it if ShiftAmt <= (32 - Size) to avoid
4674 // losing information stored in the high bits when truncating.
4675 const unsigned MaxCstSize =
4676 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
4677 if (Known.getMaxValue().ule(RHS: MaxCstSize)) {
4678 EVT MidVT = VT.isVector() ?
4679 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
4680 NumElements: VT.getVectorNumElements()) : MVT::i32;
4681
4682 EVT NewShiftVT = getShiftAmountTy(LHSTy: MidVT, DL: DAG.getDataLayout());
4683 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MidVT,
4684 Operand: Src.getOperand(i: 0));
4685 DCI.AddToWorklist(N: Trunc.getNode());
4686
4687 if (Amt.getValueType() != NewShiftVT) {
4688 Amt = DAG.getZExtOrTrunc(Op: Amt, DL: SL, VT: NewShiftVT);
4689 DCI.AddToWorklist(N: Amt.getNode());
4690 }
4691
4692 SDValue ShrunkShift = DAG.getNode(Opcode: Src.getOpcode(), DL: SL, VT: MidVT,
4693 N1: Trunc, N2: Amt);
4694 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: ShrunkShift);
4695 }
4696 }
4697 }
4698
4699 return SDValue();
4700}
4701
4702// We need to specifically handle i64 mul here to avoid unnecessary conversion
4703// instructions. If we only match on the legalized i64 mul expansion,
4704// SimplifyDemandedBits will be unable to remove them because there will be
4705// multiple uses due to the separate mul + mulh[su].
4706static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
4707 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
4708 if (Size <= 32) {
4709 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4710 return DAG.getNode(Opcode: MulOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4711 }
4712
4713 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4714 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
4715
4716 SDValue MulLo = DAG.getNode(Opcode: MulLoOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4717 SDValue MulHi = DAG.getNode(Opcode: MulHiOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4718
4719 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: MulLo, N2: MulHi);
4720}
4721
4722/// If \p V is an add of a constant 1, returns the other operand. Otherwise
4723/// return SDValue().
4724static SDValue getAddOneOp(const SDNode *V) {
4725 if (V->getOpcode() != ISD::ADD)
4726 return SDValue();
4727
4728 return isOneConstant(V: V->getOperand(Num: 1)) ? V->getOperand(Num: 0) : SDValue();
4729}
4730
4731SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
4732 DAGCombinerInfo &DCI) const {
4733 assert(N->getOpcode() == ISD::MUL);
4734 EVT VT = N->getValueType(ResNo: 0);
4735
4736 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4737 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4738 // unnecessarily). isDivergent() is used as an approximation of whether the
4739 // value is in an SGPR.
4740 if (!N->isDivergent())
4741 return SDValue();
4742
4743 unsigned Size = VT.getSizeInBits();
4744 if (VT.isVector() || Size > 64)
4745 return SDValue();
4746
4747 SelectionDAG &DAG = DCI.DAG;
4748 SDLoc DL(N);
4749
4750 SDValue N0 = N->getOperand(Num: 0);
4751 SDValue N1 = N->getOperand(Num: 1);
4752
4753 // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
4754 // matching.
4755
4756 // mul x, (add y, 1) -> add (mul x, y), x
4757 auto IsFoldableAdd = [](SDValue V) -> SDValue {
4758 SDValue AddOp = getAddOneOp(V: V.getNode());
4759 if (!AddOp)
4760 return SDValue();
4761
4762 if (V.hasOneUse() || all_of(Range: V->users(), P: [](const SDNode *U) -> bool {
4763 return U->getOpcode() == ISD::MUL;
4764 }))
4765 return AddOp;
4766
4767 return SDValue();
4768 };
4769
4770 // FIXME: The selection pattern is not properly checking for commuted
4771 // operands, so we have to place the mul in the LHS
4772 if (SDValue MulOper = IsFoldableAdd(N0)) {
4773 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1, N2: MulOper);
4774 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N1);
4775 }
4776
4777 if (SDValue MulOper = IsFoldableAdd(N1)) {
4778 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N0, N2: MulOper);
4779 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N0);
4780 }
4781
4782 // There are i16 integer mul/mad.
4783 if (isTypeLegal(VT: MVT::i16) && VT.getScalarType().bitsLE(VT: MVT::i16))
4784 return SDValue();
4785
4786 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4787 // in the source into any_extends if the result of the mul is truncated. Since
4788 // we can assume the high bits are whatever we want, use the underlying value
4789 // to avoid the unknown high bits from interfering.
4790 if (N0.getOpcode() == ISD::ANY_EXTEND)
4791 N0 = N0.getOperand(i: 0);
4792
4793 if (N1.getOpcode() == ISD::ANY_EXTEND)
4794 N1 = N1.getOperand(i: 0);
4795
4796 SDValue Mul;
4797
4798 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4799 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4800 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4801 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: false);
4802 } else if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4803 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4804 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4805 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: true);
4806 } else {
4807 return SDValue();
4808 }
4809
4810 // We need to use sext even for MUL_U24, because MUL_U24 is used
4811 // for signed multiply of 8 and 16-bit types.
4812 return DAG.getSExtOrTrunc(Op: Mul, DL, VT);
4813}
4814
4815SDValue
4816AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N,
4817 DAGCombinerInfo &DCI) const {
4818 if (N->getValueType(ResNo: 0) != MVT::i32)
4819 return SDValue();
4820
4821 SelectionDAG &DAG = DCI.DAG;
4822 SDLoc DL(N);
4823
4824 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
4825 SDValue N0 = N->getOperand(Num: 0);
4826 SDValue N1 = N->getOperand(Num: 1);
4827
4828 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4829 // in the source into any_extends if the result of the mul is truncated. Since
4830 // we can assume the high bits are whatever we want, use the underlying value
4831 // to avoid the unknown high bits from interfering.
4832 if (N0.getOpcode() == ISD::ANY_EXTEND)
4833 N0 = N0.getOperand(i: 0);
4834 if (N1.getOpcode() == ISD::ANY_EXTEND)
4835 N1 = N1.getOperand(i: 0);
4836
4837 // Try to use two fast 24-bit multiplies (one for each half of the result)
4838 // instead of one slow extending multiply.
4839 unsigned LoOpcode = 0;
4840 unsigned HiOpcode = 0;
4841 if (Signed) {
4842 if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4843 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4844 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4845 LoOpcode = AMDGPUISD::MUL_I24;
4846 HiOpcode = AMDGPUISD::MULHI_I24;
4847 }
4848 } else {
4849 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4850 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4851 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4852 LoOpcode = AMDGPUISD::MUL_U24;
4853 HiOpcode = AMDGPUISD::MULHI_U24;
4854 }
4855 }
4856 if (!LoOpcode)
4857 return SDValue();
4858
4859 SDValue Lo = DAG.getNode(Opcode: LoOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4860 SDValue Hi = DAG.getNode(Opcode: HiOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4861 DCI.CombineTo(N, Res0: Lo, Res1: Hi);
4862 return SDValue(N, 0);
4863}
4864
4865SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
4866 DAGCombinerInfo &DCI) const {
4867 EVT VT = N->getValueType(ResNo: 0);
4868
4869 if (!Subtarget->hasMulI24() || VT.isVector())
4870 return SDValue();
4871
4872 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4873 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4874 // unnecessarily). isDivergent() is used as an approximation of whether the
4875 // value is in an SGPR.
4876 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4877 // valu op anyway)
4878 if (Subtarget->hasSMulHi() && !N->isDivergent())
4879 return SDValue();
4880
4881 SelectionDAG &DAG = DCI.DAG;
4882 SDLoc DL(N);
4883
4884 SDValue N0 = N->getOperand(Num: 0);
4885 SDValue N1 = N->getOperand(Num: 1);
4886
4887 if (!isI24(Op: N0, DAG) || !isI24(Op: N1, DAG))
4888 return SDValue();
4889
4890 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4891 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4892
4893 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_I24, DL, VT: MVT::i32, N1: N0, N2: N1);
4894 DCI.AddToWorklist(N: Mulhi.getNode());
4895 return DAG.getSExtOrTrunc(Op: Mulhi, DL, VT);
4896}
4897
4898SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
4899 DAGCombinerInfo &DCI) const {
4900 EVT VT = N->getValueType(ResNo: 0);
4901
4902 if (VT.isVector() || VT.getSizeInBits() > 32 || !Subtarget->hasMulU24())
4903 return SDValue();
4904
4905 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4906 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4907 // unnecessarily). isDivergent() is used as an approximation of whether the
4908 // value is in an SGPR.
4909 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4910 // valu op anyway)
4911 if (!N->isDivergent() && Subtarget->hasSMulHi())
4912 return SDValue();
4913
4914 SelectionDAG &DAG = DCI.DAG;
4915 SDLoc DL(N);
4916
4917 SDValue N0 = N->getOperand(Num: 0);
4918 SDValue N1 = N->getOperand(Num: 1);
4919
4920 if (!isU24(Op: N0, DAG) || !isU24(Op: N1, DAG))
4921 return SDValue();
4922
4923 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4924 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4925
4926 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_U24, DL, VT: MVT::i32, N1: N0, N2: N1);
4927 DCI.AddToWorklist(N: Mulhi.getNode());
4928 return DAG.getZExtOrTrunc(Op: Mulhi, DL, VT);
4929}
4930
4931SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
4932 SDValue Op,
4933 const SDLoc &DL,
4934 unsigned Opc) const {
4935 EVT VT = Op.getValueType();
4936 if (VT.bitsGT(VT: MVT::i32))
4937 return SDValue();
4938
4939 if (VT != MVT::i32)
4940 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: Op);
4941
4942 SDValue FFBX = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, Operand: Op);
4943 if (VT != MVT::i32)
4944 FFBX = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FFBX);
4945
4946 return FFBX;
4947}
4948
4949// The native instructions return -1 on 0 input. Optimize out a select that
4950// produces -1 on 0.
4951//
4952// TODO: If zero is not undef, we could also do this if the output is compared
4953// against the bitwidth.
4954//
4955// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
4956SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
4957 SDValue LHS, SDValue RHS,
4958 DAGCombinerInfo &DCI) const {
4959 if (!isNullConstant(V: Cond.getOperand(i: 1)))
4960 return SDValue();
4961
4962 SelectionDAG &DAG = DCI.DAG;
4963 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
4964 SDValue CmpLHS = Cond.getOperand(i: 0);
4965
4966 // select (setcc x, 0, eq), -1, (ctlz_zero_poison x) -> ffbh_u32 x
4967 // select (setcc x, 0, eq), -1, (cttz_zero_poison x) -> ffbl_u32 x
4968 if (CCOpcode == ISD::SETEQ &&
4969 (isCtlzOpc(Opc: RHS.getOpcode()) || isCttzOpc(Opc: RHS.getOpcode())) &&
4970 RHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: LHS)) {
4971 unsigned Opc =
4972 isCttzOpc(Opc: RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4973 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4974 }
4975
4976 // select (setcc x, 0, ne), (ctlz_zero_poison x), -1 -> ffbh_u32 x
4977 // select (setcc x, 0, ne), (cttz_zero_poison x), -1 -> ffbl_u32 x
4978 if (CCOpcode == ISD::SETNE &&
4979 (isCtlzOpc(Opc: LHS.getOpcode()) || isCttzOpc(Opc: LHS.getOpcode())) &&
4980 LHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: RHS)) {
4981 unsigned Opc =
4982 isCttzOpc(Opc: LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4983
4984 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4985 }
4986
4987 return SDValue();
4988}
4989
4990static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
4991 unsigned Op,
4992 const SDLoc &SL,
4993 SDValue Cond,
4994 SDValue N1,
4995 SDValue N2) {
4996 SelectionDAG &DAG = DCI.DAG;
4997 EVT VT = N1.getValueType();
4998
4999 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cond,
5000 N2: N1.getOperand(i: 0), N3: N2.getOperand(i: 0));
5001 DCI.AddToWorklist(N: NewSelect.getNode());
5002 return DAG.getNode(Opcode: Op, DL: SL, VT, Operand: NewSelect);
5003}
5004
5005// Pull a free FP operation out of a select so it may fold into uses.
5006//
5007// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
5008// select c, (fneg x), k -> fneg (select c, x, (fneg k))
5009//
5010// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
5011// select c, (fabs x), +k -> fabs (select c, x, k)
5012SDValue
5013AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
5014 SDValue N) const {
5015 SelectionDAG &DAG = DCI.DAG;
5016 SDValue Cond = N.getOperand(i: 0);
5017 SDValue LHS = N.getOperand(i: 1);
5018 SDValue RHS = N.getOperand(i: 2);
5019
5020 EVT VT = N.getValueType();
5021 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
5022 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
5023 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5024 return SDValue();
5025
5026 return distributeOpThroughSelect(DCI, Op: LHS.getOpcode(),
5027 SL: SDLoc(N), Cond, N1: LHS, N2: RHS);
5028 }
5029
5030 bool Inv = false;
5031 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
5032 std::swap(a&: LHS, b&: RHS);
5033 Inv = true;
5034 }
5035
5036 // TODO: Support vector constants.
5037 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
5038 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS &&
5039 !selectSupportsSourceMods(N: N.getNode())) {
5040 SDLoc SL(N);
5041 // If one side is an fneg/fabs and the other is a constant, we can push the
5042 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
5043 SDValue NewLHS = LHS.getOperand(i: 0);
5044 SDValue NewRHS = RHS;
5045
5046 // Careful: if the neg can be folded up, don't try to pull it back down.
5047 bool ShouldFoldNeg = true;
5048
5049 if (NewLHS.hasOneUse()) {
5050 unsigned Opc = NewLHS.getOpcode();
5051 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(N: NewLHS.getNode()))
5052 ShouldFoldNeg = false;
5053 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
5054 ShouldFoldNeg = false;
5055 }
5056
5057 if (ShouldFoldNeg) {
5058 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative())
5059 return SDValue();
5060
5061 // We're going to be forced to use a source modifier anyway, there's no
5062 // point to pulling the negate out unless we can get a size reduction by
5063 // negating the constant.
5064 //
5065 // TODO: Generalize to use getCheaperNegatedExpression which doesn't know
5066 // about cheaper constants.
5067 if (NewLHS.getOpcode() == ISD::FABS &&
5068 getConstantNegateCost(C: CRHS) != NegatibleCost::Cheaper)
5069 return SDValue();
5070
5071 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5072 return SDValue();
5073
5074 if (LHS.getOpcode() == ISD::FNEG)
5075 NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5076
5077 if (Inv)
5078 std::swap(a&: NewLHS, b&: NewRHS);
5079
5080 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT,
5081 N1: Cond, N2: NewLHS, N3: NewRHS);
5082 DCI.AddToWorklist(N: NewSelect.getNode());
5083 return DAG.getNode(Opcode: LHS.getOpcode(), DL: SL, VT, Operand: NewSelect);
5084 }
5085 }
5086
5087 return SDValue();
5088}
5089
5090SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
5091 DAGCombinerInfo &DCI) const {
5092 if (SDValue Folded = foldFreeOpFromSelect(DCI, N: SDValue(N, 0)))
5093 return Folded;
5094
5095 SDValue Cond = N->getOperand(Num: 0);
5096 if (Cond.getOpcode() != ISD::SETCC)
5097 return SDValue();
5098
5099 EVT VT = N->getValueType(ResNo: 0);
5100 SDValue LHS = Cond.getOperand(i: 0);
5101 SDValue RHS = Cond.getOperand(i: 1);
5102 SDValue CC = Cond.getOperand(i: 2);
5103
5104 SDValue True = N->getOperand(Num: 1);
5105 SDValue False = N->getOperand(Num: 2);
5106
5107 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
5108 SelectionDAG &DAG = DCI.DAG;
5109 if (DAG.isConstantValueOfAnyType(N: True) &&
5110 !DAG.isConstantValueOfAnyType(N: False)) {
5111 // Swap cmp + select pair to move constant to false input.
5112 // This will allow using VOPC cndmasks more often.
5113 // select (setcc x, y), k, x -> select (setccinv x, y), x, k
5114
5115 SDLoc SL(N);
5116 ISD::CondCode NewCC =
5117 getSetCCInverse(Operation: cast<CondCodeSDNode>(Val&: CC)->get(), Type: LHS.getValueType());
5118
5119 SDValue NewCond = DAG.getSetCC(DL: SL, VT: Cond.getValueType(), LHS, RHS, Cond: NewCC);
5120 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NewCond, N2: False, N3: True);
5121 }
5122
5123 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
5124 SDValue MinMax = combineFMinMaxLegacy(DL: SDLoc(N), VT, LHS, RHS, True, False,
5125 CC, Flags: N->getFlags(), DCI);
5126 // Revisit this node so we can catch min3/max3/med3 patterns.
5127 //DCI.AddToWorklist(MinMax.getNode());
5128 return MinMax;
5129 }
5130 }
5131
5132 // There's no reason to not do this if the condition has other uses.
5133 return performCtlz_CttzCombine(SL: SDLoc(N), Cond, LHS: True, RHS: False, DCI);
5134}
5135
5136static bool isInv2Pi(const APFloat &APF) {
5137 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
5138 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
5139 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
5140
5141 return APF.bitwiseIsEqual(RHS: KF16) ||
5142 APF.bitwiseIsEqual(RHS: KF32) ||
5143 APF.bitwiseIsEqual(RHS: KF64);
5144}
5145
5146// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
5147// additional cost to negate them.
5148TargetLowering::NegatibleCost
5149AMDGPUTargetLowering::getConstantNegateCost(const ConstantFPSDNode *C) const {
5150 if (C->isZero())
5151 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5152
5153 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(APF: C->getValueAPF()))
5154 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5155
5156 return NegatibleCost::Neutral;
5157}
5158
5159bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
5160 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5161 return getConstantNegateCost(C) == NegatibleCost::Expensive;
5162 return false;
5163}
5164
5165static unsigned inverseMinMax(unsigned Opc) {
5166 switch (Opc) {
5167 case ISD::FMAXNUM:
5168 return ISD::FMINNUM;
5169 case ISD::FMINNUM:
5170 return ISD::FMAXNUM;
5171 case ISD::FMAXNUM_IEEE:
5172 return ISD::FMINNUM_IEEE;
5173 case ISD::FMINNUM_IEEE:
5174 return ISD::FMAXNUM_IEEE;
5175 case ISD::FMAXIMUM:
5176 return ISD::FMINIMUM;
5177 case ISD::FMINIMUM:
5178 return ISD::FMAXIMUM;
5179 case ISD::FMAXIMUMNUM:
5180 return ISD::FMINIMUMNUM;
5181 case ISD::FMINIMUMNUM:
5182 return ISD::FMAXIMUMNUM;
5183 case AMDGPUISD::FMAX_LEGACY:
5184 return AMDGPUISD::FMIN_LEGACY;
5185 case AMDGPUISD::FMIN_LEGACY:
5186 return AMDGPUISD::FMAX_LEGACY;
5187 default:
5188 llvm_unreachable("invalid min/max opcode");
5189 }
5190}
5191
5192/// \return true if it's profitable to try to push an fneg into its source
5193/// instruction.
5194bool AMDGPUTargetLowering::shouldFoldFNegIntoSrc(SDNode *N, SDValue N0) {
5195 // If the input has multiple uses and we can either fold the negate down, or
5196 // the other uses cannot, give up. This both prevents unprofitable
5197 // transformations and infinite loops: we won't repeatedly try to fold around
5198 // a negate that has no 'good' form.
5199 if (N0.hasOneUse()) {
5200 // This may be able to fold into the source, but at a code size cost. Don't
5201 // fold if the fold into the user is free.
5202 if (allUsesHaveSourceMods(N, CostThreshold: 0))
5203 return false;
5204 } else {
5205 if (fnegFoldsIntoOp(N: N0.getNode()) &&
5206 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N: N0.getNode())))
5207 return false;
5208 }
5209
5210 return true;
5211}
5212
5213SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
5214 DAGCombinerInfo &DCI) const {
5215 SelectionDAG &DAG = DCI.DAG;
5216 SDValue N0 = N->getOperand(Num: 0);
5217 EVT VT = N->getValueType(ResNo: 0);
5218
5219 unsigned Opc = N0.getOpcode();
5220
5221 if (!shouldFoldFNegIntoSrc(N, N0))
5222 return SDValue();
5223
5224 bool MayIgnoreSignedZeroForAllUses =
5225 N0->getFlags().hasNoSignedZeros() ||
5226 (N0.hasOneUse() && N->getFlags().hasNoSignedZeros());
5227
5228 SDLoc SL(N);
5229 switch (Opc) {
5230 case ISD::FADD: {
5231 if (!MayIgnoreSignedZeroForAllUses)
5232 return SDValue();
5233
5234 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
5235 SDValue LHS = N0.getOperand(i: 0);
5236 SDValue RHS = N0.getOperand(i: 1);
5237
5238 if (LHS.getOpcode() != ISD::FNEG)
5239 LHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5240 else
5241 LHS = LHS.getOperand(i: 0);
5242
5243 if (RHS.getOpcode() != ISD::FNEG)
5244 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5245 else
5246 RHS = RHS.getOperand(i: 0);
5247
5248 SDValue Res = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5249 if (Res.getOpcode() != ISD::FADD)
5250 return SDValue(); // Op got folded away.
5251 if (!N0.hasOneUse())
5252 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5253 return Res;
5254 }
5255 case ISD::FMUL:
5256 case AMDGPUISD::FMUL_LEGACY: {
5257 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
5258 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
5259 SDValue LHS = N0.getOperand(i: 0);
5260 SDValue RHS = N0.getOperand(i: 1);
5261
5262 if (LHS.getOpcode() == ISD::FNEG)
5263 LHS = LHS.getOperand(i: 0);
5264 else if (RHS.getOpcode() == ISD::FNEG)
5265 RHS = RHS.getOperand(i: 0);
5266 else
5267 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5268
5269 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5270 if (Res.getOpcode() != Opc)
5271 return SDValue(); // Op got folded away.
5272 if (!N0.hasOneUse())
5273 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5274 return Res;
5275 }
5276 case ISD::FMA:
5277 case ISD::FMAD: {
5278 // TODO: handle llvm.amdgcn.fma.legacy
5279 if (!MayIgnoreSignedZeroForAllUses)
5280 return SDValue();
5281
5282 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
5283 SDValue LHS = N0.getOperand(i: 0);
5284 SDValue MHS = N0.getOperand(i: 1);
5285 SDValue RHS = N0.getOperand(i: 2);
5286
5287 if (LHS.getOpcode() == ISD::FNEG)
5288 LHS = LHS.getOperand(i: 0);
5289 else if (MHS.getOpcode() == ISD::FNEG)
5290 MHS = MHS.getOperand(i: 0);
5291 else
5292 MHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: MHS);
5293
5294 if (RHS.getOpcode() != ISD::FNEG)
5295 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5296 else
5297 RHS = RHS.getOperand(i: 0);
5298
5299 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: MHS, N3: RHS);
5300 if (Res.getOpcode() != Opc)
5301 return SDValue(); // Op got folded away.
5302 if (!N0.hasOneUse())
5303 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5304 return Res;
5305 }
5306 case ISD::FMAXNUM:
5307 case ISD::FMINNUM:
5308 case ISD::FMAXNUM_IEEE:
5309 case ISD::FMINNUM_IEEE:
5310 case ISD::FMINIMUM:
5311 case ISD::FMAXIMUM:
5312 case ISD::FMINIMUMNUM:
5313 case ISD::FMAXIMUMNUM:
5314 case AMDGPUISD::FMAX_LEGACY:
5315 case AMDGPUISD::FMIN_LEGACY: {
5316 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
5317 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
5318 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
5319 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
5320
5321 SDValue LHS = N0.getOperand(i: 0);
5322 SDValue RHS = N0.getOperand(i: 1);
5323
5324 // 0 doesn't have a negated inline immediate.
5325 // TODO: This constant check should be generalized to other operations.
5326 if (isConstantCostlierToNegate(N: RHS))
5327 return SDValue();
5328
5329 // Swapping min<->max flips which operand a signed zero tie selects.
5330 if ((Opc == AMDGPUISD::FMIN_LEGACY || Opc == AMDGPUISD::FMAX_LEGACY) &&
5331 !canIgnoreLegacyMinMaxTies(DAG, Flags: N0->getFlags(), LHS, RHS))
5332 return SDValue();
5333
5334 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5335 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5336 unsigned Opposite = inverseMinMax(Opc);
5337
5338 SDValue Res = DAG.getNode(Opcode: Opposite, DL: SL, VT, N1: NegLHS, N2: NegRHS, Flags: N0->getFlags());
5339 if (Res.getOpcode() != Opposite)
5340 return SDValue(); // Op got folded away.
5341 if (!N0.hasOneUse())
5342 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5343 return Res;
5344 }
5345 case AMDGPUISD::FMED3: {
5346 // med3 sorts a NaN input as smaller than everything regardless of its sign,
5347 // so negating all operands does not sign-flip the median when an input may
5348 // be NaN.
5349 if (!N0->getFlags().hasNoNaNs())
5350 return SDValue();
5351
5352 SDValue Ops[3];
5353 for (unsigned I = 0; I < 3; ++I)
5354 Ops[I] = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N0->getOperand(Num: I), Flags: N0->getFlags());
5355
5356 SDValue Res = DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT, Ops, Flags: N0->getFlags());
5357 if (Res.getOpcode() != AMDGPUISD::FMED3)
5358 return SDValue(); // Op got folded away.
5359
5360 if (!N0.hasOneUse()) {
5361 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res);
5362 DAG.ReplaceAllUsesWith(From: N0, To: Neg);
5363
5364 for (SDNode *U : Neg->users())
5365 DCI.AddToWorklist(N: U);
5366 }
5367
5368 return Res;
5369 }
5370 case ISD::FP_EXTEND:
5371 case ISD::FTRUNC:
5372 case ISD::FRINT:
5373 case ISD::FNEARBYINT: // XXX - Should fround be handled?
5374 case ISD::FROUNDEVEN:
5375 case ISD::FSIN:
5376 case ISD::FCANONICALIZE:
5377 case AMDGPUISD::RCP:
5378 case AMDGPUISD::RCP_LEGACY:
5379 case AMDGPUISD::RCP_IFLAG:
5380 case AMDGPUISD::SIN_HW: {
5381 SDValue CvtSrc = N0.getOperand(i: 0);
5382 if (CvtSrc.getOpcode() == ISD::FNEG) {
5383 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
5384 // (fneg (rcp (fneg x))) -> (rcp x)
5385 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: CvtSrc.getOperand(i: 0));
5386 }
5387
5388 if (!N0.hasOneUse())
5389 return SDValue();
5390
5391 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
5392 // (fneg (rcp x)) -> (rcp (fneg x))
5393 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5394 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: Neg, Flags: N0->getFlags());
5395 }
5396 case ISD::FP_ROUND: {
5397 SDValue CvtSrc = N0.getOperand(i: 0);
5398
5399 if (CvtSrc.getOpcode() == ISD::FNEG) {
5400 // (fneg (fp_round (fneg x))) -> (fp_round x)
5401 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT,
5402 N1: CvtSrc.getOperand(i: 0), N2: N0.getOperand(i: 1));
5403 }
5404
5405 if (!N0.hasOneUse())
5406 return SDValue();
5407
5408 // (fneg (fp_round x)) -> (fp_round (fneg x))
5409 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5410 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Neg, N2: N0.getOperand(i: 1));
5411 }
5412 case ISD::FP16_TO_FP: {
5413 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
5414 // f16, but legalization of f16 fneg ends up pulling it out of the source.
5415 // Put the fneg back as a legal source operation that can be matched later.
5416 SDLoc SL(N);
5417
5418 SDValue Src = N0.getOperand(i: 0);
5419 EVT SrcVT = Src.getValueType();
5420
5421 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
5422 SDValue IntFNeg = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: SrcVT, N1: Src,
5423 N2: DAG.getConstant(Val: 0x8000, DL: SL, VT: SrcVT));
5424 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFNeg);
5425 }
5426 case ISD::SELECT: {
5427 // fneg (select c, a, b) -> select c, (fneg a), (fneg b)
5428 // TODO: Invert conditions of foldFreeOpFromSelect
5429 return SDValue();
5430 }
5431 case ISD::BITCAST: {
5432 SDLoc SL(N);
5433 SDValue BCSrc = N0.getOperand(i: 0);
5434 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
5435 SDValue HighBits = BCSrc.getOperand(i: BCSrc.getNumOperands() - 1);
5436 if (VT != MVT::f64 || HighBits.getValueType().getSizeInBits() != 32 ||
5437 !fnegFoldsIntoOp(N: HighBits.getNode()))
5438 return SDValue();
5439
5440 // f64 fneg only really needs to operate on the high half of of the
5441 // register, so try to force it to an f32 operation to help make use of
5442 // source modifiers.
5443 //
5444 //
5445 // fneg (f64 (bitcast (build_vector x, y))) ->
5446 // f64 (bitcast (build_vector (bitcast i32:x to f32),
5447 // (fneg (bitcast i32:y to f32)))
5448
5449 SDValue CastHi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: HighBits);
5450 SDValue NegHi = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: CastHi);
5451 SDValue CastBack =
5452 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: HighBits.getValueType(), Operand: NegHi);
5453
5454 SmallVector<SDValue, 8> Ops(BCSrc->ops());
5455 Ops.back() = CastBack;
5456 DCI.AddToWorklist(N: NegHi.getNode());
5457 SDValue Build =
5458 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: BCSrc.getValueType(), Ops);
5459 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Build);
5460
5461 if (!N0.hasOneUse())
5462 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Result));
5463 return Result;
5464 }
5465
5466 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 &&
5467 BCSrc.hasOneUse()) {
5468 // fneg (bitcast (f32 (select cond, i32:lhs, i32:rhs))) ->
5469 // select cond, (bitcast i32:lhs to f32), (bitcast i32:rhs to f32)
5470
5471 // TODO: Cast back result for multiple uses is beneficial in some cases.
5472
5473 SDValue LHS =
5474 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 1));
5475 SDValue RHS =
5476 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 2));
5477
5478 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: LHS);
5479 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHS);
5480
5481 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: BCSrc.getOperand(i: 0), N2: NegLHS,
5482 N3: NegRHS);
5483 }
5484
5485 return SDValue();
5486 }
5487 default:
5488 return SDValue();
5489 }
5490}
5491
5492SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
5493 DAGCombinerInfo &DCI) const {
5494 SelectionDAG &DAG = DCI.DAG;
5495 SDValue N0 = N->getOperand(Num: 0);
5496
5497 if (!N0.hasOneUse())
5498 return SDValue();
5499
5500 switch (N0.getOpcode()) {
5501 case ISD::FP16_TO_FP: {
5502 assert(!isTypeLegal(MVT::f16) && "should only see if f16 is illegal");
5503 SDLoc SL(N);
5504 SDValue Src = N0.getOperand(i: 0);
5505 EVT SrcVT = Src.getValueType();
5506
5507 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
5508 SDValue IntFAbs = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SrcVT, N1: Src,
5509 N2: DAG.getConstant(Val: 0x7fff, DL: SL, VT: SrcVT));
5510 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFAbs);
5511 }
5512 case ISD::FP_ROUND: {
5513 SDLoc SL(N);
5514 SDValue CvtSrc = N0.getOperand(i: 0);
5515
5516 // fabs (fp_round x) -> fp_round (fabs x)
5517 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc,
5518 Flags: N->getFlags());
5519 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: N->getValueType(ResNo: 0), N1: Abs,
5520 N2: N0.getOperand(i: 1), Flags: N0->getFlags());
5521 }
5522 default:
5523 return SDValue();
5524 }
5525}
5526
5527SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
5528 DAGCombinerInfo &DCI) const {
5529 const auto *CFP = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
5530 if (!CFP)
5531 return SDValue();
5532
5533 std::optional<APFloat> Result = AMDGPU::evaluateRcp(Val: CFP->getValueAPF());
5534 if (!Result)
5535 return SDValue();
5536
5537 return DCI.DAG.getConstantFP(Val: *Result, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
5538}
5539
5540bool AMDGPUTargetLowering::isInt64ImmLegal(SDNode *N, SelectionDAG &DAG) const {
5541 if (!Subtarget->isGCN())
5542 return false;
5543
5544 ConstantSDNode *SDConstant = dyn_cast<ConstantSDNode>(Val: N);
5545 ConstantFPSDNode *SDFPConstant = dyn_cast<ConstantFPSDNode>(Val: N);
5546 auto &ST = DAG.getSubtarget<GCNSubtarget>();
5547 const auto *TII = ST.getInstrInfo();
5548
5549 if (!ST.hasVMovB64Inst() || (!SDConstant && !SDFPConstant))
5550 return false;
5551
5552 if (ST.has64BitLiterals())
5553 return true;
5554
5555 if (SDConstant) {
5556 const APInt &APVal = SDConstant->getAPIntValue();
5557 return isUInt<32>(x: APVal.getZExtValue()) || TII->isInlineConstant(Imm: APVal);
5558 }
5559
5560 APInt Val = SDFPConstant->getValueAPF().bitcastToAPInt();
5561 return isUInt<32>(x: Val.getZExtValue()) || TII->isInlineConstant(Imm: Val);
5562}
5563
5564SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
5565 DAGCombinerInfo &DCI) const {
5566 SelectionDAG &DAG = DCI.DAG;
5567 SDLoc DL(N);
5568
5569 switch(N->getOpcode()) {
5570 default:
5571 break;
5572 case ISD::BITCAST: {
5573 EVT DestVT = N->getValueType(ResNo: 0);
5574
5575 // Push casts through vector builds. This helps avoid emitting a large
5576 // number of copies when materializing floating point vector constants.
5577 //
5578 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
5579 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
5580 if (DestVT.isVector()) {
5581 SDValue Src = N->getOperand(Num: 0);
5582 if (Src.getOpcode() == ISD::BUILD_VECTOR &&
5583 (DCI.getDAGCombineLevel() < AfterLegalizeDAG ||
5584 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: DestVT))) {
5585 EVT SrcVT = Src.getValueType();
5586 unsigned NElts = DestVT.getVectorNumElements();
5587
5588 if (SrcVT.getVectorNumElements() == NElts) {
5589 EVT DestEltVT = DestVT.getVectorElementType();
5590
5591 SmallVector<SDValue, 8> CastedElts;
5592 SDLoc SL(N);
5593 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
5594 SDValue Elt = Src.getOperand(i: I);
5595 CastedElts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: DestEltVT, Operand: Elt));
5596 }
5597
5598 return DAG.getBuildVector(VT: DestVT, DL: SL, Ops: CastedElts);
5599 }
5600 }
5601 }
5602
5603 if (DestVT.getSizeInBits() != 64 || !DestVT.isVector())
5604 break;
5605
5606 // Fold bitcasts of constants.
5607 //
5608 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
5609 // TODO: Generalize and move to DAGCombiner
5610 SDValue Src = N->getOperand(Num: 0);
5611 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Src)) {
5612 SDLoc SL(N);
5613 if (isInt64ImmLegal(N: C, DAG))
5614 break;
5615 uint64_t CVal = C->getZExtValue();
5616 SDValue BV = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5617 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5618 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5619 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: BV);
5620 }
5621
5622 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
5623 const APInt &Val = C->getValueAPF().bitcastToAPInt();
5624 SDLoc SL(N);
5625 if (isInt64ImmLegal(N: C, DAG))
5626 break;
5627 uint64_t CVal = Val.getZExtValue();
5628 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5629 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5630 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5631
5632 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: Vec);
5633 }
5634
5635 break;
5636 }
5637 case ISD::SHL:
5638 case ISD::SRA:
5639 case ISD::SRL: {
5640 // Range metadata can be invalidated when loads are converted to legal types
5641 // (e.g. v2i64 -> v4i32).
5642 // Try to convert vector shl/sra/srl before type legalization so that range
5643 // metadata can be utilized.
5644 if (!(N->getValueType(ResNo: 0).isVector() &&
5645 DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
5646 DCI.getDAGCombineLevel() < AfterLegalizeDAG)
5647 break;
5648 if (N->getOpcode() == ISD::SHL)
5649 return performShlCombine(N, DCI);
5650 if (N->getOpcode() == ISD::SRA)
5651 return performSraCombine(N, DCI);
5652 return performSrlCombine(N, DCI);
5653 }
5654 case ISD::TRUNCATE:
5655 return performTruncateCombine(N, DCI);
5656 case ISD::MUL:
5657 return performMulCombine(N, DCI);
5658 case AMDGPUISD::MUL_U24:
5659 case AMDGPUISD::MUL_I24: {
5660 if (SDValue Simplified = simplifyMul24(Node24: N, DCI))
5661 return Simplified;
5662 break;
5663 }
5664 case AMDGPUISD::MULHI_I24:
5665 case AMDGPUISD::MULHI_U24:
5666 return simplifyMul24(Node24: N, DCI);
5667 case ISD::SMUL_LOHI:
5668 case ISD::UMUL_LOHI:
5669 return performMulLoHiCombine(N, DCI);
5670 case ISD::MULHS:
5671 return performMulhsCombine(N, DCI);
5672 case ISD::MULHU:
5673 return performMulhuCombine(N, DCI);
5674 case ISD::SELECT:
5675 return performSelectCombine(N, DCI);
5676 case ISD::FNEG:
5677 return performFNegCombine(N, DCI);
5678 case ISD::FABS:
5679 return performFAbsCombine(N, DCI);
5680 case AMDGPUISD::BFE_I32:
5681 case AMDGPUISD::BFE_U32: {
5682 assert(N->getValueType(0) == MVT::i32 &&
5683 "BFE_I32/BFE_U32 is a 32-bit operation");
5684 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
5685 if (!Width)
5686 break;
5687
5688 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
5689 if (WidthVal == 0)
5690 return DAG.getConstant(Val: 0, DL, VT: MVT::i32);
5691
5692 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
5693 if (!Offset)
5694 break;
5695
5696 SDValue BitsFrom = N->getOperand(Num: 0);
5697 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
5698
5699 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
5700
5701 if (OffsetVal == 0) {
5702 // This is already sign / zero extended, so try to fold away extra BFEs.
5703 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WidthVal);
5704 if (Signed) {
5705 if (DAG.ComputeNumSignBits(Op: BitsFrom) >= 32 - WidthVal + 1)
5706 return BitsFrom;
5707
5708 // This is a sign_extend_inreg. Replace it to take advantage of existing
5709 // DAG Combines. If not eliminated, we will match back to BFE during
5710 // selection.
5711
5712 // TODO: The sext_inreg of extended types ends, although we can could
5713 // handle them in a single BFE.
5714 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32, N1: BitsFrom,
5715 N2: DAG.getValueType(SmallVT));
5716 }
5717
5718 if (DAG.MaskedValueIsZero(Op: BitsFrom,
5719 Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - WidthVal)))
5720 return BitsFrom;
5721
5722 return DAG.getZeroExtendInReg(Op: BitsFrom, DL, VT: SmallVT);
5723 }
5724
5725 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(Val&: BitsFrom)) {
5726 if (Signed) {
5727 return constantFoldBFE<int32_t>(DAG,
5728 Src0: CVal->getSExtValue(),
5729 Offset: OffsetVal,
5730 Width: WidthVal,
5731 DL);
5732 }
5733
5734 return constantFoldBFE<uint32_t>(DAG,
5735 Src0: CVal->getZExtValue(),
5736 Offset: OffsetVal,
5737 Width: WidthVal,
5738 DL);
5739 }
5740
5741 if ((OffsetVal + WidthVal) >= 32 &&
5742 !(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
5743 SDValue ShiftVal = DAG.getConstant(Val: OffsetVal, DL, VT: MVT::i32);
5744 return DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32,
5745 N1: BitsFrom, N2: ShiftVal);
5746 }
5747
5748 if (BitsFrom.hasOneUse()) {
5749 APInt Demanded = APInt::getBitsSet(numBits: 32,
5750 loBit: OffsetVal,
5751 hiBit: OffsetVal + WidthVal);
5752
5753 KnownBits Known;
5754 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
5755 !DCI.isBeforeLegalizeOps());
5756 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5757 if (TLI.ShrinkDemandedConstant(Op: BitsFrom, DemandedBits: Demanded, TLO) ||
5758 TLI.SimplifyDemandedBits(Op: BitsFrom, DemandedBits: Demanded, Known, TLO)) {
5759 DCI.CommitTargetLoweringOpt(TLO);
5760 }
5761 }
5762
5763 break;
5764 }
5765 case ISD::LOAD:
5766 return performLoadCombine(N, DCI);
5767 case ISD::STORE:
5768 return performStoreCombine(N, DCI);
5769 case AMDGPUISD::RCP:
5770 case AMDGPUISD::RCP_IFLAG:
5771 return performRcpCombine(N, DCI);
5772 case ISD::AssertZext:
5773 case ISD::AssertSext:
5774 return performAssertSZExtCombine(N, DCI);
5775 case ISD::INTRINSIC_WO_CHAIN:
5776 return performIntrinsicWOChainCombine(N, DCI);
5777 case AMDGPUISD::FMAD_FTZ: {
5778 SDValue N0 = N->getOperand(Num: 0);
5779 SDValue N1 = N->getOperand(Num: 1);
5780 SDValue N2 = N->getOperand(Num: 2);
5781 EVT VT = N->getValueType(ResNo: 0);
5782
5783 // FMAD_FTZ is a FMAD + flush denormals to zero.
5784 // We flush the inputs, the intermediate step, and the output.
5785 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Val&: N0);
5786 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
5787 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
5788 if (N0CFP && N1CFP && N2CFP) {
5789 const auto FTZ = [](const APFloat &V) {
5790 if (V.isDenormal()) {
5791 APFloat Zero(V.getSemantics(), 0);
5792 return V.isNegative() ? -Zero : Zero;
5793 }
5794 return V;
5795 };
5796
5797 APFloat V0 = FTZ(N0CFP->getValueAPF());
5798 APFloat V1 = FTZ(N1CFP->getValueAPF());
5799 APFloat V2 = FTZ(N2CFP->getValueAPF());
5800 V0.multiply(RHS: V1, RM: APFloat::rmNearestTiesToEven);
5801 V0 = FTZ(V0);
5802 V0.add(RHS: V2, RM: APFloat::rmNearestTiesToEven);
5803 return DAG.getConstantFP(Val: FTZ(V0), DL, VT);
5804 }
5805 break;
5806 }
5807 }
5808 return SDValue();
5809}
5810
5811bool AMDGPUTargetLowering::SimplifyDemandedBitsForTargetNode(
5812 SDValue Op, const APInt &OriginalDemandedBits,
5813 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
5814 unsigned Depth) const {
5815 switch (Op.getOpcode()) {
5816 case ISD::INTRINSIC_WO_CHAIN: {
5817 switch (Op.getConstantOperandVal(i: 0)) {
5818 case Intrinsic::amdgcn_readfirstlane:
5819 case Intrinsic::amdgcn_readlane:
5820 case Intrinsic::amdgcn_wwm: {
5821 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5822 DemandedElts: OriginalDemandedElts, Known, TLO, Depth: Depth + 1))
5823 return true;
5824 break;
5825 }
5826 case Intrinsic::amdgcn_set_inactive:
5827 case Intrinsic::amdgcn_set_inactive_chain_arg: {
5828 // The result is operand 1 in active lanes and operand 2 in inactive
5829 // lanes, so the known bits are the intersection of both operands.
5830 KnownBits KnownValue, KnownInactive;
5831 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5832 DemandedElts: OriginalDemandedElts, Known&: KnownValue, TLO,
5833 Depth: Depth + 1))
5834 return true;
5835 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), DemandedBits: OriginalDemandedBits,
5836 DemandedElts: OriginalDemandedElts, Known&: KnownInactive, TLO,
5837 Depth: Depth + 1))
5838 return true;
5839 Known = KnownValue.intersectWith(RHS: KnownInactive);
5840 break;
5841 }
5842 default:
5843 break;
5844 }
5845 break;
5846 }
5847 default:
5848 break;
5849 }
5850
5851 return false;
5852}
5853
5854//===----------------------------------------------------------------------===//
5855// Helper functions
5856//===----------------------------------------------------------------------===//
5857
5858SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
5859 const TargetRegisterClass *RC,
5860 Register Reg, EVT VT,
5861 const SDLoc &SL,
5862 bool RawReg) const {
5863 MachineFunction &MF = DAG.getMachineFunction();
5864 MachineRegisterInfo &MRI = MF.getRegInfo();
5865 Register VReg;
5866
5867 if (!MRI.isLiveIn(Reg)) {
5868 VReg = MRI.createVirtualRegister(RegClass: RC);
5869 MRI.addLiveIn(Reg, vreg: VReg);
5870 } else {
5871 VReg = MRI.getLiveInVirtReg(PReg: Reg);
5872 }
5873
5874 if (RawReg)
5875 return DAG.getRegister(Reg: VReg, VT);
5876
5877 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: VReg, VT);
5878}
5879
5880// This may be called multiple times, and nothing prevents creating multiple
5881// objects at the same offset. See if we already defined this object.
5882static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
5883 int64_t Offset) {
5884 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
5885 if (MFI.getObjectOffset(ObjectIdx: I) == Offset) {
5886 assert(MFI.getObjectSize(I) == Size);
5887 return I;
5888 }
5889 }
5890
5891 return MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
5892}
5893
5894SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
5895 EVT VT,
5896 const SDLoc &SL,
5897 int64_t Offset) const {
5898 MachineFunction &MF = DAG.getMachineFunction();
5899 MachineFrameInfo &MFI = MF.getFrameInfo();
5900 int FI = getOrCreateFixedStackObject(MFI, Size: VT.getStoreSize(), Offset);
5901
5902 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
5903 SDValue Ptr = DAG.getFrameIndex(FI, VT: MVT::i32);
5904
5905 return DAG.getLoad(VT, dl: SL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: SrcPtrInfo, Alignment: Align(4),
5906 MMOFlags: MachineMemOperand::MODereferenceable |
5907 MachineMemOperand::MOInvariant);
5908}
5909
5910SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
5911 const SDLoc &SL,
5912 SDValue Chain,
5913 SDValue ArgVal,
5914 int64_t Offset) const {
5915 MachineFunction &MF = DAG.getMachineFunction();
5916 MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
5917 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5918
5919 SDValue Ptr = DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32);
5920 // Stores to the argument stack area are relative to the stack pointer.
5921 SDValue SP =
5922 DAG.getCopyFromReg(Chain, dl: SL, Reg: Info->getStackPtrOffsetReg(), VT: MVT::i32);
5923 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: SP, N2: Ptr);
5924 SDValue Store = DAG.getStore(Chain, dl: SL, Val: ArgVal, Ptr, PtrInfo: DstInfo, Alignment: Align(4),
5925 MMOFlags: MachineMemOperand::MODereferenceable);
5926 return Store;
5927}
5928
5929SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
5930 const TargetRegisterClass *RC,
5931 EVT VT, const SDLoc &SL,
5932 const ArgDescriptor &Arg) const {
5933 assert(Arg && "Attempting to load missing argument");
5934
5935 SDValue V = Arg.isRegister() ?
5936 CreateLiveInRegister(DAG, RC, Reg: Arg.getRegister(), VT, SL) :
5937 loadStackInputValue(DAG, VT, SL, Offset: Arg.getStackOffset());
5938
5939 if (!Arg.isMasked())
5940 return V;
5941
5942 unsigned Mask = Arg.getMask();
5943 unsigned Shift = llvm::countr_zero<unsigned>(Val: Mask);
5944 V = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: V,
5945 N2: DAG.getShiftAmountConstant(Val: Shift, VT, DL: SL));
5946 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT, N1: V,
5947 N2: DAG.getConstant(Val: Mask >> Shift, DL: SL, VT));
5948}
5949
5950uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5951 uint64_t ExplicitKernArgSize, const ImplicitParameter Param) const {
5952 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
5953 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr();
5954 uint64_t ArgOffset =
5955 alignTo(Size: ExplicitKernArgSize, A: Alignment) + ExplicitArgOffset;
5956 switch (Param) {
5957 case FIRST_IMPLICIT:
5958 return ArgOffset;
5959 case PRIVATE_BASE:
5960 return ArgOffset + AMDGPU::ImplicitArg::PRIVATE_BASE_OFFSET;
5961 case SHARED_BASE:
5962 return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET;
5963 case QUEUE_PTR:
5964 return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET;
5965 }
5966 llvm_unreachable("unexpected implicit parameter type");
5967}
5968
5969uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5970 const MachineFunction &MF, const ImplicitParameter Param) const {
5971 const AMDGPUMachineFunctionInfo *MFI =
5972 MF.getInfo<AMDGPUMachineFunctionInfo>();
5973 return getImplicitParameterOffset(ExplicitKernArgSize: MFI->getExplicitKernArgSize(), Param);
5974}
5975
5976SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
5977 SelectionDAG &DAG, int Enabled,
5978 int &RefinementSteps,
5979 bool &UseOneConstNR,
5980 bool Reciprocal) const {
5981 EVT VT = Operand.getValueType();
5982
5983 if (VT == MVT::f32) {
5984 RefinementSteps = 0;
5985 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(Operand), VT, Operand);
5986 }
5987
5988 // TODO: There is also f64 rsq instruction, but the documentation is less
5989 // clear on its precision.
5990
5991 return SDValue();
5992}
5993
5994SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
5995 SelectionDAG &DAG, int Enabled,
5996 int &RefinementSteps) const {
5997 EVT VT = Operand.getValueType();
5998
5999 if (VT == MVT::f32) {
6000 // Reciprocal, < 1 ulp error.
6001 //
6002 // This reciprocal approximation converges to < 0.5 ulp error with one
6003 // newton rhapson performed with two fused multiple adds (FMAs).
6004
6005 RefinementSteps = 0;
6006 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SDLoc(Operand), VT, Operand);
6007 }
6008
6009 // TODO: There is also f64 rcp instruction, but the documentation is less
6010 // clear on its precision.
6011
6012 return SDValue();
6013}
6014
6015static unsigned workitemIntrinsicDim(unsigned ID) {
6016 switch (ID) {
6017 case Intrinsic::amdgcn_workitem_id_x:
6018 return 0;
6019 case Intrinsic::amdgcn_workitem_id_y:
6020 return 1;
6021 case Intrinsic::amdgcn_workitem_id_z:
6022 return 2;
6023 default:
6024 llvm_unreachable("not a workitem intrinsic");
6025 }
6026}
6027
6028void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
6029 const SDValue Op, KnownBits &Known,
6030 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
6031
6032 Known.resetAll(); // Don't know anything.
6033
6034 unsigned Opc = Op.getOpcode();
6035
6036 switch (Opc) {
6037 default:
6038 break;
6039 case AMDGPUISD::CARRY:
6040 case AMDGPUISD::BORROW: {
6041 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31);
6042 break;
6043 }
6044
6045 case AMDGPUISD::BFE_I32:
6046 case AMDGPUISD::BFE_U32: {
6047 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6048 if (!CWidth)
6049 return;
6050
6051 uint32_t Width = CWidth->getZExtValue() & 0x1f;
6052
6053 if (Opc == AMDGPUISD::BFE_U32)
6054 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - Width);
6055
6056 break;
6057 }
6058 case AMDGPUISD::FP_TO_FP16: {
6059 unsigned BitWidth = Known.getBitWidth();
6060
6061 // High bits are zero.
6062 Known.Zero = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 16);
6063 break;
6064 }
6065 case AMDGPUISD::MUL_U24:
6066 case AMDGPUISD::MUL_I24: {
6067 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6068 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6069 unsigned BitWidth = Op.getScalarValueSizeInBits();
6070
6071 // Sign/Zero extend from 24 bits.
6072 if (Opc == AMDGPUISD::MUL_I24) {
6073 LHSKnown = LHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6074 RHSKnown = RHSKnown.trunc(BitWidth: 24).sext(BitWidth);
6075 } else {
6076 LHSKnown = LHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6077 RHSKnown = RHSKnown.trunc(BitWidth: 24).zext(BitWidth);
6078 }
6079
6080 // TODO: SelfMultiply can be poison, but not undef.
6081 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
6082 if (SelfMultiply)
6083 SelfMultiply &= DAG.isGuaranteedNotToBeUndefOrPoison(
6084 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
6085 Depth: Depth + 1);
6086
6087 Known = KnownBits::mul(LHS: LHSKnown, RHS: RHSKnown, NoUndefSelfMultiply: SelfMultiply);
6088 break;
6089 }
6090 case AMDGPUISD::PERM: {
6091 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6092 if (!CMask)
6093 return;
6094
6095 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6096 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6097 unsigned Sel = CMask->getZExtValue();
6098
6099 for (unsigned I = 0; I < 32; I += 8) {
6100 unsigned SelBits = Sel & 0xff;
6101 if (SelBits < 4) {
6102 SelBits *= 8;
6103 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6104 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6105 } else if (SelBits < 7) {
6106 SelBits = (SelBits & 3) * 8;
6107 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6108 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6109 } else if (SelBits == 0x0c) {
6110 Known.Zero |= 0xFFull << I;
6111 } else if (SelBits > 0x0c) {
6112 Known.One |= 0xFFull << I;
6113 }
6114 Sel >>= 8;
6115 }
6116 break;
6117 }
6118 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
6119 Known.Zero.setHighBits(24);
6120 break;
6121 }
6122 case AMDGPUISD::BUFFER_LOAD_USHORT: {
6123 Known.Zero.setHighBits(16);
6124 break;
6125 }
6126 case AMDGPUISD::LDS: {
6127 auto *GA = cast<GlobalAddressSDNode>(Val: Op.getOperand(i: 0).getNode());
6128 Align Alignment = GA->getGlobal()->getPointerAlignment(DL: DAG.getDataLayout());
6129
6130 Known.Zero.setHighBits(16);
6131 Known.Zero.setLowBits(Log2(A: Alignment));
6132 break;
6133 }
6134 case AMDGPUISD::SMIN3:
6135 case AMDGPUISD::SMAX3:
6136 case AMDGPUISD::SMED3:
6137 case AMDGPUISD::UMIN3:
6138 case AMDGPUISD::UMAX3:
6139 case AMDGPUISD::UMED3: {
6140 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6141 if (Known2.isUnknown())
6142 break;
6143
6144 KnownBits Known1 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6145 if (Known1.isUnknown())
6146 break;
6147
6148 KnownBits Known0 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6149 if (Known0.isUnknown())
6150 break;
6151
6152 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
6153 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
6154 Known.One = Known0.One & Known1.One & Known2.One;
6155 break;
6156 }
6157 case ISD::INTRINSIC_WO_CHAIN: {
6158 unsigned IID = Op.getConstantOperandVal(i: 0);
6159 switch (IID) {
6160 case Intrinsic::amdgcn_workitem_id_x:
6161 case Intrinsic::amdgcn_workitem_id_y:
6162 case Intrinsic::amdgcn_workitem_id_z: {
6163 unsigned MaxValue = Subtarget->getMaxWorkitemID(
6164 Kernel: DAG.getMachineFunction().getFunction(), Dimension: workitemIntrinsicDim(ID: IID));
6165 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
6166 break;
6167 }
6168 case Intrinsic::amdgcn_readfirstlane:
6169 case Intrinsic::amdgcn_readlane:
6170 // Result is the data operand's value from some lane.
6171 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6172 break;
6173 default:
6174 break;
6175 }
6176 }
6177 }
6178}
6179
6180unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
6181 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6182 unsigned Depth) const {
6183 switch (Op.getOpcode()) {
6184 case AMDGPUISD::BFE_I32: {
6185 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6186 if (!Width)
6187 return 1;
6188
6189 unsigned SignBits = 32 - (Width->getZExtValue() & 0x1f) + 1;
6190 if (!isNullConstant(V: Op.getOperand(i: 1)))
6191 return SignBits;
6192
6193 // TODO: Could probably figure something out with non-0 offsets.
6194 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6195 return std::max(a: SignBits, b: Op0SignBits);
6196 }
6197
6198 case AMDGPUISD::BFE_U32: {
6199 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6200 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
6201 }
6202
6203 case AMDGPUISD::CARRY:
6204 case AMDGPUISD::BORROW:
6205 return 31;
6206 case AMDGPUISD::BUFFER_LOAD_BYTE:
6207 return 25;
6208 case AMDGPUISD::BUFFER_LOAD_SHORT:
6209 return 17;
6210 case AMDGPUISD::BUFFER_LOAD_UBYTE:
6211 return 24;
6212 case AMDGPUISD::BUFFER_LOAD_USHORT:
6213 return 16;
6214 case AMDGPUISD::FP_TO_FP16:
6215 return 16;
6216 case AMDGPUISD::SMIN3:
6217 case AMDGPUISD::SMAX3:
6218 case AMDGPUISD::SMED3:
6219 case AMDGPUISD::UMIN3:
6220 case AMDGPUISD::UMAX3:
6221 case AMDGPUISD::UMED3: {
6222 unsigned Tmp2 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6223 if (Tmp2 == 1)
6224 return 1; // Early out.
6225
6226 unsigned Tmp1 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6227 if (Tmp1 == 1)
6228 return 1; // Early out.
6229
6230 unsigned Tmp0 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6231 if (Tmp0 == 1)
6232 return 1; // Early out.
6233
6234 return std::min(l: {Tmp0, Tmp1, Tmp2});
6235 }
6236 default:
6237 return 1;
6238 }
6239}
6240
6241unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
6242 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
6243 const MachineRegisterInfo &MRI, unsigned Depth) const {
6244 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
6245 if (!MI)
6246 return 1;
6247
6248 // TODO: Check range metadata on MMO.
6249 switch (MI->getOpcode()) {
6250 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
6251 return 25;
6252 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
6253 return 17;
6254 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
6255 return 24;
6256 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
6257 return 16;
6258 case AMDGPU::G_AMDGPU_SMED3:
6259 case AMDGPU::G_AMDGPU_UMED3: {
6260 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
6261 unsigned Tmp2 = Analysis.computeNumSignBits(R: Src2, DemandedElts, Depth: Depth + 1);
6262 if (Tmp2 == 1)
6263 return 1;
6264 unsigned Tmp1 = Analysis.computeNumSignBits(R: Src1, DemandedElts, Depth: Depth + 1);
6265 if (Tmp1 == 1)
6266 return 1;
6267 unsigned Tmp0 = Analysis.computeNumSignBits(R: Src0, DemandedElts, Depth: Depth + 1);
6268 if (Tmp0 == 1)
6269 return 1;
6270 return std::min(l: {Tmp0, Tmp1, Tmp2});
6271 }
6272 default:
6273 return 1;
6274 }
6275}
6276
6277bool AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode(
6278 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6279 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
6280 unsigned Opcode = Op.getOpcode();
6281 switch (Opcode) {
6282 case AMDGPUISD::BFE_I32:
6283 case AMDGPUISD::BFE_U32:
6284 return false;
6285 }
6286 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
6287 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
6288}
6289
6290bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
6291 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
6292 unsigned Depth) const {
6293 unsigned Opcode = Op.getOpcode();
6294 switch (Opcode) {
6295 case AMDGPUISD::FMIN_LEGACY:
6296 case AMDGPUISD::FMAX_LEGACY:
6297 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6298 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6299 case AMDGPUISD::FMUL_LEGACY:
6300 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
6301 if (SNaN)
6302 return true;
6303 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6304 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6305 }
6306 case AMDGPUISD::FMED3:
6307 case AMDGPUISD::FMIN3:
6308 case AMDGPUISD::FMAX3:
6309 case AMDGPUISD::FMINIMUM3:
6310 case AMDGPUISD::FMAXIMUM3:
6311 case AMDGPUISD::FMAD_FTZ: {
6312 if (SNaN)
6313 return true;
6314 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6315 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6316 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6317 }
6318 case AMDGPUISD::CVT_F32_UBYTE0:
6319 case AMDGPUISD::CVT_F32_UBYTE1:
6320 case AMDGPUISD::CVT_F32_UBYTE2:
6321 case AMDGPUISD::CVT_F32_UBYTE3:
6322 return true;
6323
6324 case AMDGPUISD::RCP:
6325 case AMDGPUISD::RSQ:
6326 case AMDGPUISD::RCP_LEGACY:
6327 case AMDGPUISD::RSQ_CLAMP: {
6328 if (SNaN)
6329 return true;
6330
6331 // TODO: Need is known positive check.
6332 return false;
6333 }
6334 case ISD::FLDEXP:
6335 case AMDGPUISD::FRACT: {
6336 if (SNaN)
6337 return true;
6338 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6339 }
6340 case AMDGPUISD::DIV_SCALE:
6341 case AMDGPUISD::DIV_FMAS:
6342 case AMDGPUISD::DIV_FIXUP:
6343 // TODO: Refine on operands.
6344 return SNaN;
6345 case AMDGPUISD::SIN_HW:
6346 case AMDGPUISD::COS_HW: {
6347 // TODO: Need check for infinity
6348 return SNaN;
6349 }
6350 case ISD::INTRINSIC_WO_CHAIN: {
6351 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
6352 // TODO: Handle more intrinsics
6353 switch (IntrinsicID) {
6354 case Intrinsic::amdgcn_cubeid:
6355 case Intrinsic::amdgcn_cvt_off_f32_i4:
6356 return true;
6357
6358 case Intrinsic::amdgcn_frexp_mant: {
6359 if (SNaN)
6360 return true;
6361 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6362 }
6363 case Intrinsic::amdgcn_cvt_pkrtz: {
6364 if (SNaN)
6365 return true;
6366 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6367 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6368 }
6369 case Intrinsic::amdgcn_rcp:
6370 case Intrinsic::amdgcn_rsq:
6371 case Intrinsic::amdgcn_rcp_legacy:
6372 case Intrinsic::amdgcn_rsq_legacy:
6373 case Intrinsic::amdgcn_rsq_clamp:
6374 case Intrinsic::amdgcn_tanh: {
6375 if (SNaN)
6376 return true;
6377
6378 // TODO: Need is known positive check.
6379 return false;
6380 }
6381 case Intrinsic::amdgcn_trig_preop:
6382 case Intrinsic::amdgcn_fdot2:
6383 // TODO: Refine on operand
6384 return SNaN;
6385 case Intrinsic::amdgcn_fma_legacy:
6386 if (SNaN)
6387 return true;
6388 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6389 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1) &&
6390 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 3), SNaN, Depth: Depth + 1);
6391 default:
6392 return false;
6393 }
6394 }
6395 default:
6396 return false;
6397 }
6398}
6399
6400bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
6401 Register N0, Register N1) const {
6402 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
6403}
6404