1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This is the parent TargetLowering class for hardware code gen
11/// targets.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUISelLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUMachineFunctionInfo.h"
19#include "AMDGPUMemoryUtils.h"
20#include "AMDGPUSelectionDAGInfo.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/Analysis.h"
23#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/Support/AMDGPUAddrSpace.h"
28#include "llvm/Support/KnownBits.h"
29#include "llvm/Target/TargetMachine.h"
30
31using namespace llvm;
32
33#define GET_CALLING_CONV_IMPL
34#include "AMDGPUGenCallingConv.inc"
35
36// Find a larger type to do a load / store of a vector with.
37EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
38 unsigned StoreSize = VT.getStoreSizeInBits();
39 if (StoreSize <= 32)
40 return EVT::getIntegerVT(Context&: Ctx, BitWidth: StoreSize);
41
42 if (StoreSize % 32 == 0)
43 return EVT::getVectorVT(Context&: Ctx, VT: MVT::i32, NumElements: StoreSize / 32);
44
45 return VT;
46}
47
48unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
49 return DAG.computeKnownBits(Op).countMaxActiveBits();
50}
51
52unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
53 // In order for this to be a signed 24-bit value, bit 23, must
54 // be a sign bit.
55 return DAG.ComputeMaxSignificantBits(Op);
56}
57
58KnownBits AMDGPUTargetLowering::computeKnownBitsForMul24(
59 const KnownBits &LHSKnown, const KnownBits &RHSKnown, unsigned BitWidth,
60 bool IsSigned, bool SelfMultiply) {
61 // Sign/Zero extend from 24 bits.
62 KnownBits LHS24 = LHSKnown.trunc(BitWidth: 24);
63 KnownBits RHS24 = RHSKnown.trunc(BitWidth: 24);
64 if (IsSigned)
65 return KnownBits::mul(LHS: LHS24.sext(BitWidth), RHS: RHS24.sext(BitWidth),
66 NoUndefSelfMultiply: SelfMultiply);
67 return KnownBits::mul(LHS: LHS24.zext(BitWidth), RHS: RHS24.zext(BitWidth),
68 NoUndefSelfMultiply: SelfMultiply);
69}
70
71AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
72 const TargetSubtargetInfo &STI,
73 const AMDGPUSubtarget &AMDGPUSTI)
74 : TargetLowering(TM, STI), Subtarget(&AMDGPUSTI) {
75 // Always lower memset, memcpy, and memmove intrinsics to load/store
76 // instructions, rather then generating calls to memset, mempcy or memmove.
77 MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U;
78 MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U;
79 MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U;
80
81 // Enable ganging up loads and stores in the memcpy DAG lowering.
82 MaxGluedStoresPerMemcpy = 16;
83
84 // Lower floating point store/load to integer store/load to reduce the number
85 // of patterns in tablegen.
86 setOperationAction(Op: ISD::LOAD, VT: MVT::f32, Action: Promote);
87 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f32, DestVT: MVT::i32);
88
89 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f32, Action: Promote);
90 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
91
92 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f32, Action: Promote);
93 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
94
95 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f32, Action: Promote);
96 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
97
98 setOperationAction(Op: ISD::LOAD, VT: MVT::v5f32, Action: Promote);
99 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
100
101 setOperationAction(Op: ISD::LOAD, VT: MVT::v6f32, Action: Promote);
102 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
103
104 setOperationAction(Op: ISD::LOAD, VT: MVT::v7f32, Action: Promote);
105 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
106
107 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f32, Action: Promote);
108 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
109
110 setOperationAction(Op: ISD::LOAD, VT: MVT::v9f32, Action: Promote);
111 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
112
113 setOperationAction(Op: ISD::LOAD, VT: MVT::v10f32, Action: Promote);
114 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
115
116 setOperationAction(Op: ISD::LOAD, VT: MVT::v11f32, Action: Promote);
117 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
118
119 setOperationAction(Op: ISD::LOAD, VT: MVT::v12f32, Action: Promote);
120 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
121
122 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f32, Action: Promote);
123 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
124
125 setOperationAction(Op: ISD::LOAD, VT: MVT::v32f32, Action: Promote);
126 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
127
128 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Promote);
129 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i64, DestVT: MVT::v2i32);
130
131 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
132 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
133
134 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Promote);
135 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f64, DestVT: MVT::v2i32);
136
137 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Promote);
138 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
139
140 setOperationAction(Op: ISD::LOAD, VT: MVT::v3i64, Action: Promote);
141 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
142
143 setOperationAction(Op: ISD::LOAD, VT: MVT::v4i64, Action: Promote);
144 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
145
146 setOperationAction(Op: ISD::LOAD, VT: MVT::v3f64, Action: Promote);
147 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
148
149 setOperationAction(Op: ISD::LOAD, VT: MVT::v4f64, Action: Promote);
150 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
151
152 setOperationAction(Op: ISD::LOAD, VT: MVT::v8i64, Action: Promote);
153 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
154
155 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f64, Action: Promote);
156 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
157
158 setOperationAction(Op: ISD::LOAD, VT: MVT::v16i64, Action: Promote);
159 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
160
161 setOperationAction(Op: ISD::LOAD, VT: MVT::v16f64, Action: Promote);
162 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
163
164 setOperationAction(Op: ISD::LOAD, VT: MVT::i128, Action: Promote);
165 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::i128, DestVT: MVT::v4i32);
166
167 // There are no 64-bit extloads. These should be done as a 32-bit extload and
168 // an extension to 64-bit.
169 for (MVT VT : MVT::integer_valuetypes())
170 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i64, MemVT: VT,
171 Action: Expand);
172
173 for (MVT VT : MVT::integer_valuetypes()) {
174 if (VT == MVT::i64)
175 continue;
176
177 for (auto Op : {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}) {
178 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i1, Action: Promote);
179 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i8, Action: Legal);
180 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i16, Action: Legal);
181 setLoadExtAction(ExtType: Op, ValVT: VT, MemVT: MVT::i32, Action: Expand);
182 }
183 }
184
185 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
186 for (auto MemVT :
187 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
188 setLoadExtAction(ExtTypes: {ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, ValVT: VT, MemVT,
189 Action: Expand);
190
191 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
192 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
193 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
194 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
195 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
196 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
197 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
198 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
199 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v5f32, MemVT: MVT::v5f16, Action: Expand);
200 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v6f32, MemVT: MVT::v6f16, Action: Expand);
201 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v7f32, MemVT: MVT::v7f16, Action: Expand);
202 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
203 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
204 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
205 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
206 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
207 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
208
209 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
210 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
211 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
212 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
213 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
214 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
215
216 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
217 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
218 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
219 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
220 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
221 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
222 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
223 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
224 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
225 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
226 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
227 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
228
229 setOperationAction(Op: ISD::STORE, VT: MVT::f32, Action: Promote);
230 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f32, DestVT: MVT::i32);
231
232 setOperationAction(Op: ISD::STORE, VT: MVT::v2f32, Action: Promote);
233 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
234
235 setOperationAction(Op: ISD::STORE, VT: MVT::v3f32, Action: Promote);
236 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
237
238 setOperationAction(Op: ISD::STORE, VT: MVT::v4f32, Action: Promote);
239 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
240
241 setOperationAction(Op: ISD::STORE, VT: MVT::v5f32, Action: Promote);
242 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
243
244 setOperationAction(Op: ISD::STORE, VT: MVT::v6f32, Action: Promote);
245 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
246
247 setOperationAction(Op: ISD::STORE, VT: MVT::v7f32, Action: Promote);
248 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
249
250 setOperationAction(Op: ISD::STORE, VT: MVT::v8f32, Action: Promote);
251 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f32, DestVT: MVT::v8i32);
252
253 setOperationAction(Op: ISD::STORE, VT: MVT::v9f32, Action: Promote);
254 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
255
256 setOperationAction(Op: ISD::STORE, VT: MVT::v10f32, Action: Promote);
257 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
258
259 setOperationAction(Op: ISD::STORE, VT: MVT::v11f32, Action: Promote);
260 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
261
262 setOperationAction(Op: ISD::STORE, VT: MVT::v12f32, Action: Promote);
263 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
264
265 setOperationAction(Op: ISD::STORE, VT: MVT::v16f32, Action: Promote);
266 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f32, DestVT: MVT::v16i32);
267
268 setOperationAction(Op: ISD::STORE, VT: MVT::v32f32, Action: Promote);
269 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v32f32, DestVT: MVT::v32i32);
270
271 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Promote);
272 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i64, DestVT: MVT::v2i32);
273
274 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
275 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v4i32);
276
277 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Promote);
278 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f64, DestVT: MVT::v2i32);
279
280 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Promote);
281 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v2f64, DestVT: MVT::v4i32);
282
283 setOperationAction(Op: ISD::STORE, VT: MVT::v3i64, Action: Promote);
284 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3i64, DestVT: MVT::v6i32);
285
286 setOperationAction(Op: ISD::STORE, VT: MVT::v3f64, Action: Promote);
287 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v3f64, DestVT: MVT::v6i32);
288
289 setOperationAction(Op: ISD::STORE, VT: MVT::v4i64, Action: Promote);
290 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4i64, DestVT: MVT::v8i32);
291
292 setOperationAction(Op: ISD::STORE, VT: MVT::v4f64, Action: Promote);
293 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v4f64, DestVT: MVT::v8i32);
294
295 setOperationAction(Op: ISD::STORE, VT: MVT::v8i64, Action: Promote);
296 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8i64, DestVT: MVT::v16i32);
297
298 setOperationAction(Op: ISD::STORE, VT: MVT::v8f64, Action: Promote);
299 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v8f64, DestVT: MVT::v16i32);
300
301 setOperationAction(Op: ISD::STORE, VT: MVT::v16i64, Action: Promote);
302 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16i64, DestVT: MVT::v32i32);
303
304 setOperationAction(Op: ISD::STORE, VT: MVT::v16f64, Action: Promote);
305 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::v16f64, DestVT: MVT::v32i32);
306
307 setOperationAction(Op: ISD::STORE, VT: MVT::i128, Action: Promote);
308 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::i128, DestVT: MVT::v4i32);
309
310 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i1, Action: Expand);
311 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i8, Action: Expand);
312 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i16, Action: Expand);
313 setTruncStoreAction(ValVT: MVT::i64, MemVT: MVT::i32, Action: Expand);
314
315 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i1, Action: Expand);
316 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i8, Action: Expand);
317 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i16, Action: Expand);
318 setTruncStoreAction(ValVT: MVT::v2i64, MemVT: MVT::v2i32, Action: Expand);
319
320 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
321 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
322 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2bf16, Action: Expand);
323 setTruncStoreAction(ValVT: MVT::v2f32, MemVT: MVT::v2f16, Action: Expand);
324 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3bf16, Action: Expand);
325 setTruncStoreAction(ValVT: MVT::v3f32, MemVT: MVT::v3f16, Action: Expand);
326 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4bf16, Action: Expand);
327 setTruncStoreAction(ValVT: MVT::v4f32, MemVT: MVT::v4f16, Action: Expand);
328 setTruncStoreAction(ValVT: MVT::v5f32, MemVT: MVT::v5f16, Action: Expand);
329 setTruncStoreAction(ValVT: MVT::v6f32, MemVT: MVT::v6f16, Action: Expand);
330 setTruncStoreAction(ValVT: MVT::v7f32, MemVT: MVT::v7f16, Action: Expand);
331 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8bf16, Action: Expand);
332 setTruncStoreAction(ValVT: MVT::v8f32, MemVT: MVT::v8f16, Action: Expand);
333 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16bf16, Action: Expand);
334 setTruncStoreAction(ValVT: MVT::v16f32, MemVT: MVT::v16f16, Action: Expand);
335 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32bf16, Action: Expand);
336 setTruncStoreAction(ValVT: MVT::v32f32, MemVT: MVT::v32f16, Action: Expand);
337
338 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
339 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
340 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
341
342 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Expand);
343 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2bf16, Action: Expand);
344 setTruncStoreAction(ValVT: MVT::v2f64, MemVT: MVT::v2f16, Action: Expand);
345
346 setTruncStoreAction(ValVT: MVT::v3i32, MemVT: MVT::v3i8, Action: Expand);
347
348 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i32, Action: Expand);
349 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i16, Action: Expand);
350 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i8, Action: Expand);
351 setTruncStoreAction(ValVT: MVT::v3i64, MemVT: MVT::v3i1, Action: Expand);
352 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f32, Action: Expand);
353 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3bf16, Action: Expand);
354 setTruncStoreAction(ValVT: MVT::v3f64, MemVT: MVT::v3f16, Action: Expand);
355
356 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i32, Action: Expand);
357 setTruncStoreAction(ValVT: MVT::v4i64, MemVT: MVT::v4i16, Action: Expand);
358 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f32, Action: Expand);
359 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4bf16, Action: Expand);
360 setTruncStoreAction(ValVT: MVT::v4f64, MemVT: MVT::v4f16, Action: Expand);
361
362 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i1, Action: Expand);
363 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i8, Action: Expand);
364 setTruncStoreAction(ValVT: MVT::v5i32, MemVT: MVT::v5i16, Action: Expand);
365
366 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i1, Action: Expand);
367 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i8, Action: Expand);
368 setTruncStoreAction(ValVT: MVT::v6i32, MemVT: MVT::v6i16, Action: Expand);
369
370 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i1, Action: Expand);
371 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i8, Action: Expand);
372 setTruncStoreAction(ValVT: MVT::v7i32, MemVT: MVT::v7i16, Action: Expand);
373
374 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f32, Action: Expand);
375 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8bf16, Action: Expand);
376 setTruncStoreAction(ValVT: MVT::v8f64, MemVT: MVT::v8f16, Action: Expand);
377
378 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f32, Action: Expand);
379 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16bf16, Action: Expand);
380 setTruncStoreAction(ValVT: MVT::v16f64, MemVT: MVT::v16f16, Action: Expand);
381 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i16, Action: Expand);
382 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
383 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i8, Action: Expand);
384 setTruncStoreAction(ValVT: MVT::v16i64, MemVT: MVT::v16i1, Action: Expand);
385
386 setOperationAction(Ops: ISD::Constant, VTs: {MVT::i32, MVT::i64}, Action: Legal);
387 setOperationAction(Ops: ISD::ConstantFP, VTs: {MVT::f32, MVT::f64}, Action: Legal);
388
389 setOperationAction(Ops: {ISD::BR_JT, ISD::BRIND}, VT: MVT::Other, Action: Expand);
390
391 // For R600, this is totally unsupported, just custom lower to produce an
392 // error.
393 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32, Action: Custom);
394
395 // Library functions. These default to Expand, but we have instructions
396 // for them.
397 setOperationAction(
398 Ops: {ISD::FCEIL, ISD::FABS, ISD::FFLOOR, ISD::FROUNDEVEN, ISD::FTRUNC},
399 VTs: {MVT::f16, MVT::f32}, Action: Legal);
400 setOperationAction(Ops: {ISD::FMINNUM, ISD::FMAXNUM}, VT: MVT::f32, Action: Legal);
401
402 setOperationAction(Op: ISD::FLOG2, VT: MVT::f32, Action: Custom);
403 setOperationAction(Ops: ISD::FROUND, VTs: {MVT::f32, MVT::f64}, Action: Custom);
404 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND},
405 VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
406
407 setOperationAction(
408 Ops: {ISD::FLOG, ISD::FLOG10, ISD::FEXP, ISD::FEXP2, ISD::FEXP10, ISD::FPOW},
409 VT: MVT::f32, Action: Custom);
410 setOperationAction(Ops: {ISD::FEXP, ISD::FEXP2, ISD::FEXP10}, VT: MVT::f64, Action: Custom);
411
412 setOperationAction(Ops: ISD::FNEARBYINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
413
414 setOperationAction(Ops: ISD::FRINT, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Custom);
415
416 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VTs: {MVT::f16, MVT::f32, MVT::f64},
417 Action: Expand);
418
419 setOperationAction(Ops: ISD::FREM, VTs: {MVT::f16, MVT::f32, MVT::f64}, Action: Expand);
420 setOperationAction(Ops: ISD::IS_FPCLASS, VTs: {MVT::f32, MVT::f64}, Action: Legal);
421 setOperationAction(Ops: {ISD::FLOG2, ISD::FEXP2}, VT: MVT::f16, Action: Custom);
422
423 setOperationAction(Ops: {ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, VT: MVT::f16,
424 Action: Custom);
425
426 setOperationAction(Ops: ISD::FCANONICALIZE, VTs: {MVT::f32, MVT::f64}, Action: Legal);
427
428 // FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
429 // scalarization code. Can be removed when IS_FPCLASS expand isn't called by
430 // default unless marked custom/legal.
431 setOperationAction(Ops: ISD::IS_FPCLASS,
432 VTs: {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
433 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
434 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64,
435 MVT::v16f64},
436 Action: Custom);
437
438 // Expand to fneg + fadd.
439 setOperationAction(Op: ISD::FSUB, VT: MVT::f64, Action: Expand);
440
441 setOperationAction(Ops: ISD::CONCAT_VECTORS,
442 VTs: {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
443 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
444 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
445 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
446 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
447 Action: Custom);
448
449 setOperationAction(
450 Ops: ISD::EXTRACT_SUBVECTOR,
451 VTs: {MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32,
452 MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32,
453 MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32,
454 MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32,
455 MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
456 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
457 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64},
458 Action: Custom);
459
460 setOperationAction(Ops: {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP}, VT: MVT::f64,
461 Action: Expand);
462 setOperationAction(Ops: ISD::FP_TO_FP16, VTs: {MVT::f64, MVT::f32}, Action: Custom);
463
464 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
465 for (MVT VT : ScalarIntVTs) {
466 // These should use [SU]DIVREM, so set them to expand
467 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT,
468 Action: Expand);
469
470 // GPU does not have divrem function for signed or unsigned.
471 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT, Action: Custom);
472
473 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
474 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
475
476 setOperationAction(Ops: {ISD::BSWAP, ISD::CTTZ, ISD::CTLZ}, VT, Action: Expand);
477 }
478
479 // The hardware supports 32-bit FSHR, but not FSHL.
480 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Legal);
481
482 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VTs: {MVT::i32, MVT::i64}, Action: Expand);
483
484 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT: MVT::i16, Action: Expand);
485
486 setOperationAction(Ops: {ISD::MUL, ISD::MULHU, ISD::MULHS}, VT: MVT::i64, Action: Expand);
487 setOperationAction(Ops: {ISD::UINT_TO_FP, ISD::SINT_TO_FP, ISD::FP_TO_SINT,
488 ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
489 ISD::FP_TO_UINT_SAT},
490 VT: MVT::i64, Action: Custom);
491 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: Expand);
492
493 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VT: MVT::i32,
494 Action: Legal);
495
496 setOperationAction(
497 Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON},
498 VT: MVT::i64, Action: Custom);
499
500 for (auto VT : {MVT::i8, MVT::i16})
501 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT, Action: Custom);
502
503 static const MVT::SimpleValueType VectorIntTypes[] = {
504 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
505 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
506
507 for (MVT VT : VectorIntTypes) {
508 // Expand the following operations for the current type by default.
509 // clang-format off
510 setOperationAction(Ops: {ISD::ADD, ISD::AND,
511 ISD::FP_TO_SINT, ISD::FP_TO_UINT,
512 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
513 ISD::MUL, ISD::MULHU,
514 ISD::MULHS, ISD::OR,
515 ISD::SHL, ISD::SRA,
516 ISD::SRL, ISD::ROTL,
517 ISD::ROTR, ISD::SUB,
518 ISD::SINT_TO_FP, ISD::UINT_TO_FP,
519 ISD::SDIV, ISD::UDIV,
520 ISD::SREM, ISD::UREM,
521 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
522 ISD::SDIVREM, ISD::UDIVREM,
523 ISD::SELECT, ISD::VSELECT,
524 ISD::SELECT_CC, ISD::XOR,
525 ISD::BSWAP, ISD::CTPOP,
526 ISD::CTTZ, ISD::CTLZ,
527 ISD::VECTOR_SHUFFLE, ISD::SETCC,
528 ISD::ADDRSPACECAST},
529 VT, Action: Expand);
530 // clang-format on
531 }
532
533 static const MVT::SimpleValueType FloatVectorTypes[] = {
534 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
535 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
536
537 for (MVT VT : FloatVectorTypes) {
538 setOperationAction(
539 Ops: {ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM,
540 ISD::FADD, ISD::FCEIL, ISD::FCOS,
541 ISD::FDIV, ISD::FEXP2, ISD::FEXP,
542 ISD::FEXP10, ISD::FLOG2, ISD::FREM,
543 ISD::FLOG, ISD::FLOG10, ISD::FPOW,
544 ISD::FFLOOR, ISD::FTRUNC, ISD::FMUL,
545 ISD::FMA, ISD::FRINT, ISD::FNEARBYINT,
546 ISD::FSQRT, ISD::FSIN, ISD::FSUB,
547 ISD::FNEG, ISD::VSELECT, ISD::SELECT_CC,
548 ISD::FCOPYSIGN, ISD::VECTOR_SHUFFLE, ISD::SETCC,
549 ISD::FCANONICALIZE, ISD::FROUNDEVEN},
550 VT, Action: Expand);
551 }
552
553 // This causes using an unrolled select operation rather than expansion with
554 // bit operations. This is in general better, but the alternative using BFI
555 // instructions may be better if the select sources are SGPRs.
556 setOperationAction(Op: ISD::SELECT, VT: MVT::v2f32, Action: Promote);
557 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v2f32, DestVT: MVT::v2i32);
558
559 setOperationAction(Op: ISD::SELECT, VT: MVT::v3f32, Action: Promote);
560 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v3f32, DestVT: MVT::v3i32);
561
562 setOperationAction(Op: ISD::SELECT, VT: MVT::v4f32, Action: Promote);
563 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v4f32, DestVT: MVT::v4i32);
564
565 setOperationAction(Op: ISD::SELECT, VT: MVT::v5f32, Action: Promote);
566 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v5f32, DestVT: MVT::v5i32);
567
568 setOperationAction(Op: ISD::SELECT, VT: MVT::v6f32, Action: Promote);
569 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v6f32, DestVT: MVT::v6i32);
570
571 setOperationAction(Op: ISD::SELECT, VT: MVT::v7f32, Action: Promote);
572 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v7f32, DestVT: MVT::v7i32);
573
574 setOperationAction(Op: ISD::SELECT, VT: MVT::v9f32, Action: Promote);
575 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v9f32, DestVT: MVT::v9i32);
576
577 setOperationAction(Op: ISD::SELECT, VT: MVT::v10f32, Action: Promote);
578 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v10f32, DestVT: MVT::v10i32);
579
580 setOperationAction(Op: ISD::SELECT, VT: MVT::v11f32, Action: Promote);
581 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v11f32, DestVT: MVT::v11i32);
582
583 setOperationAction(Op: ISD::SELECT, VT: MVT::v12f32, Action: Promote);
584 AddPromotedToType(Opc: ISD::SELECT, OrigVT: MVT::v12f32, DestVT: MVT::v12i32);
585
586 setSchedulingPreference(Sched::RegPressure);
587 setJumpIsExpensive(true);
588
589 setMinCmpXchgSizeInBits(32);
590 setSupportsUnalignedAtomics(false);
591
592 PredictableSelectIsExpensive = false;
593
594 // We want to find all load dependencies for long chains of stores to enable
595 // merging into very wide vectors. The problem is with vectors with > 4
596 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
597 // vectors are a legal type, even though we have to split the loads
598 // usually. When we can more precisely specify load legality per address
599 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
600 // smarter so that they can figure out what to do in 2 iterations without all
601 // N > 4 stores on the same chain.
602 GatherAllAliasesMaxDepth = 16;
603
604 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
605 // about these during lowering.
606 MaxStoresPerMemcpy = 0xffffffff;
607 MaxStoresPerMemmove = 0xffffffff;
608 MaxStoresPerMemset = 0xffffffff;
609
610 // The expansion for 64-bit division is enormous.
611 if (static_cast<const AMDGPUTargetMachine &>(TM).getCLOpts().bypass_slow_div)
612 addBypassSlowDiv(SlowBitWidth: 64, FastBitWidth: 32);
613
614 setTargetDAGCombine({ISD::BITCAST, ISD::SHL,
615 ISD::SRA, ISD::SRL,
616 ISD::TRUNCATE, ISD::MUL,
617 ISD::SMUL_LOHI, ISD::UMUL_LOHI,
618 ISD::MULHU, ISD::MULHS,
619 ISD::SELECT, ISD::SELECT_CC,
620 ISD::STORE, ISD::FADD,
621 ISD::FSUB, ISD::FNEG,
622 ISD::FABS, ISD::AssertZext,
623 ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN});
624
625 setMaxAtomicSizeInBitsSupported(64);
626 setMaxDivRemBitWidthSupported(64);
627 setMaxLargeFPConvertBitWidthSupported(64);
628}
629
630//===----------------------------------------------------------------------===//
631// Target Information
632//===----------------------------------------------------------------------===//
633
634LLVM_READNONE
635static bool fnegFoldsIntoOpcode(unsigned Opc) {
636 switch (Opc) {
637 case ISD::FADD:
638 case ISD::FSUB:
639 case ISD::FMUL:
640 case ISD::FMA:
641 case ISD::FMAD:
642 case ISD::FMINNUM:
643 case ISD::FMAXNUM:
644 case ISD::FMINNUM_IEEE:
645 case ISD::FMAXNUM_IEEE:
646 case ISD::FMINIMUM:
647 case ISD::FMAXIMUM:
648 case ISD::FMINIMUMNUM:
649 case ISD::FMAXIMUMNUM:
650 case ISD::SELECT:
651 case ISD::FSIN:
652 case ISD::FTRUNC:
653 case ISD::FRINT:
654 case ISD::FNEARBYINT:
655 case ISD::FROUNDEVEN:
656 case ISD::FCANONICALIZE:
657 case AMDGPUISD::RCP:
658 case AMDGPUISD::RCP_LEGACY:
659 case AMDGPUISD::RCP_IFLAG:
660 case AMDGPUISD::SIN_HW:
661 case AMDGPUISD::FMUL_LEGACY:
662 case AMDGPUISD::FMIN_LEGACY:
663 case AMDGPUISD::FMAX_LEGACY:
664 case AMDGPUISD::FMED3:
665 // TODO: handle llvm.amdgcn.fma.legacy
666 return true;
667 case ISD::BITCAST:
668 llvm_unreachable("bitcast is special cased");
669 default:
670 return false;
671 }
672}
673
674static bool fnegFoldsIntoOp(const SDNode *N) {
675 unsigned Opc = N->getOpcode();
676 if (Opc == ISD::BITCAST) {
677 // TODO: Is there a benefit to checking the conditions performFNegCombine
678 // does? We don't for the other cases.
679 SDValue BCSrc = N->getOperand(Num: 0);
680 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
681 return BCSrc.getNumOperands() == 2 &&
682 BCSrc.getOperand(i: 1).getValueSizeInBits() == 32;
683 }
684
685 return BCSrc.getOpcode() == ISD::SELECT && BCSrc.getValueType() == MVT::f32;
686 }
687
688 return fnegFoldsIntoOpcode(Opc);
689}
690
691/// \p returns true if the operation will definitely need to use a 64-bit
692/// encoding, and thus will use a VOP3 encoding regardless of the source
693/// modifiers.
694LLVM_READONLY
695static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
696 return (N->getNumOperands() > 2 && N->getOpcode() != ISD::SELECT) ||
697 VT == MVT::f64;
698}
699
700/// Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the
701/// type for ISD::SELECT.
702LLVM_READONLY
703static bool selectSupportsSourceMods(const SDNode *N) {
704 // TODO: Only applies if select will be vector
705 return N->getValueType(ResNo: 0) == MVT::f32;
706}
707
708// Most FP instructions support source modifiers, but this could be refined
709// slightly.
710LLVM_READONLY
711static bool hasSourceMods(const SDNode *N) {
712 if (isa<MemSDNode>(Val: N))
713 return false;
714
715 switch (N->getOpcode()) {
716 case ISD::CopyToReg:
717 case ISD::FDIV:
718 case ISD::FREM:
719 case ISD::INLINEASM:
720 case ISD::INLINEASM_BR:
721 case AMDGPUISD::DIV_SCALE:
722 case ISD::INTRINSIC_W_CHAIN:
723
724 // TODO: Should really be looking at the users of the bitcast. These are
725 // problematic because bitcasts are used to legalize all stores to integer
726 // types.
727 case ISD::BITCAST:
728 return false;
729 case ISD::INTRINSIC_WO_CHAIN: {
730 switch (N->getConstantOperandVal(Num: 0)) {
731 case Intrinsic::amdgcn_interp_p1:
732 case Intrinsic::amdgcn_interp_p2:
733 case Intrinsic::amdgcn_interp_mov:
734 case Intrinsic::amdgcn_interp_p1_f16:
735 case Intrinsic::amdgcn_interp_p2_f16:
736 return false;
737 default:
738 return true;
739 }
740 }
741 case ISD::SELECT:
742 return selectSupportsSourceMods(N);
743 default:
744 return true;
745 }
746}
747
748bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N,
749 unsigned CostThreshold) {
750 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
751 // it is truly free to use a source modifier in all cases. If there are
752 // multiple users but for each one will necessitate using VOP3, there will be
753 // a code size increase. Try to avoid increasing code size unless we know it
754 // will save on the instruction count.
755 unsigned NumMayIncreaseSize = 0;
756 MVT VT = N->getValueType(ResNo: 0).getScalarType().getSimpleVT();
757
758 assert(!N->use_empty());
759
760 // XXX - Should this limit number of uses to check?
761 for (const SDNode *U : N->users()) {
762 if (!hasSourceMods(N: U))
763 return false;
764
765 if (!opMustUseVOP3Encoding(N: U, VT)) {
766 if (++NumMayIncreaseSize > CostThreshold)
767 return false;
768 }
769 }
770
771 return true;
772}
773
774EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
775 ISD::NodeType ExtendKind) const {
776 assert(!VT.isVector() && "only scalar expected");
777
778 // Round to the next multiple of 32-bits.
779 unsigned Size = VT.getSizeInBits();
780 if (Size <= 32)
781 return MVT::i32;
782 return EVT::getIntegerVT(Context, BitWidth: 32 * ((Size + 31) / 32));
783}
784
785unsigned AMDGPUTargetLowering::getVectorIdxWidth(const DataLayout &) const {
786 return 32;
787}
788
789bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
790 return true;
791}
792
793// The backend supports 32 and 64 bit floating point immediates.
794// FIXME: Why are we reporting vectors of FP immediates as legal?
795TargetLowering::FPImmAction
796AMDGPUTargetLowering::getFPImmAction(const APFloat &Imm, EVT VT,
797 bool ForCodeSize) const {
798 return isTypeLegal(VT: VT.getScalarType()) ? FPImmAction::Legal
799 : FPImmAction::ConstantPool;
800}
801
802// We don't want to shrink f64 / f32 constants.
803bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
804 EVT ScalarVT = VT.getScalarType();
805 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
806}
807
808bool AMDGPUTargetLowering::shouldReduceLoadWidth(
809 SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
810 std::optional<unsigned> ByteOffset) const {
811 // TODO: This may be worth removing. Check regression tests for diffs.
812 if (!TargetLoweringBase::shouldReduceLoadWidth(Load: N, ExtTy, NewVT, ByteOffset))
813 return false;
814
815 unsigned NewSize = NewVT.getStoreSizeInBits();
816
817 // If we are reducing to a 32-bit load or a smaller multi-dword load,
818 // this is always better.
819 if (NewSize >= 32)
820 return true;
821
822 EVT OldVT = N->getValueType(ResNo: 0);
823 unsigned OldSize = OldVT.getStoreSizeInBits();
824
825 MemSDNode *MN = cast<MemSDNode>(Val: N);
826 unsigned AS = MN->getAddressSpace();
827 // Do not shrink an aligned scalar load to sub-dword.
828 // Scalar engine cannot do sub-dword loads.
829 // Do not enable for gfx1250+ even though it has sub-dword loads because
830 // this will convert:
831 // i16 = trunc (zextload i16->i32)
832 // to:
833 // i16 = (load i16)
834 // This transformation will be reversed by LowerLOAD resulting in an infinite
835 // loop. Also, tablegen already has a pattern to match zextload i16->i32, but
836 // load i16 will not be matched since there is no instruction that does it.
837 if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) &&
838 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
839 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
840 (isa<LoadSDNode>(Val: N) && AS == AMDGPUAS::GLOBAL_ADDRESS &&
841 MN->isInvariant())) &&
842 AMDGPU::isUniformMMO(MMO: MN->getMemOperand()))
843 return false;
844
845 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
846 // extloads, so doing one requires using a buffer_load. In cases where we
847 // still couldn't use a scalar load, using the wider load shouldn't really
848 // hurt anything.
849
850 // If the old size already had to be an extload, there's no harm in continuing
851 // to reduce the width.
852 return (OldSize < 32);
853}
854
855bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
856 const SelectionDAG &DAG,
857 const MachineMemOperand &MMO) const {
858
859 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
860
861 if (LoadTy.getScalarType() == MVT::i32)
862 return false;
863
864 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
865 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
866
867 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
868 return false;
869
870 unsigned Fast = 0;
871 return allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
872 VT: CastTy, MMO, Fast: &Fast) &&
873 Fast;
874}
875
876// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
877// profitable with the expansion for 64-bit since it's generally good to
878// speculate things.
879bool AMDGPUTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
880 return true;
881}
882
883bool AMDGPUTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
884 return true;
885}
886
887bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const {
888 switch (N->getOpcode()) {
889 case ISD::EntryToken:
890 case ISD::TokenFactor:
891 return true;
892 case ISD::INTRINSIC_WO_CHAIN: {
893 unsigned IntrID = N->getConstantOperandVal(Num: 0);
894 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
895 }
896 case ISD::INTRINSIC_W_CHAIN: {
897 unsigned IntrID = N->getConstantOperandVal(Num: 1);
898 return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
899 }
900 case ISD::LOAD:
901 if (cast<LoadSDNode>(Val: N)->getMemOperand()->getAddrSpace() ==
902 AMDGPUAS::CONSTANT_ADDRESS_32BIT)
903 return true;
904 return false;
905 case AMDGPUISD::SETCC: // ballot-style instruction
906 return true;
907 }
908 return false;
909}
910
911SDValue AMDGPUTargetLowering::getNegatedExpression(
912 SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
913 NegatibleCost &Cost, unsigned Depth) const {
914
915 switch (Op.getOpcode()) {
916 case ISD::FMA:
917 case ISD::FMAD: {
918 // Negating a fma is not free if it has users without source mods.
919 if (!allUsesHaveSourceMods(N: Op.getNode()))
920 return SDValue();
921 break;
922 }
923 case AMDGPUISD::RCP: {
924 SDValue Src = Op.getOperand(i: 0);
925 EVT VT = Op.getValueType();
926 SDLoc SL(Op);
927
928 SDValue NegSrc = getNegatedExpression(Op: Src, DAG, LegalOperations,
929 ForCodeSize, Cost, Depth: Depth + 1);
930 if (NegSrc)
931 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SL, VT, Operand: NegSrc, Flags: Op->getFlags());
932 return SDValue();
933 }
934 default:
935 break;
936 }
937
938 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps: LegalOperations,
939 OptForSize: ForCodeSize, Cost, Depth);
940}
941
942//===---------------------------------------------------------------------===//
943// Target Properties
944//===---------------------------------------------------------------------===//
945
946bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
947 assert(VT.isFloatingPoint());
948
949 // Packed operations do not have a fabs modifier.
950 // Report this based on the end legalized type.
951 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
952}
953
954bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
955 assert(VT.isFloatingPoint());
956 // Report this based on the end legalized type.
957 VT = VT.getScalarType();
958 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
959}
960
961bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT,
962 unsigned NumElem,
963 unsigned AS) const {
964 return true;
965}
966
967bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
968 // There are few operations which truly have vector input operands. Any vector
969 // operation is going to involve operations on each component, and a
970 // build_vector will be a copy per element, so it always makes sense to use a
971 // build_vector input in place of the extracted element to avoid a copy into a
972 // super register.
973 //
974 // We should probably only do this if all users are extracts only, but this
975 // should be the common case.
976 return true;
977}
978
979bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
980 // Truncate is just accessing a subregister.
981
982 unsigned SrcSize = Source.getSizeInBits();
983 unsigned DestSize = Dest.getSizeInBits();
984
985 return DestSize < SrcSize && DestSize % 32 == 0 ;
986}
987
988bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
989 // Truncate is just accessing a subregister.
990
991 unsigned SrcSize = Source->getScalarSizeInBits();
992 unsigned DestSize = Dest->getScalarSizeInBits();
993
994 if (DestSize== 16 && Subtarget->has16BitInsts())
995 return SrcSize >= 32;
996
997 return DestSize < SrcSize && DestSize % 32 == 0;
998}
999
1000bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
1001 unsigned SrcSize = Src->getScalarSizeInBits();
1002 unsigned DestSize = Dest->getScalarSizeInBits();
1003
1004 if (SrcSize == 16 && Subtarget->has16BitInsts())
1005 return DestSize >= 32;
1006
1007 return SrcSize == 32 && DestSize == 64;
1008}
1009
1010bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
1011 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
1012 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
1013 // this will enable reducing 64-bit operations the 32-bit, which is always
1014 // good.
1015
1016 if (Src == MVT::i16)
1017 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1018
1019 return Src == MVT::i32 && Dest == MVT::i64;
1020}
1021
1022bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
1023 EVT DestVT) const {
1024 switch (N->getOpcode()) {
1025 case ISD::ABS:
1026 case ISD::ADD:
1027 case ISD::SUB:
1028 case ISD::SHL:
1029 case ISD::SRL:
1030 case ISD::SRA:
1031 case ISD::AND:
1032 case ISD::OR:
1033 case ISD::XOR:
1034 case ISD::MUL:
1035 case ISD::SETCC:
1036 case ISD::SELECT:
1037 case ISD::SMIN:
1038 case ISD::SMAX:
1039 case ISD::UMIN:
1040 case ISD::UMAX:
1041 case ISD::USUBSAT:
1042 case ISD::UADDSAT:
1043 if (isTypeLegal(VT: MVT::i16) &&
1044 (!DestVT.isVector() ||
1045 !isOperationLegal(Op: ISD::ADD, VT: MVT::v2i16))) { // Check if VOP3P
1046 // Don't narrow back down to i16 if promoted to i32 already.
1047 if (!N->isDivergent() && DestVT.isInteger() &&
1048 DestVT.getScalarSizeInBits() > 1 &&
1049 DestVT.getScalarSizeInBits() <= 16 &&
1050 SrcVT.getScalarSizeInBits() > 16) {
1051 return false;
1052 }
1053 }
1054 return true;
1055 default:
1056 break;
1057 }
1058
1059 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
1060 // limited number of native 64-bit operations. Shrinking an operation to fit
1061 // in a single 32-bit register should always be helpful. As currently used,
1062 // this is much less general than the name suggests, and is only used in
1063 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
1064 // not profitable, and may actually be harmful.
1065 if (isa<LoadSDNode>(Val: N))
1066 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
1067
1068 return true;
1069}
1070
1071bool AMDGPUTargetLowering::isDesirableToCommuteWithShift(
1072 const SDNode* N, CombineLevel Level) const {
1073 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
1074 N->getOpcode() == ISD::SRL) &&
1075 "Expected shift op");
1076
1077 SDValue ShiftLHS = N->getOperand(Num: 0);
1078 if (!ShiftLHS->hasOneUse())
1079 return false;
1080
1081 if (ShiftLHS.getOpcode() == ISD::SIGN_EXTEND &&
1082 !ShiftLHS.getOperand(i: 0)->hasOneUse())
1083 return false;
1084
1085 // Always commute pre-type legalization and right shifts.
1086 // We're looking for shl(or(x,y),z) patterns.
1087 if (Level < CombineLevel::AfterLegalizeTypes ||
1088 N->getOpcode() != ISD::SHL || N->getOperand(Num: 0).getOpcode() != ISD::OR)
1089 return true;
1090
1091 // If only user is a i32 right-shift, then don't destroy a BFE pattern.
1092 if (N->getValueType(ResNo: 0) == MVT::i32 && N->hasOneUse() &&
1093 (N->user_begin()->getOpcode() == ISD::SRA ||
1094 N->user_begin()->getOpcode() == ISD::SRL))
1095 return false;
1096
1097 // Don't destroy or(shl(load_zext(),c), load_zext()) patterns.
1098 auto IsShiftAndLoad = [](SDValue LHS, SDValue RHS) {
1099 if (LHS.getOpcode() != ISD::SHL)
1100 return false;
1101 auto *RHSLd = dyn_cast<LoadSDNode>(Val&: RHS);
1102 auto *LHS0 = dyn_cast<LoadSDNode>(Val: LHS.getOperand(i: 0));
1103 auto *LHS1 = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
1104 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() == ISD::ZEXTLOAD &&
1105 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1106 RHSLd->getExtensionType() == ISD::ZEXTLOAD;
1107 };
1108 SDValue LHS = N->getOperand(Num: 0).getOperand(i: 0);
1109 SDValue RHS = N->getOperand(Num: 0).getOperand(i: 1);
1110 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS));
1111}
1112
1113//===---------------------------------------------------------------------===//
1114// TargetLowering Callbacks
1115//===---------------------------------------------------------------------===//
1116
1117CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
1118 bool IsVarArg) {
1119 switch (CC) {
1120 case CallingConv::AMDGPU_VS:
1121 case CallingConv::AMDGPU_GS:
1122 case CallingConv::AMDGPU_PS:
1123 case CallingConv::AMDGPU_CS:
1124 case CallingConv::AMDGPU_HS:
1125 case CallingConv::AMDGPU_ES:
1126 case CallingConv::AMDGPU_LS:
1127 return CC_AMDGPU;
1128 case CallingConv::AMDGPU_CS_Chain:
1129 case CallingConv::AMDGPU_CS_ChainPreserve:
1130 return CC_AMDGPU_CS_CHAIN;
1131 case CallingConv::C:
1132 case CallingConv::Fast:
1133 case CallingConv::Cold:
1134 return CC_AMDGPU_Func;
1135 case CallingConv::AMDGPU_Gfx:
1136 case CallingConv::AMDGPU_Gfx_WholeWave:
1137 return CC_SI_Gfx;
1138 case CallingConv::AMDGPU_KERNEL:
1139 case CallingConv::SPIR_KERNEL:
1140 default:
1141 reportFatalUsageError(reason: "unsupported calling convention for call");
1142 }
1143}
1144
1145CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
1146 bool IsVarArg) {
1147 switch (CC) {
1148 case CallingConv::AMDGPU_KERNEL:
1149 case CallingConv::SPIR_KERNEL:
1150 llvm_unreachable("kernels should not be handled here");
1151 case CallingConv::AMDGPU_VS:
1152 case CallingConv::AMDGPU_GS:
1153 case CallingConv::AMDGPU_PS:
1154 case CallingConv::AMDGPU_CS:
1155 case CallingConv::AMDGPU_CS_Chain:
1156 case CallingConv::AMDGPU_CS_ChainPreserve:
1157 case CallingConv::AMDGPU_HS:
1158 case CallingConv::AMDGPU_ES:
1159 case CallingConv::AMDGPU_LS:
1160 return RetCC_SI_Shader;
1161 case CallingConv::AMDGPU_Gfx:
1162 case CallingConv::AMDGPU_Gfx_WholeWave:
1163 return RetCC_SI_Gfx;
1164 case CallingConv::C:
1165 case CallingConv::Fast:
1166 case CallingConv::Cold:
1167 return RetCC_AMDGPU_Func;
1168 default:
1169 reportFatalUsageError(reason: "unsupported calling convention");
1170 }
1171}
1172
1173/// The SelectionDAGBuilder will automatically promote function arguments
1174/// with illegal types. However, this does not work for the AMDGPU targets
1175/// since the function arguments are stored in memory as these illegal types.
1176/// In order to handle this properly we need to get the original types sizes
1177/// from the LLVM IR Function and fixup the ISD:InputArg values before
1178/// passing them to AnalyzeFormalArguments()
1179
1180/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
1181/// input values across multiple registers. Each item in the Ins array
1182/// represents a single value that will be stored in registers. Ins[x].VT is
1183/// the value type of the value that will be stored in the register, so
1184/// whatever SDNode we lower the argument to needs to be this type.
1185///
1186/// In order to correctly lower the arguments we need to know the size of each
1187/// argument. Since Ins[x].VT gives us the size of the register that will
1188/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
1189/// for the original function argument so that we can deduce the correct memory
1190/// type to use for Ins[x]. In most cases the correct memory type will be
1191/// Ins[x].ArgVT. However, this will not always be the case. If, for example,
1192/// we have a kernel argument of type v8i8, this argument will be split into
1193/// 8 parts and each part will be represented by its own item in the Ins array.
1194/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
1195/// the argument before it was split. From this, we deduce that the memory type
1196/// for each individual part is i8. We pass the memory type as LocVT to the
1197/// calling convention analysis function and the register type (Ins[x].VT) as
1198/// the ValVT.
1199void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
1200 CCState &State,
1201 const SmallVectorImpl<ISD::InputArg> &Ins) const {
1202 const MachineFunction &MF = State.getMachineFunction();
1203 const Function &Fn = MF.getFunction();
1204 LLVMContext &Ctx = Fn.getContext();
1205 const unsigned ExplicitOffset = Subtarget->getExplicitKernelArgOffset();
1206 CallingConv::ID CC = Fn.getCallingConv();
1207
1208 Align MaxAlign = Align(1);
1209 uint64_t ExplicitArgOffset = 0;
1210 const DataLayout &DL = Fn.getDataLayout();
1211
1212 unsigned InIndex = 0;
1213
1214 for (const Argument &Arg : Fn.args()) {
1215 const bool IsByRef = Arg.hasByRefAttr();
1216 Type *BaseArgTy = Arg.getType();
1217 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1218 Align Alignment = DL.getValueOrABITypeAlignment(
1219 Alignment: IsByRef ? Arg.getParamAlign() : std::nullopt, Ty: MemArgTy);
1220 MaxAlign = std::max(a: Alignment, b: MaxAlign);
1221 uint64_t AllocSize = DL.getTypeAllocSize(Ty: MemArgTy);
1222
1223 uint64_t ArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + ExplicitOffset;
1224 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: Alignment) + AllocSize;
1225
1226 // We're basically throwing away everything passed into us and starting over
1227 // to get accurate in-memory offsets. The "PartOffset" is completely useless
1228 // to us as computed in Ins.
1229 //
1230 // We also need to figure out what type legalization is trying to do to get
1231 // the correct memory offsets.
1232
1233 SmallVector<EVT, 16> ValueVTs;
1234 SmallVector<uint64_t, 16> Offsets;
1235 ComputeValueVTs(TLI: *this, DL, Ty: BaseArgTy, ValueVTs, /*MemVTs=*/nullptr,
1236 FixedOffsets: &Offsets, StartingOffset: ArgOffset);
1237
1238 for (unsigned Value = 0, NumValues = ValueVTs.size();
1239 Value != NumValues; ++Value) {
1240 uint64_t BasePartOffset = Offsets[Value];
1241
1242 EVT ArgVT = ValueVTs[Value];
1243 EVT MemVT = ArgVT;
1244 MVT RegisterVT = getRegisterTypeForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1245 unsigned NumRegs = getNumRegistersForCallingConv(Context&: Ctx, CC, VT: ArgVT);
1246
1247 if (NumRegs == 1) {
1248 // This argument is not split, so the IR type is the memory type.
1249 if (ArgVT.isExtended()) {
1250 // We have an extended type, like i24, so we should just use the
1251 // register type.
1252 MemVT = RegisterVT;
1253 } else {
1254 MemVT = ArgVT;
1255 }
1256 } else if (ArgVT.isVector() && RegisterVT.isVector() &&
1257 ArgVT.getScalarType() == RegisterVT.getScalarType()) {
1258 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
1259 // We have a vector value which has been split into a vector with
1260 // the same scalar type, but fewer elements. This should handle
1261 // all the floating-point vector types.
1262 MemVT = RegisterVT;
1263 } else if (ArgVT.isVector() &&
1264 ArgVT.getVectorNumElements() == NumRegs) {
1265 // This arg has been split so that each element is stored in a separate
1266 // register.
1267 MemVT = ArgVT.getScalarType();
1268 } else if (ArgVT.isExtended()) {
1269 // We have an extended type, like i65.
1270 MemVT = RegisterVT;
1271 } else {
1272 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
1273 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0);
1274 if (RegisterVT.isInteger()) {
1275 MemVT = EVT::getIntegerVT(Context&: State.getContext(), BitWidth: MemoryBits);
1276 } else if (RegisterVT.isVector()) {
1277 assert(!RegisterVT.getScalarType().isFloatingPoint());
1278 unsigned NumElements = RegisterVT.getVectorNumElements();
1279 assert(MemoryBits % NumElements == 0);
1280 // This vector type has been split into another vector type with
1281 // a different elements size.
1282 EVT ScalarVT = EVT::getIntegerVT(Context&: State.getContext(),
1283 BitWidth: MemoryBits / NumElements);
1284 MemVT = EVT::getVectorVT(Context&: State.getContext(), VT: ScalarVT, NumElements);
1285 } else {
1286 llvm_unreachable("cannot deduce memory type.");
1287 }
1288 }
1289
1290 // Convert one element vectors to scalar.
1291 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
1292 MemVT = MemVT.getScalarType();
1293
1294 // Round up vec3/vec5 argument.
1295 if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
1296 MemVT = MemVT.getPow2VectorType(Context&: State.getContext());
1297 } else if (!MemVT.isSimple() && !MemVT.isVector()) {
1298 MemVT = MemVT.getRoundIntegerType(Context&: State.getContext());
1299 }
1300
1301 unsigned PartOffset = 0;
1302 for (unsigned i = 0; i != NumRegs; ++i) {
1303 State.addLoc(V: CCValAssign::getCustomMem(ValNo: InIndex++, ValVT: RegisterVT,
1304 Offset: BasePartOffset + PartOffset,
1305 LocVT: MemVT.getSimpleVT(),
1306 HTP: CCValAssign::Full));
1307 PartOffset += MemVT.getStoreSize();
1308 }
1309 }
1310 }
1311}
1312
1313SDValue AMDGPUTargetLowering::LowerReturn(
1314 SDValue Chain, CallingConv::ID CallConv,
1315 bool isVarArg,
1316 const SmallVectorImpl<ISD::OutputArg> &Outs,
1317 const SmallVectorImpl<SDValue> &OutVals,
1318 const SDLoc &DL, SelectionDAG &DAG) const {
1319 // FIXME: Fails for r600 tests
1320 //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
1321 // "wave terminate should not have return values");
1322 return DAG.getNode(Opcode: AMDGPUISD::ENDPGM, DL, VT: MVT::Other, Operand: Chain);
1323}
1324
1325//===---------------------------------------------------------------------===//
1326// Target specific lowering
1327//===---------------------------------------------------------------------===//
1328
1329/// Selects the correct CCAssignFn for a given CallingConvention value.
1330CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1331 bool IsVarArg) {
1332 return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1333}
1334
1335CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1336 bool IsVarArg) {
1337 return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1338}
1339
1340SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1341 SelectionDAG &DAG,
1342 MachineFrameInfo &MFI,
1343 int ClobberedFI) const {
1344 SmallVector<SDValue, 8> ArgChains;
1345 int64_t FirstByte = MFI.getObjectOffset(ObjectIdx: ClobberedFI);
1346 int64_t LastByte = FirstByte + MFI.getObjectSize(ObjectIdx: ClobberedFI) - 1;
1347
1348 // Include the original chain at the beginning of the list. When this is
1349 // used by target LowerCall hooks, this helps legalize find the
1350 // CALLSEQ_BEGIN node.
1351 ArgChains.push_back(Elt: Chain);
1352
1353 // Add a chain value for each stack argument corresponding
1354 for (SDNode *U : DAG.getEntryNode().getNode()->users()) {
1355 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U)) {
1356 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr())) {
1357 if (FI->getIndex() < 0) {
1358 int64_t InFirstByte = MFI.getObjectOffset(ObjectIdx: FI->getIndex());
1359 int64_t InLastByte = InFirstByte;
1360 InLastByte += MFI.getObjectSize(ObjectIdx: FI->getIndex()) - 1;
1361
1362 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1363 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1364 ArgChains.push_back(Elt: SDValue(L, 1));
1365 }
1366 }
1367 }
1368 }
1369
1370 // Build a tokenfactor for all the chains.
1371 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
1372}
1373
1374SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1375 SmallVectorImpl<SDValue> &InVals,
1376 StringRef Reason) const {
1377 SDValue Callee = CLI.Callee;
1378 SelectionDAG &DAG = CLI.DAG;
1379
1380 const Function &Fn = DAG.getMachineFunction().getFunction();
1381
1382 StringRef FuncName("<unknown>");
1383
1384 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Val&: Callee))
1385 FuncName = G->getSymbol();
1386 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
1387 FuncName = G->getGlobal()->getName();
1388
1389 DAG.getContext()->diagnose(
1390 DI: DiagnosticInfoUnsupported(Fn, Reason + FuncName, CLI.DL.getDebugLoc()));
1391
1392 if (!CLI.IsTailCall) {
1393 for (ISD::InputArg &Arg : CLI.Ins)
1394 InVals.push_back(Elt: DAG.getPOISON(VT: Arg.VT));
1395 }
1396
1397 // FIXME: Hack because R600 doesn't handle callseq pseudos yet.
1398 if (getTargetMachine().getTargetTriple().getArch() == Triple::r600)
1399 return CLI.Chain;
1400
1401 SDValue Chain = DAG.getCALLSEQ_START(Chain: CLI.Chain, InSize: 0, OutSize: 0, DL: CLI.DL);
1402 return DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, /*InGlue=*/Glue: SDValue(), DL: CLI.DL);
1403}
1404
1405SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1406 SmallVectorImpl<SDValue> &InVals) const {
1407 return lowerUnhandledCall(CLI, InVals, Reason: "unsupported call to function ");
1408}
1409
1410SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1411 SelectionDAG &DAG) const {
1412 const Function &Fn = DAG.getMachineFunction().getFunction();
1413
1414 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1415 Fn, "unsupported dynamic alloca", SDLoc(Op).getDebugLoc()));
1416 auto Ops = {DAG.getConstant(Val: 0, DL: SDLoc(), VT: Op.getValueType()), Op.getOperand(i: 0)};
1417 return DAG.getMergeValues(Ops, dl: SDLoc());
1418}
1419
1420SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1421 SelectionDAG &DAG) const {
1422 switch (Op.getOpcode()) {
1423 default:
1424 Op->print(OS&: errs(), G: &DAG);
1425 llvm_unreachable("Custom lowering code for this "
1426 "instruction is not implemented yet!");
1427 break;
1428 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1429 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1430 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1431 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1432 case ISD::SDIVREM:
1433 return LowerSDIVREM(Op, DAG);
1434 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1435 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1436 case ISD::FRINT: return LowerFRINT(Op, DAG);
1437 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1438 case ISD::FROUNDEVEN:
1439 return LowerFROUNDEVEN(Op, DAG);
1440 case ISD::FROUND: return LowerFROUND(Op, DAG);
1441 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1442 case ISD::FLOG2:
1443 return LowerFLOG2(Op, DAG);
1444 case ISD::FLOG:
1445 case ISD::FLOG10:
1446 return LowerFLOGCommon(Op, DAG);
1447 case ISD::FEXP:
1448 case ISD::FEXP10:
1449 return lowerFEXP(Op, DAG);
1450 case ISD::FEXP2:
1451 return lowerFEXP2(Op, DAG);
1452 case ISD::FPOW:
1453 return lowerFPOW(Op, DAG);
1454 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1455 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1456 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1457 case ISD::FP_TO_SINT:
1458 case ISD::FP_TO_UINT:
1459 return LowerFP_TO_INT(Op, DAG);
1460 case ISD::FP_TO_SINT_SAT:
1461 case ISD::FP_TO_UINT_SAT:
1462 return LowerFP_TO_INT_SAT(Op, DAG);
1463 case ISD::CTTZ:
1464 case ISD::CTTZ_ZERO_POISON:
1465 case ISD::CTLZ:
1466 case ISD::CTLZ_ZERO_POISON:
1467 return LowerCTLZ_CTTZ(Op, DAG);
1468 case ISD::CTLS:
1469 return LowerCTLS(Op, DAG);
1470 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1471 }
1472 return Op;
1473}
1474
1475void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
1476 SmallVectorImpl<SDValue> &Results,
1477 SelectionDAG &DAG) const {
1478 switch (N->getOpcode()) {
1479 case ISD::SIGN_EXTEND_INREG:
1480 // Different parts of legalization seem to interpret which type of
1481 // sign_extend_inreg is the one to check for custom lowering. The extended
1482 // from type is what really matters, but some places check for custom
1483 // lowering of the result type. This results in trying to use
1484 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1485 // nothing here and let the illegal result integer be handled normally.
1486 return;
1487 case ISD::FLOG2:
1488 if (SDValue Lowered = LowerFLOG2(Op: SDValue(N, 0), DAG))
1489 Results.push_back(Elt: Lowered);
1490 return;
1491 case ISD::FLOG:
1492 case ISD::FLOG10:
1493 if (SDValue Lowered = LowerFLOGCommon(Op: SDValue(N, 0), DAG))
1494 Results.push_back(Elt: Lowered);
1495 return;
1496 case ISD::FEXP2:
1497 if (SDValue Lowered = lowerFEXP2(Op: SDValue(N, 0), DAG))
1498 Results.push_back(Elt: Lowered);
1499 return;
1500 case ISD::FEXP:
1501 case ISD::FEXP10:
1502 if (SDValue Lowered = lowerFEXP(Op: SDValue(N, 0), DAG))
1503 Results.push_back(Elt: Lowered);
1504 return;
1505 case ISD::CTLZ:
1506 case ISD::CTLZ_ZERO_POISON:
1507 if (auto Lowered = lowerCTLZResults(Op: SDValue(N, 0u), DAG))
1508 Results.push_back(Elt: Lowered);
1509 return;
1510 default:
1511 return;
1512 }
1513}
1514
1515SDValue AMDGPUTargetLowering::LowerBlockAddress(SDValue Op,
1516 SelectionDAG &DAG) const {
1517 BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val&: Op);
1518 SDLoc SL(Op);
1519 EVT VT = Op.getValueType();
1520 return DAG.getTargetBlockAddress(BA: BA->getBlockAddress(), VT, Offset: BA->getOffset(),
1521 TargetFlags: BA->getTargetFlags());
1522}
1523
1524SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI,
1525 SDValue Op,
1526 SelectionDAG &DAG) const {
1527
1528 const DataLayout &DL = DAG.getDataLayout();
1529 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Val&: Op);
1530 const GlobalValue *GV = G->getGlobal();
1531
1532 if (G->getAddressSpace() == AMDGPUAS::BARRIER) {
1533 const GlobalVariable *GVar = cast<GlobalVariable>(Val: GV);
1534
1535 if (!AMDGPU::isNamedBarrier(GV: *GVar)) {
1536 const Function &Fn = DAG.getMachineFunction().getFunction();
1537 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1538 Fn, "unsupported use of BARRIER address space",
1539 SDLoc(Op).getDebugLoc(), DS_Error));
1540 return DAG.getPOISON(VT: Op.getValueType());
1541 }
1542
1543 unsigned Offset = MFI->allocateBarrierGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1544 return DAG.getConstant(Val: Offset, DL: SDLoc(Op), VT: Op.getValueType());
1545 }
1546
1547 if (!MFI->isModuleEntryFunction()) {
1548 if (std::optional<uint32_t> Address =
1549 AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(
1550 GV: *GV, AS: AMDGPUAS::LOCAL_ADDRESS)) {
1551 return DAG.getConstant(Val: *Address, DL: SDLoc(Op), VT: Op.getValueType());
1552 }
1553 }
1554
1555 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1556 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
1557 if (!MFI->isModuleEntryFunction() &&
1558 GV->getName() != "llvm.amdgcn.module.lds") {
1559 SDLoc DL(Op);
1560 const Function &Fn = DAG.getMachineFunction().getFunction();
1561 DAG.getContext()->diagnose(DI: DiagnosticInfoUnsupported(
1562 Fn, "local memory global used by non-kernel function",
1563 DL.getDebugLoc(), DS_Warning));
1564
1565 // We currently don't have a way to correctly allocate LDS objects that
1566 // aren't directly associated with a kernel. We do force inlining of
1567 // functions that use local objects. However, if these dead functions are
1568 // not eliminated, we don't want a compile time error. Just emit a warning
1569 // and a trap, since there should be no callable path here.
1570 SDValue Trap = DAG.getNode(Opcode: ISD::TRAP, DL, VT: MVT::Other, Operand: DAG.getEntryNode());
1571 SDValue OutputChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
1572 N1: Trap, N2: DAG.getRoot());
1573 DAG.setRoot(OutputChain);
1574 return DAG.getPOISON(VT: Op.getValueType());
1575 }
1576
1577 // TODO: We could emit code to handle the initialization somewhere.
1578 // We ignore the initializer for now and legalize it to allow selection.
1579 // The initializer will anyway get errored out during assembly emission.
1580 unsigned Offset = MFI->allocateLDSGlobal(DL, GV: *cast<GlobalVariable>(Val: GV));
1581 // A constant byte offset (e.g. from a GEP into an array of named barriers)
1582 // folds directly into the allocated LDS address.
1583 return DAG.getConstant(Val: Offset + G->getOffset(), DL: SDLoc(Op),
1584 VT: Op.getValueType());
1585 }
1586 return SDValue();
1587}
1588
1589SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1590 SelectionDAG &DAG) const {
1591 SmallVector<SDValue, 8> Args;
1592 SDLoc SL(Op);
1593
1594 EVT VT = Op.getValueType();
1595 if (VT.getVectorElementType().getSizeInBits() < 32) {
1596 unsigned OpBitSize = Op.getOperand(i: 0).getValueType().getSizeInBits();
1597 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1598 unsigned NewNumElt = OpBitSize / 32;
1599 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1600 : EVT::getVectorVT(Context&: *DAG.getContext(),
1601 VT: MVT::i32, NumElements: NewNumElt);
1602 for (const SDUse &U : Op->ops()) {
1603 SDValue In = U.get();
1604 SDValue NewIn = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewEltVT, Operand: In);
1605 if (NewNumElt > 1)
1606 DAG.ExtractVectorElements(Op: NewIn, Args);
1607 else
1608 Args.push_back(Elt: NewIn);
1609 }
1610
1611 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
1612 NumElements: NewNumElt * Op.getNumOperands());
1613 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1614 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: BV);
1615 }
1616 }
1617
1618 for (const SDUse &U : Op->ops())
1619 DAG.ExtractVectorElements(Op: U.get(), Args);
1620
1621 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1622}
1623
1624SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1625 SelectionDAG &DAG) const {
1626 SDLoc SL(Op);
1627 SmallVector<SDValue, 8> Args;
1628 unsigned Start = Op.getConstantOperandVal(i: 1);
1629 EVT VT = Op.getValueType();
1630 EVT SrcVT = Op.getOperand(i: 0).getValueType();
1631
1632 if (VT.getScalarSizeInBits() == 16 && Start % 2 == 0) {
1633 unsigned NumElt = VT.getVectorNumElements();
1634 unsigned NumSrcElt = SrcVT.getVectorNumElements();
1635 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 && "expect legal types");
1636
1637 // Extract 32-bit registers at a time.
1638 EVT NewSrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumSrcElt / 2);
1639 EVT NewVT = NumElt == 2
1640 ? MVT::i32
1641 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32, NumElements: NumElt / 2);
1642 SDValue Tmp = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NewSrcVT, Operand: Op.getOperand(i: 0));
1643
1644 DAG.ExtractVectorElements(Op: Tmp, Args, Start: Start / 2, Count: NumElt / 2);
1645 if (NumElt == 2)
1646 Tmp = Args[0];
1647 else
1648 Tmp = DAG.getBuildVector(VT: NewVT, DL: SL, Ops: Args);
1649
1650 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Tmp);
1651 }
1652
1653 DAG.ExtractVectorElements(Op: Op.getOperand(i: 0), Args, Start,
1654 Count: VT.getVectorNumElements());
1655
1656 return DAG.getBuildVector(VT: Op.getValueType(), DL: SL, Ops: Args);
1657}
1658
1659// TODO: Handle fabs too
1660static SDValue peekFNeg(SDValue Val) {
1661 if (Val.getOpcode() == ISD::FNEG)
1662 return Val.getOperand(i: 0);
1663
1664 return Val;
1665}
1666
1667// SelectionDAG twin of AMDGPUCombinerHelper::canIgnoreLegacyMinMaxTies.
1668static bool canIgnoreLegacyMinMaxTies(const SelectionDAG &DAG,
1669 SDNodeFlags Flags, SDValue LHS,
1670 SDValue RHS) {
1671 return Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
1672 DAG.isKnownNeverLogicalZero(Op: RHS);
1673}
1674
1675SDValue AMDGPUTargetLowering::combineFMinMaxLegacyImpl(
1676 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1677 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1678 SelectionDAG &DAG = DCI.DAG;
1679 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Val&: CC)->get();
1680 assert(CCOpcode != ISD::SETCC_INVALID && "Invalid setcc condcode!");
1681
1682 switch (CCOpcode) {
1683 case ISD::SETOLE:
1684 case ISD::SETOLT:
1685 case ISD::SETLE:
1686 case ISD::SETLT:
1687 case ISD::SETOGE:
1688 case ISD::SETOGT:
1689 case ISD::SETGE:
1690 case ISD::SETGT:
1691 // Only do this after legalization to avoid interfering with other combines
1692 // which might occur.
1693 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1694 !DCI.isCalledByLegalizer())
1695 return SDValue();
1696 break;
1697 default:
1698 break;
1699 }
1700
1701 // Canonicalize so the select returns the compare's LHS on a true predicate.
1702 if (LHS != True)
1703 CCOpcode = ISD::getSetCCInverse(Operation: CCOpcode, Type: VT);
1704
1705 unsigned Opc;
1706 bool Swap; // Emit (rhs, lhs) instead of (lhs, rhs).
1707 switch (CCOpcode) {
1708 case ISD::SETOLT:
1709 case ISD::SETLT:
1710 case ISD::SETOLE:
1711 Opc = AMDGPUISD::FMIN_LEGACY;
1712 Swap = false;
1713 break;
1714 case ISD::SETULE:
1715 case ISD::SETLE:
1716 case ISD::SETULT:
1717 Opc = AMDGPUISD::FMIN_LEGACY;
1718 Swap = true;
1719 break;
1720 case ISD::SETOGE:
1721 case ISD::SETGE:
1722 case ISD::SETOGT:
1723 Opc = AMDGPUISD::FMAX_LEGACY;
1724 Swap = false;
1725 break;
1726 case ISD::SETUGT:
1727 case ISD::SETGT:
1728 case ISD::SETUGE:
1729 Opc = AMDGPUISD::FMAX_LEGACY;
1730 Swap = true;
1731 break;
1732 default:
1733 return SDValue();
1734 }
1735
1736 // For these predicates the NaN-correct operand order is the signed zero
1737 // tie-incorrect one, so the fold needs the tie to be unobservable.
1738 if ((CCOpcode == ISD::SETOLE || CCOpcode == ISD::SETULT ||
1739 CCOpcode == ISD::SETOGT || CCOpcode == ISD::SETUGE) &&
1740 !canIgnoreLegacyMinMaxTies(DAG, Flags, LHS, RHS))
1741 return SDValue();
1742
1743 if (Swap)
1744 std::swap(a&: LHS, b&: RHS);
1745 return DAG.getNode(Opcode: Opc, DL, VT, N1: LHS, N2: RHS, Flags);
1746}
1747
1748/// Generate Min/Max node
1749SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(
1750 const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True,
1751 SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const {
1752 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
1753 return combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True, False, CC, Flags,
1754 DCI);
1755
1756 SelectionDAG &DAG = DCI.DAG;
1757
1758 // If we can't directly match this, try to see if we can fold an fneg to
1759 // match.
1760
1761 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
1762 ConstantFPSDNode *CFalse = dyn_cast<ConstantFPSDNode>(Val&: False);
1763 SDValue NegTrue = peekFNeg(Val: True);
1764
1765 // Undo the combine foldFreeOpFromSelect does if it helps us match the
1766 // fmin/fmax.
1767 //
1768 // select (fcmp olt (lhs, K)), (fneg lhs), -K
1769 // -> fneg (fmin_legacy lhs, K)
1770 //
1771 // TODO: Use getNegatedExpression
1772 if (LHS == NegTrue && CFalse && CRHS) {
1773 APFloat NegRHS = neg(X: CRHS->getValueAPF());
1774 if (NegRHS == CFalse->getValueAPF()) {
1775 SDValue Combined = combineFMinMaxLegacyImpl(DL, VT, LHS, RHS, True: NegTrue,
1776 False, CC, Flags, DCI);
1777 if (Combined)
1778 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Combined);
1779 return SDValue();
1780 }
1781 }
1782
1783 return SDValue();
1784}
1785
1786std::pair<SDValue, SDValue>
1787AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1788 SDLoc SL(Op);
1789
1790 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1791
1792 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
1793 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1794
1795 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: Zero);
1796 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1797
1798 return std::pair(Lo, Hi);
1799}
1800
1801SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1802 SDLoc SL(Op);
1803
1804 SDValue Vec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::v2i32, Operand: Op);
1805 const SDValue One = DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32);
1806 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: MVT::i32, N1: Vec, N2: One);
1807}
1808
1809// Split a vector type into two parts. The first part is a power of two vector.
1810// The second part is whatever is left over, and is a scalar if it would
1811// otherwise be a 1-vector.
1812std::pair<EVT, EVT>
1813AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1814 EVT LoVT, HiVT;
1815 EVT EltVT = VT.getVectorElementType();
1816 unsigned NumElts = VT.getVectorNumElements();
1817 unsigned LoNumElts = PowerOf2Ceil(A: (NumElts + 1) / 2);
1818 LoVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: LoNumElts);
1819 HiVT = NumElts - LoNumElts == 1
1820 ? EltVT
1821 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts - LoNumElts);
1822 return std::pair(LoVT, HiVT);
1823}
1824
1825// Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1826// scalar.
1827std::pair<SDValue, SDValue>
1828AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1829 const EVT &LoVT, const EVT &HiVT,
1830 SelectionDAG &DAG) const {
1831 EVT VT = N.getValueType();
1832 assert(LoVT.getVectorNumElements() +
1833 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
1834 VT.getVectorNumElements() &&
1835 "More vector elements requested than available!");
1836 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: LoVT, N1: N,
1837 N2: DAG.getVectorIdxConstant(Val: 0, DL));
1838
1839 unsigned LoNumElts = LoVT.getVectorNumElements();
1840
1841 if (HiVT.isVector()) {
1842 unsigned HiNumElts = HiVT.getVectorNumElements();
1843 if ((VT.getVectorNumElements() % HiNumElts) == 0) {
1844 // Avoid creating an extract_subvector with an index that isn't a multiple
1845 // of the result type.
1846 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
1847 N2: DAG.getConstant(Val: LoNumElts, DL, VT: MVT::i32));
1848 return {Lo, Hi};
1849 }
1850
1851 SmallVector<SDValue, 8> Elts;
1852 DAG.ExtractVectorElements(Op: N, Args&: Elts, /*Start=*/LoNumElts,
1853 /*Count=*/HiNumElts);
1854 SDValue Hi = DAG.getBuildVector(VT: HiVT, DL, Ops: Elts);
1855 return {Lo, Hi};
1856 }
1857
1858 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: HiVT, N1: N,
1859 N2: DAG.getVectorIdxConstant(Val: LoNumElts, DL));
1860 return {Lo, Hi};
1861}
1862
1863SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1864 SelectionDAG &DAG) const {
1865 LoadSDNode *Load = cast<LoadSDNode>(Val: Op);
1866 EVT VT = Op.getValueType();
1867 SDLoc SL(Op);
1868
1869
1870 // If this is a 2 element vector, we really want to scalarize and not create
1871 // weird 1 element vectors.
1872 if (VT.getVectorNumElements() == 2) {
1873 SDValue Ops[2];
1874 std::tie(args&: Ops[0], args&: Ops[1]) = scalarizeVectorLoad(LD: Load, DAG);
1875 return DAG.getMergeValues(Ops, dl: SL);
1876 }
1877
1878 SDValue BasePtr = Load->getBasePtr();
1879 EVT MemVT = Load->getMemoryVT();
1880
1881 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1882
1883 EVT LoVT, HiVT;
1884 EVT LoMemVT, HiMemVT;
1885 SDValue Lo, Hi;
1886
1887 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1888 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1889 std::tie(args&: Lo, args&: Hi) = splitVector(N: Op, DL: SL, LoVT, HiVT, DAG);
1890
1891 unsigned Size = LoMemVT.getStoreSize();
1892 Align BaseAlign = Load->getAlign();
1893 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1894
1895 SDValue LoLoad = DAG.getExtLoad(
1896 ExtType: Load->getExtensionType(), dl: SL, VT: LoVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1897 MemVT: LoMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1898 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Size));
1899 SDValue HiLoad = DAG.getExtLoad(
1900 ExtType: Load->getExtensionType(), dl: SL, VT: HiVT, Chain: Load->getChain(), Ptr: HiPtr,
1901 PtrInfo: SrcValue.getWithOffset(O: LoMemVT.getStoreSize()), MemVT: HiMemVT, Alignment: HiAlign,
1902 MMOFlags: Load->getMemOperand()->getFlags(), Metadata: Load->getAAInfo());
1903
1904 SDValue Join;
1905 if (LoVT == HiVT) {
1906 // This is the case that the vector is power of two so was evenly split.
1907 Join = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT, N1: LoLoad, N2: HiLoad);
1908 } else {
1909 Join = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SL, VT, N1: DAG.getPOISON(VT), N2: LoLoad,
1910 N3: DAG.getVectorIdxConstant(Val: 0, DL: SL));
1911 Join = DAG.getNode(
1912 Opcode: HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, DL: SL,
1913 VT, N1: Join, N2: HiLoad,
1914 N3: DAG.getVectorIdxConstant(Val: LoVT.getVectorNumElements(), DL: SL));
1915 }
1916
1917 SDValue Ops[] = {Join, DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other,
1918 N1: LoLoad.getValue(R: 1), N2: HiLoad.getValue(R: 1))};
1919
1920 return DAG.getMergeValues(Ops, dl: SL);
1921}
1922
1923SDValue AMDGPUTargetLowering::WidenOrSplitVectorLoad(SDValue Op,
1924 SelectionDAG &DAG) const {
1925 LoadSDNode *Load = cast<LoadSDNode>(Val&: Op);
1926 EVT VT = Op.getValueType();
1927 SDValue BasePtr = Load->getBasePtr();
1928 EVT MemVT = Load->getMemoryVT();
1929 SDLoc SL(Op);
1930 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1931 Align BaseAlign = Load->getAlign();
1932 unsigned NumElements = MemVT.getVectorNumElements();
1933
1934 // Widen from vec3 to vec4 when the load is at least 8-byte aligned
1935 // or 16-byte fully dereferenceable. Otherwise, split the vector load.
1936 if (NumElements != 3 ||
1937 (BaseAlign < Align(8) &&
1938 !SrcValue.isDereferenceable(Size: 16, C&: *DAG.getContext(), DL: DAG.getDataLayout())))
1939 return SplitVectorLoad(Op, DAG);
1940
1941 assert(NumElements == 3);
1942
1943 EVT WideVT =
1944 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(), NumElements: 4);
1945 EVT WideMemVT =
1946 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getVectorElementType(), NumElements: 4);
1947 SDValue WideLoad = DAG.getExtLoad(
1948 ExtType: Load->getExtensionType(), dl: SL, VT: WideVT, Chain: Load->getChain(), Ptr: BasePtr, PtrInfo: SrcValue,
1949 MemVT: WideMemVT, Alignment: BaseAlign, MMOFlags: Load->getMemOperand()->getFlags());
1950 return DAG.getMergeValues(
1951 Ops: {DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SL, VT, N1: WideLoad,
1952 N2: DAG.getVectorIdxConstant(Val: 0, DL: SL)),
1953 WideLoad.getValue(R: 1)},
1954 dl: SL);
1955}
1956
1957SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1958 SelectionDAG &DAG) const {
1959 StoreSDNode *Store = cast<StoreSDNode>(Val&: Op);
1960 SDValue Val = Store->getValue();
1961 EVT VT = Val.getValueType();
1962
1963 // If this is a 2 element vector, we really want to scalarize and not create
1964 // weird 1 element vectors.
1965 if (VT.getVectorNumElements() == 2)
1966 return scalarizeVectorStore(ST: Store, DAG);
1967
1968 EVT MemVT = Store->getMemoryVT();
1969 SDValue Chain = Store->getChain();
1970 SDValue BasePtr = Store->getBasePtr();
1971 SDLoc SL(Op);
1972
1973 EVT LoVT, HiVT;
1974 EVT LoMemVT, HiMemVT;
1975 SDValue Lo, Hi;
1976
1977 std::tie(args&: LoVT, args&: HiVT) = getSplitDestVTs(VT, DAG);
1978 std::tie(args&: LoMemVT, args&: HiMemVT) = getSplitDestVTs(VT: MemVT, DAG);
1979 std::tie(args&: Lo, args&: Hi) = splitVector(N: Val, DL: SL, LoVT, HiVT, DAG);
1980
1981 SDValue HiPtr = DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: LoMemVT.getStoreSize());
1982
1983 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
1984 Align BaseAlign = Store->getAlign();
1985 unsigned Size = LoMemVT.getStoreSize();
1986 Align HiAlign = commonAlignment(A: BaseAlign, Offset: Size);
1987
1988 SDValue LoStore =
1989 DAG.getTruncStore(Chain, dl: SL, Val: Lo, Ptr: BasePtr, PtrInfo: SrcValue, SVT: LoMemVT, Alignment: BaseAlign,
1990 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
1991 SDValue HiStore = DAG.getTruncStore(
1992 Chain, dl: SL, Val: Hi, Ptr: HiPtr, PtrInfo: SrcValue.getWithOffset(O: Size), SVT: HiMemVT, Alignment: HiAlign,
1993 MMOFlags: Store->getMemOperand()->getFlags(), Metadata: Store->getAAInfo());
1994
1995 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, N1: LoStore, N2: HiStore);
1996}
1997
1998// This is a shortcut for integer division because we have fast i32<->f32
1999// conversions, and fast f32 reciprocal instructions.
2000SDValue AMDGPUTargetLowering::LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG,
2001 bool Sign) const {
2002 SDLoc DL(Op);
2003 EVT VT = Op.getValueType();
2004 assert(VT == MVT::i32 && "LowerDIVREMToFloat expects an i32");
2005
2006 SDValue LHS = Op.getOperand(i: 0);
2007 SDValue RHS = Op.getOperand(i: 1);
2008 MVT IntVT = MVT::i32;
2009 MVT FltVT = MVT::f32;
2010
2011 unsigned LHSSignBits;
2012 unsigned RHSSignBits;
2013 if (Sign) {
2014 LHSSignBits = DAG.ComputeNumSignBits(Op: LHS);
2015 RHSSignBits = DAG.ComputeNumSignBits(Op: RHS);
2016 if (LHSSignBits < 9 || RHSSignBits < 9)
2017 return SDValue();
2018 } else {
2019 KnownBits LHSKnown = DAG.computeKnownBits(Op: LHS);
2020 KnownBits RHSKnown = DAG.computeKnownBits(Op: RHS);
2021
2022 LHSSignBits = LHSKnown.countMinLeadingZeros();
2023 RHSSignBits = RHSKnown.countMinLeadingZeros();
2024 }
2025
2026 unsigned BitSize = VT.getSizeInBits();
2027 unsigned SignBits = std::min(a: LHSSignBits, b: RHSSignBits);
2028 unsigned DivBits = BitSize - SignBits;
2029 if (Sign)
2030 ++DivBits;
2031
2032 // In order to avoid problems due to 1 ulp accuracy issues with v_rcp_f32,
2033 // limit LowerDIVREMToFloat to:
2034 // [-0x400000,0x3FFFFF] for Sign
2035 // [ 0x000000,0x3FFFFF] for !Sign
2036 // This matches what is done in expandDivRemToFloatImpl.
2037 if (DivBits > (Sign ? 23u : 22u))
2038 return SDValue();
2039
2040 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
2041 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
2042
2043 // int ia = (int)LHS;
2044 SDValue ia = LHS;
2045
2046 // int ib, (int)RHS;
2047 SDValue ib = RHS;
2048
2049 // The calculation:
2050 // fq = fa*recip(fb)
2051 // may be too small due to the 1ulp accuracy in the recip
2052 // operation and rounding issues. Since fq is truncated to produce
2053 // an integer value it may be too small by one. This is
2054 // dealt with by incrementing fa by 1ulp:
2055 // fq = (fa+1ulp)*recip(fb)
2056 // This will increase fa's magnitude by at most 0.5
2057 // (i.e. when fabs(fa)==0x400000 the LSB of the mantissa represents 0.5).
2058 // Thus, this method is safe since fa must be incremented by at least 1.0
2059 // for the quotient to increase by one.
2060 SDValue fa = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ia);
2061 SDValue faAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: fa);
2062 SDValue faIncremented = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: faAsInt,
2063 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
2064 fa = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: FltVT, Operand: faIncremented);
2065
2066 // float fb = (float)ib;
2067 SDValue fb = DAG.getNode(Opcode: ToFp, DL, VT: FltVT, Operand: ib);
2068
2069 SDValue fq = DAG.getNode(Opcode: ISD::FMUL, DL, VT: FltVT,
2070 N1: fa, N2: DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: FltVT, Operand: fb));
2071
2072 // fq = trunc(fq);
2073 fq = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: FltVT, Operand: fq);
2074
2075 // int iq = (int)fq;
2076 SDValue Div = DAG.getNode(Opcode: ToInt, DL, VT: IntVT, Operand: fq);
2077
2078 // Rem needs compensation, it's easier to recompute it
2079 SDValue Rem = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Div, N2: RHS);
2080 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Rem);
2081
2082 return DAG.getMergeValues(Ops: { Div, Rem }, dl: DL);
2083}
2084
2085void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
2086 SelectionDAG &DAG,
2087 SmallVectorImpl<SDValue> &Results) const {
2088 SDLoc DL(Op);
2089 EVT VT = Op.getValueType();
2090
2091 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
2092
2093 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2094
2095 SDValue One = DAG.getConstant(Val: 1, DL, VT: HalfVT);
2096 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HalfVT);
2097
2098 //HiLo split
2099 SDValue LHS_Lo, LHS_Hi;
2100 SDValue LHS = Op.getOperand(i: 0);
2101 std::tie(args&: LHS_Lo, args&: LHS_Hi) = DAG.SplitScalar(N: LHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2102
2103 SDValue RHS_Lo, RHS_Hi;
2104 SDValue RHS = Op.getOperand(i: 1);
2105 std::tie(args&: RHS_Lo, args&: RHS_Hi) = DAG.SplitScalar(N: RHS, DL, LoVT: HalfVT, HiVT: HalfVT);
2106
2107 if (DAG.MaskedValueIsZero(Op: RHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32)) &&
2108 DAG.MaskedValueIsZero(Op: LHS, Mask: APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32))) {
2109
2110 SDValue Res = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2111 N1: LHS_Lo, N2: RHS_Lo);
2112
2113 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 0), Zero});
2114 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Res.getValue(R: 1), Zero});
2115
2116 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV));
2117 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM));
2118 return;
2119 }
2120
2121 if (isTypeLegal(VT: MVT::i64)) {
2122 // The algorithm here is based on ideas from "Software Integer Division",
2123 // Tom Rodeheffer, August 2008.
2124
2125 MachineFunction &MF = DAG.getMachineFunction();
2126 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2127
2128 // Compute denominator reciprocal.
2129 unsigned FMAD =
2130 !Subtarget->hasMadMacF32Insts() ? (unsigned)ISD::FMA
2131 : MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign()
2132 ? (unsigned)ISD::FMAD
2133 : (unsigned)AMDGPUISD::FMAD_FTZ;
2134
2135 SDValue Cvt_Lo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Lo);
2136 SDValue Cvt_Hi = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: MVT::f32, Operand: RHS_Hi);
2137 SDValue Mad1 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Cvt_Hi,
2138 N2: DAG.getConstantFP(Val: APInt(32, 0x4f800000).bitsToFloat(), DL, VT: MVT::f32),
2139 N3: Cvt_Lo);
2140 SDValue Rcp = DAG.getNode(Opcode: AMDGPUISD::RCP, DL, VT: MVT::f32, Operand: Mad1);
2141 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Rcp,
2142 N2: DAG.getConstantFP(Val: APInt(32, 0x5f7ffffc).bitsToFloat(), DL, VT: MVT::f32));
2143 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f32, N1: Mul1,
2144 N2: DAG.getConstantFP(Val: APInt(32, 0x2f800000).bitsToFloat(), DL, VT: MVT::f32));
2145 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT: MVT::f32, Operand: Mul2);
2146 SDValue Mad2 = DAG.getNode(Opcode: FMAD, DL, VT: MVT::f32, N1: Trunc,
2147 N2: DAG.getConstantFP(Val: APInt(32, 0xcf800000).bitsToFloat(), DL, VT: MVT::f32),
2148 N3: Mul1);
2149 SDValue Rcp_Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Mad2);
2150 SDValue Rcp_Hi = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL, VT: HalfVT, Operand: Trunc);
2151 SDValue Rcp64 = DAG.getBitcast(VT,
2152 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Rcp_Lo, Rcp_Hi}));
2153
2154 SDValue Zero64 = DAG.getConstant(Val: 0, DL, VT);
2155 SDValue One64 = DAG.getConstant(Val: 1, DL, VT);
2156 SDValue Zero1 = DAG.getConstant(Val: 0, DL, VT: MVT::i1);
2157 SDVTList HalfCarryVT = DAG.getVTList(VT1: HalfVT, VT2: MVT::i1);
2158
2159 // First round of UNR (Unsigned integer Newton-Raphson).
2160 SDValue Neg_RHS = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero64, N2: RHS);
2161 SDValue Mullo1 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Rcp64);
2162 SDValue Mulhi1 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Rcp64, N2: Mullo1);
2163 SDValue Mulhi1_Lo, Mulhi1_Hi;
2164 std::tie(args&: Mulhi1_Lo, args&: Mulhi1_Hi) =
2165 DAG.SplitScalar(N: Mulhi1, DL, LoVT: HalfVT, HiVT: HalfVT);
2166 SDValue Add1_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Lo,
2167 N2: Mulhi1_Lo, N3: Zero1);
2168 SDValue Add1_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Rcp_Hi,
2169 N2: Mulhi1_Hi, N3: Add1_Lo.getValue(R: 1));
2170 SDValue Add1 = DAG.getBitcast(VT,
2171 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add1_Lo, Add1_Hi}));
2172
2173 // Second round of UNR.
2174 SDValue Mullo2 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Neg_RHS, N2: Add1);
2175 SDValue Mulhi2 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Add1, N2: Mullo2);
2176 SDValue Mulhi2_Lo, Mulhi2_Hi;
2177 std::tie(args&: Mulhi2_Lo, args&: Mulhi2_Hi) =
2178 DAG.SplitScalar(N: Mulhi2, DL, LoVT: HalfVT, HiVT: HalfVT);
2179 SDValue Add2_Lo = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Lo,
2180 N2: Mulhi2_Lo, N3: Zero1);
2181 SDValue Add2_Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: HalfCarryVT, N1: Add1_Hi,
2182 N2: Mulhi2_Hi, N3: Add2_Lo.getValue(R: 1));
2183 SDValue Add2 = DAG.getBitcast(VT,
2184 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Add2_Lo, Add2_Hi}));
2185
2186 SDValue Mulhi3 = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: LHS, N2: Add2);
2187
2188 SDValue Mul3 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: RHS, N2: Mulhi3);
2189
2190 SDValue Mul3_Lo, Mul3_Hi;
2191 std::tie(args&: Mul3_Lo, args&: Mul3_Hi) = DAG.SplitScalar(N: Mul3, DL, LoVT: HalfVT, HiVT: HalfVT);
2192 SDValue Sub1_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Lo,
2193 N2: Mul3_Lo, N3: Zero1);
2194 SDValue Sub1_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: LHS_Hi,
2195 N2: Mul3_Hi, N3: Sub1_Lo.getValue(R: 1));
2196 SDValue Sub1_Mi = DAG.getNode(Opcode: ISD::SUB, DL, VT: HalfVT, N1: LHS_Hi, N2: Mul3_Hi);
2197 SDValue Sub1 = DAG.getBitcast(VT,
2198 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub1_Lo, Sub1_Hi}));
2199
2200 SDValue MinusOne = DAG.getConstant(Val: 0xffffffffu, DL, VT: HalfVT);
2201 SDValue C1 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2202 Cond: ISD::SETUGE);
2203 SDValue C2 = DAG.getSelectCC(DL, LHS: Sub1_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2204 Cond: ISD::SETUGE);
2205 SDValue C3 = DAG.getSelectCC(DL, LHS: Sub1_Hi, RHS: RHS_Hi, True: C2, False: C1, Cond: ISD::SETEQ);
2206
2207 // TODO: Here and below portions of the code can be enclosed into if/endif.
2208 // Currently control flow is unconditional and we have 4 selects after
2209 // potential endif to substitute PHIs.
2210
2211 // if C3 != 0 ...
2212 SDValue Sub2_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Lo,
2213 N2: RHS_Lo, N3: Zero1);
2214 SDValue Sub2_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub1_Mi,
2215 N2: RHS_Hi, N3: Sub1_Lo.getValue(R: 1));
2216 SDValue Sub2_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2217 N2: Zero, N3: Sub2_Lo.getValue(R: 1));
2218 SDValue Sub2 = DAG.getBitcast(VT,
2219 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub2_Lo, Sub2_Hi}));
2220
2221 SDValue Add3 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Mulhi3, N2: One64);
2222
2223 SDValue C4 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: MinusOne, False: Zero,
2224 Cond: ISD::SETUGE);
2225 SDValue C5 = DAG.getSelectCC(DL, LHS: Sub2_Lo, RHS: RHS_Lo, True: MinusOne, False: Zero,
2226 Cond: ISD::SETUGE);
2227 SDValue C6 = DAG.getSelectCC(DL, LHS: Sub2_Hi, RHS: RHS_Hi, True: C5, False: C4, Cond: ISD::SETEQ);
2228
2229 // if (C6 != 0)
2230 SDValue Add4 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Add3, N2: One64);
2231
2232 SDValue Sub3_Lo = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Lo,
2233 N2: RHS_Lo, N3: Zero1);
2234 SDValue Sub3_Mi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub2_Mi,
2235 N2: RHS_Hi, N3: Sub2_Lo.getValue(R: 1));
2236 SDValue Sub3_Hi = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: HalfCarryVT, N1: Sub3_Mi,
2237 N2: Zero, N3: Sub3_Lo.getValue(R: 1));
2238 SDValue Sub3 = DAG.getBitcast(VT,
2239 V: DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {Sub3_Lo, Sub3_Hi}));
2240
2241 // endif C6
2242 // endif C3
2243
2244 SDValue Sel1 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Add4, False: Add3, Cond: ISD::SETNE);
2245 SDValue Div = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel1, False: Mulhi3, Cond: ISD::SETNE);
2246
2247 SDValue Sel2 = DAG.getSelectCC(DL, LHS: C6, RHS: Zero, True: Sub3, False: Sub2, Cond: ISD::SETNE);
2248 SDValue Rem = DAG.getSelectCC(DL, LHS: C3, RHS: Zero, True: Sel2, False: Sub1, Cond: ISD::SETNE);
2249
2250 Results.push_back(Elt: Div);
2251 Results.push_back(Elt: Rem);
2252
2253 return;
2254 }
2255
2256 // r600 expandion.
2257 // Get Speculative values
2258 SDValue DIV_Part = DAG.getNode(Opcode: ISD::UDIV, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2259 SDValue REM_Part = DAG.getNode(Opcode: ISD::UREM, DL, VT: HalfVT, N1: LHS_Hi, N2: RHS_Lo);
2260
2261 SDValue REM_Lo = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: REM_Part, False: LHS_Hi, Cond: ISD::SETEQ);
2262 SDValue REM = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {REM_Lo, Zero});
2263 REM = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: REM);
2264
2265 SDValue DIV_Hi = DAG.getSelectCC(DL, LHS: RHS_Hi, RHS: Zero, True: DIV_Part, False: Zero, Cond: ISD::SETEQ);
2266 SDValue DIV_Lo = Zero;
2267
2268 const unsigned halfBitWidth = HalfVT.getSizeInBits();
2269
2270 for (unsigned i = 0; i < halfBitWidth; ++i) {
2271 const unsigned bitPos = halfBitWidth - i - 1;
2272 SDValue POS = DAG.getConstant(Val: bitPos, DL, VT: HalfVT);
2273 // Get value of high bit
2274 SDValue HBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: HalfVT, N1: LHS_Lo, N2: POS);
2275 HBit = DAG.getNode(Opcode: ISD::AND, DL, VT: HalfVT, N1: HBit, N2: One);
2276 HBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: HBit);
2277
2278 // Shift
2279 REM = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: REM, N2: DAG.getConstant(Val: 1, DL, VT));
2280 // Add LHS high bit
2281 REM = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: REM, N2: HBit);
2282
2283 SDValue BIT = DAG.getConstant(Val: 1ULL << bitPos, DL, VT: HalfVT);
2284 SDValue realBIT = DAG.getSelectCC(DL, LHS: REM, RHS, True: BIT, False: Zero, Cond: ISD::SETUGE);
2285
2286 DIV_Lo = DAG.getNode(Opcode: ISD::OR, DL, VT: HalfVT, N1: DIV_Lo, N2: realBIT);
2287
2288 // Update REM
2289 SDValue REM_sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: REM, N2: RHS);
2290 REM = DAG.getSelectCC(DL, LHS: REM, RHS, True: REM_sub, False: REM, Cond: ISD::SETUGE);
2291 }
2292
2293 SDValue DIV = DAG.getBuildVector(VT: MVT::v2i32, DL, Ops: {DIV_Lo, DIV_Hi});
2294 DIV = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: DIV);
2295 Results.push_back(Elt: DIV);
2296 Results.push_back(Elt: REM);
2297}
2298
2299SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
2300 SelectionDAG &DAG) const {
2301 SDLoc DL(Op);
2302 EVT VT = Op.getValueType();
2303
2304 if (VT == MVT::i64) {
2305 SmallVector<SDValue, 2> Results;
2306 LowerUDIVREM64(Op, DAG, Results);
2307 return DAG.getMergeValues(Ops: Results, dl: DL);
2308 }
2309
2310 if (VT == MVT::i32) {
2311 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: false))
2312 return Res;
2313 }
2314
2315 SDValue X = Op.getOperand(i: 0);
2316 SDValue Y = Op.getOperand(i: 1);
2317
2318 // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
2319 // algorithm used here.
2320
2321 // Initial estimate of inv(y).
2322 SDValue Z = DAG.getNode(Opcode: AMDGPUISD::URECIP, DL, VT, Operand: Y);
2323
2324 // One round of UNR.
2325 SDValue NegY = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Y);
2326 SDValue NegYZ = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: NegY, N2: Z);
2327 Z = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Z,
2328 N2: DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: Z, N2: NegYZ));
2329
2330 // Quotient/remainder estimate.
2331 SDValue Q = DAG.getNode(Opcode: ISD::MULHU, DL, VT, N1: X, N2: Z);
2332 SDValue R =
2333 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: X, N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Q, N2: Y));
2334
2335 // First quotient/remainder refinement.
2336 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2337 SDValue One = DAG.getConstant(Val: 1, DL, VT);
2338 SDValue Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2339 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2340 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2341 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2342 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2343
2344 // Second quotient/remainder refinement.
2345 Cond = DAG.getSetCC(DL, VT: CCVT, LHS: R, RHS: Y, Cond: ISD::SETUGE);
2346 Q = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2347 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Q, N2: One), N3: Q);
2348 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond,
2349 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: R, N2: Y), N3: R);
2350
2351 return DAG.getMergeValues(Ops: {Q, R}, dl: DL);
2352}
2353
2354SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
2355 SelectionDAG &DAG) const {
2356 SDLoc DL(Op);
2357 EVT VT = Op.getValueType();
2358
2359 SDValue LHS = Op.getOperand(i: 0);
2360 SDValue RHS = Op.getOperand(i: 1);
2361
2362 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
2363 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
2364
2365 if (VT == MVT::i32) {
2366 if (SDValue Res = LowerDIVREMToFloat(Op, DAG, Sign: true))
2367 return Res;
2368 }
2369
2370 // LHS must have > 33 sign-bits to ensure that LHS != -2147483648
2371 // Otherwise 32-bit division cannot be used safely.
2372 // -2147483648/1 and -2147483648/-1 are not equal,
2373 // but they produce the same lower 32-bit result.
2374 if (VT == MVT::i64 && DAG.ComputeNumSignBits(Op: LHS) > 33 &&
2375 DAG.ComputeNumSignBits(Op: RHS) > 32) {
2376 EVT HalfVT = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
2377
2378 //HiLo split
2379 SDValue LHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: LHS, N2: Zero);
2380 SDValue RHS_Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HalfVT, N1: RHS, N2: Zero);
2381 SDValue DIVREM = DAG.getNode(Opcode: ISD::SDIVREM, DL, VTList: DAG.getVTList(VT1: HalfVT, VT2: HalfVT),
2382 N1: LHS_Lo, N2: RHS_Lo);
2383 SDValue Res[2] = {
2384 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 0)),
2385 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: DIVREM.getValue(R: 1))
2386 };
2387 return DAG.getMergeValues(Ops: Res, dl: DL);
2388 }
2389
2390 SDValue LHSign = DAG.getSelectCC(DL, LHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2391 SDValue RHSign = DAG.getSelectCC(DL, LHS: RHS, RHS: Zero, True: NegOne, False: Zero, Cond: ISD::SETLT);
2392 SDValue DSign = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHSign, N2: RHSign);
2393 SDValue RSign = LHSign; // Remainder sign is the same as LHS
2394
2395 LHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: LHSign);
2396 RHS = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: RHS, N2: RHSign);
2397
2398 LHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LHS, N2: LHSign);
2399 RHS = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: RHS, N2: RHSign);
2400
2401 SDValue Div = DAG.getNode(Opcode: ISD::UDIVREM, DL, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
2402 SDValue Rem = Div.getValue(R: 1);
2403
2404 Div = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Div, N2: DSign);
2405 Rem = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Rem, N2: RSign);
2406
2407 Div = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Div, N2: DSign);
2408 Rem = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Rem, N2: RSign);
2409
2410 SDValue Res[2] = {
2411 Div,
2412 Rem
2413 };
2414 return DAG.getMergeValues(Ops: Res, dl: DL);
2415}
2416
2417SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2418 SDLoc SL(Op);
2419 SDValue Src = Op.getOperand(i: 0);
2420
2421 // result = trunc(src)
2422 // if (src > 0.0 && src != result)
2423 // result += 1.0
2424
2425 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2426
2427 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2428 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT: MVT::f64);
2429
2430 EVT SetCCVT =
2431 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2432
2433 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOGT);
2434 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2435 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2436
2437 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: One, N3: Zero);
2438 // TODO: Should this propagate fast-math-flags?
2439 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2440}
2441
2442static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2443 SelectionDAG &DAG) {
2444 const unsigned FractBits = 52;
2445 const unsigned ExpBits = 11;
2446
2447 SDValue ExpPart = DAG.getNode(Opcode: AMDGPUISD::BFE_U32, DL: SL, VT: MVT::i32,
2448 N1: Hi,
2449 N2: DAG.getConstant(Val: FractBits - 32, DL: SL, VT: MVT::i32),
2450 N3: DAG.getConstant(Val: ExpBits, DL: SL, VT: MVT::i32));
2451 SDValue Exp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ExpPart,
2452 N2: DAG.getConstant(Val: 1023, DL: SL, VT: MVT::i32));
2453
2454 return Exp;
2455}
2456
2457SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2458 SDLoc SL(Op);
2459 SDValue Src = Op.getOperand(i: 0);
2460
2461 assert(Op.getValueType() == MVT::f64);
2462
2463 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: MVT::i32);
2464
2465 // Extract the upper half, since this is where we will find the sign and
2466 // exponent.
2467 SDValue Hi = getHiHalf64(Op: Src, DAG);
2468
2469 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2470
2471 const unsigned FractBits = 52;
2472
2473 // Extract the sign bit.
2474 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, DL: SL, VT: MVT::i32);
2475 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: Hi, N2: SignBitMask);
2476
2477 // Extend back to 64-bits.
2478 SDValue SignBit64 = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Zero, SignBit});
2479 SignBit64 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: SignBit64);
2480
2481 SDValue BcInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Src);
2482 const SDValue FractMask
2483 = DAG.getConstant(Val: (UINT64_C(1) << FractBits) - 1, DL: SL, VT: MVT::i64);
2484
2485 SDValue Shr = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: FractMask, N2: Exp);
2486 SDValue Not = DAG.getNOT(DL: SL, Val: Shr, VT: MVT::i64);
2487 SDValue Tmp0 = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i64, N1: BcInt, N2: Not);
2488
2489 EVT SetCCVT =
2490 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i32);
2491
2492 const SDValue FiftyOne = DAG.getConstant(Val: FractBits - 1, DL: SL, VT: MVT::i32);
2493
2494 SDValue ExpLt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: Zero, Cond: ISD::SETLT);
2495 SDValue ExpGt51 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Exp, RHS: FiftyOne, Cond: ISD::SETGT);
2496
2497 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpLt0, N2: SignBit64, N3: Tmp0);
2498 SDValue Tmp2 = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::i64, N1: ExpGt51, N2: BcInt, N3: Tmp1);
2499
2500 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f64, Operand: Tmp2);
2501}
2502
2503SDValue AMDGPUTargetLowering::LowerFROUNDEVEN(SDValue Op,
2504 SelectionDAG &DAG) const {
2505 SDLoc SL(Op);
2506 SDValue Src = Op.getOperand(i: 0);
2507
2508 assert(Op.getValueType() == MVT::f64);
2509
2510 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2511 SDValue C1 = DAG.getConstantFP(Val: C1Val, DL: SL, VT: MVT::f64);
2512 SDValue CopySign = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT: MVT::f64, N1: C1, N2: Src);
2513
2514 // TODO: Should this propagate fast-math-flags?
2515
2516 SDValue Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Src, N2: CopySign);
2517 SDValue Tmp2 = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT: MVT::f64, N1: Tmp1, N2: CopySign);
2518
2519 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: MVT::f64, Operand: Src);
2520
2521 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2522 SDValue C2 = DAG.getConstantFP(Val: C2Val, DL: SL, VT: MVT::f64);
2523
2524 EVT SetCCVT =
2525 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2526 SDValue Cond = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Fabs, RHS: C2, Cond: ISD::SETOGT);
2527
2528 return DAG.getSelect(DL: SL, VT: MVT::f64, Cond, LHS: Src, RHS: Tmp2);
2529}
2530
2531SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op,
2532 SelectionDAG &DAG) const {
2533 // FNEARBYINT and FRINT are the same, except in their handling of FP
2534 // exceptions. Those aren't really meaningful for us, and OpenCL only has
2535 // rint, so just treat them as equivalent.
2536 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT: Op.getValueType(),
2537 Operand: Op.getOperand(i: 0));
2538}
2539
2540SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2541 auto VT = Op.getValueType();
2542 auto Arg = Op.getOperand(i: 0u);
2543 return DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SDLoc(Op), VT, Operand: Arg);
2544}
2545
2546// XXX - May require not supporting f32 denormals?
2547
2548// Don't handle v2f16. The extra instructions to scalarize and repack around the
2549// compare and vselect end up producing worse code than scalarizing the whole
2550// operation.
2551SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2552 SDLoc SL(Op);
2553 SDValue X = Op.getOperand(i: 0);
2554 EVT VT = Op.getValueType();
2555
2556 SDValue T = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: X);
2557
2558 // TODO: Should this propagate fast-math-flags?
2559
2560 SDValue Diff = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: T);
2561
2562 SDValue AbsDiff = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Diff);
2563
2564 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2565 const SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2566
2567 EVT SetCCVT =
2568 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2569
2570 const SDValue Half = DAG.getConstantFP(Val: 0.5, DL: SL, VT);
2571 SDValue Cmp = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: AbsDiff, RHS: Half, Cond: ISD::SETOGE);
2572 SDValue OneOrZeroFP = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cmp, N2: One, N3: Zero);
2573
2574 SDValue SignedOffset = DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: OneOrZeroFP, N2: X);
2575 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: T, N2: SignedOffset);
2576}
2577
2578SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2579 SDLoc SL(Op);
2580 SDValue Src = Op.getOperand(i: 0);
2581
2582 // result = trunc(src);
2583 // if (src < 0.0 && src != result)
2584 // result += -1.0.
2585
2586 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: MVT::f64, Operand: Src);
2587
2588 const SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT: MVT::f64);
2589 const SDValue NegOne = DAG.getConstantFP(Val: -1.0, DL: SL, VT: MVT::f64);
2590
2591 EVT SetCCVT =
2592 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::f64);
2593
2594 SDValue Lt0 = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Zero, Cond: ISD::SETOLT);
2595 SDValue NeTrunc = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: Trunc, Cond: ISD::SETONE);
2596 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: Lt0, N2: NeTrunc);
2597
2598 SDValue Add = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f64, N1: And, N2: NegOne, N3: Zero);
2599 // TODO: Should this propagate fast-math-flags?
2600 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: Trunc, N2: Add);
2601}
2602
2603/// Return true if it's known that \p Src can never be an f32 denormal value.
2604static bool valueIsKnownNeverF32Denorm(SDValue Src) {
2605 switch (Src.getOpcode()) {
2606 case ISD::FP_EXTEND:
2607 return Src.getOperand(i: 0).getValueType() == MVT::f16;
2608 case ISD::FP16_TO_FP:
2609 case ISD::FFREXP:
2610 case ISD::FSQRT:
2611 case AMDGPUISD::LOG:
2612 case AMDGPUISD::EXP:
2613 return true;
2614 case ISD::INTRINSIC_WO_CHAIN: {
2615 unsigned IntrinsicID = Src.getConstantOperandVal(i: 0);
2616 switch (IntrinsicID) {
2617 case Intrinsic::amdgcn_frexp_mant:
2618 case Intrinsic::amdgcn_log:
2619 case Intrinsic::amdgcn_log_clamp:
2620 case Intrinsic::amdgcn_exp2:
2621 case Intrinsic::amdgcn_sqrt:
2622 return true;
2623 default:
2624 return false;
2625 }
2626 }
2627 default:
2628 return false;
2629 }
2630
2631 llvm_unreachable("covered opcode switch");
2632}
2633
2634bool AMDGPUTargetLowering::allowApproxFunc(const SelectionDAG &DAG,
2635 SDNodeFlags Flags) {
2636 return Flags.hasApproximateFuncs();
2637}
2638
2639bool AMDGPUTargetLowering::needsDenormHandlingF32(const SelectionDAG &DAG,
2640 SDValue Src,
2641 SDNodeFlags Flags) {
2642 return !valueIsKnownNeverF32Denorm(Src) &&
2643 DAG.getMachineFunction()
2644 .getDenormalMode(FPType: APFloat::IEEEsingle())
2645 .Input != DenormalMode::PreserveSign;
2646}
2647
2648SDValue AMDGPUTargetLowering::getIsFinite(SelectionDAG &DAG, SDValue Src,
2649 SDNodeFlags Flags) const {
2650 SDLoc SL(Src);
2651 EVT VT = Src.getValueType();
2652 const fltSemantics &Semantics = VT.getFltSemantics();
2653 SDValue Inf = DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL: SL, VT);
2654
2655 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: Src, Flags);
2656 SDValue IsFinite = DAG.getSetCC(
2657 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Fabs,
2658 RHS: Inf, Cond: ISD::SETOLT);
2659 return IsFinite;
2660}
2661
2662/// If denormal handling is required return the scaled input to FLOG2, and the
2663/// check for denormal range. Otherwise, return null values.
2664std::pair<SDValue, SDValue>
2665AMDGPUTargetLowering::getScaledLogInput(SelectionDAG &DAG, const SDLoc SL,
2666 SDValue Src, SDNodeFlags Flags) const {
2667 if (!needsDenormHandlingF32(DAG, Src, Flags))
2668 return {};
2669
2670 MVT VT = MVT::f32;
2671 const fltSemantics &Semantics = APFloat::IEEEsingle();
2672 SDValue SmallestNormal =
2673 DAG.getConstantFP(Val: APFloat::getSmallestNormalized(Sem: Semantics), DL: SL, VT);
2674
2675 SDValue IsLtSmallestNormal = DAG.getSetCC(
2676 DL: SL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT), LHS: Src,
2677 RHS: SmallestNormal, Cond: ISD::SETOLT);
2678
2679 SDValue Scale32 = DAG.getConstantFP(Val: 0x1.0p+32, DL: SL, VT);
2680 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
2681 SDValue ScaleFactor =
2682 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: Scale32, N3: One, Flags);
2683
2684 SDValue ScaledInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Src, N2: ScaleFactor, Flags);
2685 return {ScaledInput, IsLtSmallestNormal};
2686}
2687
2688SDValue AMDGPUTargetLowering::LowerFLOG2(SDValue Op, SelectionDAG &DAG) const {
2689 // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
2690 // If we have to handle denormals, scale up the input and adjust the result.
2691
2692 // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
2693 // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
2694
2695 SDLoc SL(Op);
2696 EVT VT = Op.getValueType();
2697 SDValue Src = Op.getOperand(i: 0);
2698 SDNodeFlags Flags = Op->getFlags();
2699
2700 if (VT == MVT::f16) {
2701 // Nothing in half is a denormal when promoted to f32.
2702 assert(!isTypeLegal(VT));
2703 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
2704 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
2705 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
2706 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
2707 }
2708
2709 auto [ScaledInput, IsLtSmallestNormal] =
2710 getScaledLogInput(DAG, SL, Src, Flags);
2711 if (!ScaledInput)
2712 return DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: Src, Flags);
2713
2714 SDValue Log2 = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2715
2716 SDValue ThirtyTwo = DAG.getConstantFP(Val: 32.0, DL: SL, VT);
2717 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
2718 SDValue ResultOffset =
2719 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsLtSmallestNormal, N2: ThirtyTwo, N3: Zero);
2720 return DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: Log2, N2: ResultOffset, Flags);
2721}
2722
2723static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X,
2724 SDValue Y, SDValue C, SDNodeFlags Flags = SDNodeFlags()) {
2725 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Y, Flags);
2726 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: C, Flags);
2727}
2728
2729SDValue AMDGPUTargetLowering::LowerFLOGCommon(SDValue Op,
2730 SelectionDAG &DAG) const {
2731 SDValue X = Op.getOperand(i: 0);
2732 EVT VT = Op.getValueType();
2733 SDNodeFlags Flags = Op->getFlags();
2734 SDLoc DL(Op);
2735 const bool IsLog10 = Op.getOpcode() == ISD::FLOG10;
2736 assert(IsLog10 || Op.getOpcode() == ISD::FLOG);
2737
2738 if (VT == MVT::f16 || Flags.hasApproximateFuncs()) {
2739 // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
2740 // depending on !fpmath metadata.
2741
2742 bool PromoteToF32 = VT == MVT::f16 && (!Flags.hasApproximateFuncs() ||
2743 !isTypeLegal(VT: MVT::f16));
2744
2745 if (PromoteToF32) {
2746 // Log and multiply in f32 is always good enough for f16.
2747 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2748 }
2749
2750 SDValue Lowered = LowerFLOGUnsafe(Op: X, SL: DL, DAG, IsLog10, Flags);
2751 if (PromoteToF32) {
2752 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Lowered,
2753 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32), Flags);
2754 }
2755
2756 return Lowered;
2757 }
2758
2759 SDValue ScaledInput, IsScaled;
2760 if (VT == MVT::f16)
2761 X = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: X, Flags);
2762 else {
2763 std::tie(args&: ScaledInput, args&: IsScaled) = getScaledLogInput(DAG, SL: DL, Src: X, Flags);
2764 if (ScaledInput)
2765 X = ScaledInput;
2766 }
2767
2768 SDValue Y = DAG.getNode(Opcode: AMDGPUISD::LOG, DL, VT, Operand: X, Flags);
2769
2770 SDValue R;
2771 if (Subtarget->hasFastFMAF32()) {
2772 // c+cc are ln(2)/ln(10) to more than 49 bits
2773 const float c_log10 = 0x1.344134p-2f;
2774 const float cc_log10 = 0x1.09f79ep-26f;
2775
2776 // c + cc is ln(2) to more than 49 bits
2777 const float c_log = 0x1.62e42ep-1f;
2778 const float cc_log = 0x1.efa39ep-25f;
2779
2780 SDValue C = DAG.getConstantFP(Val: IsLog10 ? c_log10 : c_log, DL, VT);
2781 SDValue CC = DAG.getConstantFP(Val: IsLog10 ? cc_log10 : cc_log, DL, VT);
2782 // This adds correction terms for which contraction may lead to an increase
2783 // in the error of the approximation, so disable it.
2784 Flags.setAllowContract(false);
2785 R = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Y, N2: C, Flags);
2786 SDValue NegR = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: R, Flags);
2787 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: C, N3: NegR, Flags);
2788 SDValue FMA1 = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Y, N2: CC, N3: FMA0, Flags);
2789 R = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: R, N2: FMA1, Flags);
2790 } else {
2791 // ch+ct is ln(2)/ln(10) to more than 36 bits
2792 const float ch_log10 = 0x1.344000p-2f;
2793 const float ct_log10 = 0x1.3509f6p-18f;
2794
2795 // ch + ct is ln(2) to more than 36 bits
2796 const float ch_log = 0x1.62e000p-1f;
2797 const float ct_log = 0x1.0bfbe8p-15f;
2798
2799 SDValue CH = DAG.getConstantFP(Val: IsLog10 ? ch_log10 : ch_log, DL, VT);
2800 SDValue CT = DAG.getConstantFP(Val: IsLog10 ? ct_log10 : ct_log, DL, VT);
2801
2802 SDValue YAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Y);
2803 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL, VT: MVT::i32);
2804 SDValue YHInt = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: YAsInt, N2: MaskConst);
2805 SDValue YH = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f32, Operand: YHInt);
2806 SDValue YT = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: YH, Flags);
2807 // This adds correction terms for which contraction may lead to an increase
2808 // in the error of the approximation, so disable it.
2809 Flags.setAllowContract(false);
2810 SDValue YTCT = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: YT, N2: CT, Flags);
2811 SDValue Mad0 = getMad(DAG, SL: DL, VT, X: YH, Y: CT, C: YTCT, Flags);
2812 SDValue Mad1 = getMad(DAG, SL: DL, VT, X: YT, Y: CH, C: Mad0, Flags);
2813 R = getMad(DAG, SL: DL, VT, X: YH, Y: CH, C: Mad1);
2814 }
2815
2816 const bool IsFiniteOnly = Flags.hasNoNaNs() && Flags.hasNoInfs();
2817
2818 // TODO: Check if known finite from source value.
2819 if (!IsFiniteOnly) {
2820 SDValue IsFinite = getIsFinite(DAG, Src: Y, Flags);
2821 R = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsFinite, N2: R, N3: Y, Flags);
2822 }
2823
2824 if (IsScaled) {
2825 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL, VT);
2826 SDValue ShiftK =
2827 DAG.getConstantFP(Val: IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f, DL, VT);
2828 SDValue Shift =
2829 DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: IsScaled, N2: ShiftK, N3: Zero, Flags);
2830 R = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: R, N2: Shift, Flags);
2831 }
2832
2833 return R;
2834}
2835
2836// Do f32 fast math expansion for flog2 or flog10. This is accurate enough for a
2837// promote f16 operation.
2838SDValue AMDGPUTargetLowering::LowerFLOGUnsafe(SDValue Src, const SDLoc &SL,
2839 SelectionDAG &DAG, bool IsLog10,
2840 SDNodeFlags Flags) const {
2841 EVT VT = Src.getValueType();
2842 unsigned LogOp =
2843 VT == MVT::f32 ? (unsigned)AMDGPUISD::LOG : (unsigned)ISD::FLOG2;
2844
2845 double Log2BaseInverted =
2846 IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
2847
2848 if (VT == MVT::f32) {
2849 auto [ScaledInput, IsScaled] = getScaledLogInput(DAG, SL, Src, Flags);
2850 if (ScaledInput) {
2851 SDValue LogSrc = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: ScaledInput, Flags);
2852 SDValue ScaledResultOffset =
2853 DAG.getConstantFP(Val: -32.0 * Log2BaseInverted, DL: SL, VT);
2854
2855 SDValue Zero = DAG.getConstantFP(Val: 0.0f, DL: SL, VT);
2856
2857 SDValue ResultOffset = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: IsScaled,
2858 N2: ScaledResultOffset, N3: Zero, Flags);
2859
2860 SDValue Log2Inv = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2861
2862 if (Subtarget->hasFastFMAF32())
2863 return DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: LogSrc, N2: Log2Inv, N3: ResultOffset,
2864 Flags);
2865 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: LogSrc, N2: Log2Inv, Flags);
2866 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Mul, N2: ResultOffset);
2867 }
2868 }
2869
2870 SDValue Log2Operand = DAG.getNode(Opcode: LogOp, DL: SL, VT, Operand: Src, Flags);
2871 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Val: Log2BaseInverted, DL: SL, VT);
2872
2873 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Log2Operand, N2: Log2BaseInvertedOperand,
2874 Flags);
2875}
2876
2877// This expansion gives a result slightly better than 1ulp.
2878SDValue AMDGPUTargetLowering::lowerFEXPF64(SDValue Op,
2879 SelectionDAG &DAG) const {
2880 SDLoc DL(Op);
2881 SDValue X = Op.getOperand(i: 0);
2882
2883 // TODO: Check if reassoc is safe. There is an output change in exp2 and
2884 // exp10, which slightly increases ulp.
2885 SDNodeFlags Flags = Op->getFlags() & ~SDNodeFlags::AllowReassociation;
2886
2887 SDValue DN, F, T;
2888
2889 if (Op.getOpcode() == ISD::FEXP2) {
2890 // dn = rint(x)
2891 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: X, Flags);
2892 // f = x - dn
2893 F = DAG.getNode(Opcode: ISD::FSUB, DL, VT: MVT::f64, N1: X, N2: DN, Flags);
2894 // t = f*C1 + f*C2
2895 SDValue C1 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2896 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2897 SDValue Mul2 = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C2, Flags);
2898 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C1, N3: Mul2, Flags);
2899 } else if (Op.getOpcode() == ISD::FEXP10) {
2900 // dn = rint(x * C1)
2901 SDValue C1 = DAG.getConstantFP(Val: 0x1.a934f0979a371p+1, DL, VT: MVT::f64);
2902 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2903 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2904
2905 // f = FMA(-dn, C2, FMA(-dn, C3, x))
2906 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2907 SDValue C2 = DAG.getConstantFP(Val: -0x1.9dc1da994fd21p-59, DL, VT: MVT::f64);
2908 SDValue C3 = DAG.getConstantFP(Val: 0x1.34413509f79ffp-2, DL, VT: MVT::f64);
2909 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2910 F = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2911
2912 // t = FMA(f, C4, f*C5)
2913 SDValue C4 = DAG.getConstantFP(Val: 0x1.26bb1bbb55516p+1, DL, VT: MVT::f64);
2914 SDValue C5 = DAG.getConstantFP(Val: -0x1.f48ad494ea3e9p-53, DL, VT: MVT::f64);
2915 SDValue MulF = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: F, N2: C5, Flags);
2916 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: F, N2: C4, N3: MulF, Flags);
2917 } else { // ISD::FEXP
2918 // dn = rint(x * C1)
2919 SDValue C1 = DAG.getConstantFP(Val: 0x1.71547652b82fep+0, DL, VT: MVT::f64);
2920 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT: MVT::f64, N1: X, N2: C1, Flags);
2921 DN = DAG.getNode(Opcode: ISD::FRINT, DL, VT: MVT::f64, Operand: Mul, Flags);
2922
2923 // t = FMA(-dn, C2, FMA(-dn, C3, x))
2924 SDValue NegDN = DAG.getNode(Opcode: ISD::FNEG, DL, VT: MVT::f64, Operand: DN, Flags);
2925 SDValue C2 = DAG.getConstantFP(Val: 0x1.abc9e3b39803fp-56, DL, VT: MVT::f64);
2926 SDValue C3 = DAG.getConstantFP(Val: 0x1.62e42fefa39efp-1, DL, VT: MVT::f64);
2927 SDValue Inner = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C3, N3: X, Flags);
2928 T = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: NegDN, N2: C2, N3: Inner, Flags);
2929 }
2930
2931 // Polynomial expansion for p
2932 SDValue P = DAG.getConstantFP(Val: 0x1.ade156a5dcb37p-26, DL, VT: MVT::f64);
2933 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2934 N3: DAG.getConstantFP(Val: 0x1.28af3fca7ab0cp-22, DL, VT: MVT::f64),
2935 Flags);
2936 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2937 N3: DAG.getConstantFP(Val: 0x1.71dee623fde64p-19, DL, VT: MVT::f64),
2938 Flags);
2939 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2940 N3: DAG.getConstantFP(Val: 0x1.a01997c89e6b0p-16, DL, VT: MVT::f64),
2941 Flags);
2942 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2943 N3: DAG.getConstantFP(Val: 0x1.a01a014761f6ep-13, DL, VT: MVT::f64),
2944 Flags);
2945 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2946 N3: DAG.getConstantFP(Val: 0x1.6c16c1852b7b0p-10, DL, VT: MVT::f64),
2947 Flags);
2948 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2949 N3: DAG.getConstantFP(Val: 0x1.1111111122322p-7, DL, VT: MVT::f64), Flags);
2950 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2951 N3: DAG.getConstantFP(Val: 0x1.55555555502a1p-5, DL, VT: MVT::f64), Flags);
2952 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2953 N3: DAG.getConstantFP(Val: 0x1.5555555555511p-3, DL, VT: MVT::f64), Flags);
2954 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P,
2955 N3: DAG.getConstantFP(Val: 0x1.000000000000bp-1, DL, VT: MVT::f64), Flags);
2956
2957 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT: MVT::f64);
2958
2959 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
2960 P = DAG.getNode(Opcode: ISD::FMA, DL, VT: MVT::f64, N1: T, N2: P, N3: One, Flags);
2961
2962 // z = ldexp(p, (int)dn)
2963 SDValue DNInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: MVT::i32, Operand: DN);
2964 SDValue Z = DAG.getNode(Opcode: ISD::FLDEXP, DL, VT: MVT::f64, N1: P, N2: DNInt, Flags);
2965
2966 // Overflow/underflow guards
2967 SDValue CondHi = DAG.getSetCC(
2968 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: 1024.0, DL, VT: MVT::f64), Cond: ISD::SETULE);
2969
2970 if (!Flags.hasNoInfs()) {
2971 SDValue PInf = DAG.getConstantFP(Val: std::numeric_limits<double>::infinity(),
2972 DL, VT: MVT::f64);
2973 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondHi, LHS: Z, RHS: PInf, Flags);
2974 }
2975
2976 SDValue CondLo = DAG.getSetCC(
2977 DL, VT: MVT::i1, LHS: X, RHS: DAG.getConstantFP(Val: -1075.0, DL, VT: MVT::f64), Cond: ISD::SETUGE);
2978 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: MVT::f64);
2979 Z = DAG.getSelect(DL, VT: MVT::f64, Cond: CondLo, LHS: Z, RHS: Zero, Flags);
2980
2981 return Z;
2982}
2983
2984SDValue AMDGPUTargetLowering::lowerFEXP2(SDValue Op, SelectionDAG &DAG) const {
2985 // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
2986 // If we have to handle denormals, scale up the input and adjust the result.
2987
2988 EVT VT = Op.getValueType();
2989 if (VT == MVT::f64)
2990 return lowerFEXPF64(Op, DAG);
2991
2992 SDLoc SL(Op);
2993 SDValue Src = Op.getOperand(i: 0);
2994 SDNodeFlags Flags = Op->getFlags();
2995
2996 if (VT == MVT::f16) {
2997 // Nothing in half is a denormal when promoted to f32.
2998 assert(!isTypeLegal(MVT::f16));
2999 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3000 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Ext, Flags);
3001 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Log,
3002 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3003 }
3004
3005 assert(VT == MVT::f32);
3006
3007 if (!needsDenormHandlingF32(DAG, Src, Flags))
3008 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT: MVT::f32, Operand: Src, Flags);
3009
3010 // bool needs_scaling = x < -0x1.f80000p+6f;
3011 // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
3012
3013 // -nextafter(128.0, -1)
3014 SDValue RangeCheckConst = DAG.getConstantFP(Val: -0x1.f80000p+6f, DL: SL, VT);
3015
3016 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3017
3018 SDValue NeedsScaling =
3019 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: Src, RHS: RangeCheckConst, Cond: ISD::SETOLT);
3020
3021 SDValue SixtyFour = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3022 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3023
3024 SDValue AddOffset =
3025 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: SixtyFour, N3: Zero);
3026
3027 SDValue AddInput = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: Src, N2: AddOffset, Flags);
3028 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: AddInput, Flags);
3029
3030 SDValue TwoExpNeg64 = DAG.getConstantFP(Val: 0x1.0p-64f, DL: SL, VT);
3031 SDValue One = DAG.getConstantFP(Val: 1.0, DL: SL, VT);
3032 SDValue ResultScale =
3033 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: TwoExpNeg64, N3: One);
3034
3035 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScale, Flags);
3036}
3037
3038SDValue AMDGPUTargetLowering::lowerFEXPUnsafeImpl(SDValue X, const SDLoc &SL,
3039 SelectionDAG &DAG,
3040 SDNodeFlags Flags,
3041 bool IsExp10) const {
3042 // exp(x) -> exp2(M_LOG2E_F * x);
3043 // exp10(x) -> exp2(log2(10) * x);
3044 EVT VT = X.getValueType();
3045 SDValue Const =
3046 DAG.getConstantFP(Val: IsExp10 ? 0x1.a934f0p+1f : numbers::log2e, DL: SL, VT);
3047
3048 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: Const, Flags);
3049 return DAG.getNode(Opcode: VT == MVT::f32 ? (unsigned)AMDGPUISD::EXP
3050 : (unsigned)ISD::FEXP2,
3051 DL: SL, VT, Operand: Mul, Flags);
3052}
3053
3054SDValue AMDGPUTargetLowering::lowerFEXPUnsafe(SDValue X, const SDLoc &SL,
3055 SelectionDAG &DAG,
3056 SDNodeFlags Flags) const {
3057 EVT VT = X.getValueType();
3058 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags))
3059 return lowerFEXPUnsafeImpl(X, SL, DAG, Flags, /*IsExp10=*/false);
3060
3061 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3062
3063 SDValue Threshold = DAG.getConstantFP(Val: -0x1.5d58a0p+6f, DL: SL, VT);
3064 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3065
3066 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+6f, DL: SL, VT);
3067
3068 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3069
3070 SDValue AdjustedX =
3071 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3072
3073 const SDValue Log2E = DAG.getConstantFP(Val: numbers::log2e, DL: SL, VT);
3074 SDValue ExpInput = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: Log2E, Flags);
3075
3076 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: ExpInput, Flags);
3077
3078 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.969d48p-93f, DL: SL, VT);
3079 SDValue AdjustedResult =
3080 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2, N2: ResultScaleFactor, Flags);
3081
3082 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: Exp2,
3083 Flags);
3084}
3085
3086/// Emit approx-funcs appropriate lowering for exp10. inf/nan should still be
3087/// handled correctly.
3088SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL,
3089 SelectionDAG &DAG,
3090 SDNodeFlags Flags) const {
3091 const EVT VT = X.getValueType();
3092
3093 const unsigned Exp2Op = VT == MVT::f32 ? static_cast<unsigned>(AMDGPUISD::EXP)
3094 : static_cast<unsigned>(ISD::FEXP2);
3095
3096 if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, Src: X, Flags)) {
3097 // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
3098 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3099 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3100
3101 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K0, Flags);
3102 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3103 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: K1, Flags);
3104 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3105 return DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1);
3106 }
3107
3108 // bool s = x < -0x1.2f7030p+5f;
3109 // x += s ? 0x1.0p+5f : 0.0f;
3110 // exp10 = exp2(x * 0x1.a92000p+1f) *
3111 // exp2(x * 0x1.4f0978p-11f) *
3112 // (s ? 0x1.9f623ep-107f : 1.0f);
3113
3114 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3115
3116 SDValue Threshold = DAG.getConstantFP(Val: -0x1.2f7030p+5f, DL: SL, VT);
3117 SDValue NeedsScaling = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: Threshold, Cond: ISD::SETOLT);
3118
3119 SDValue ScaleOffset = DAG.getConstantFP(Val: 0x1.0p+5f, DL: SL, VT);
3120 SDValue ScaledX = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: X, N2: ScaleOffset, Flags);
3121 SDValue AdjustedX =
3122 DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: ScaledX, N3: X);
3123
3124 SDValue K0 = DAG.getConstantFP(Val: 0x1.a92000p+1f, DL: SL, VT);
3125 SDValue K1 = DAG.getConstantFP(Val: 0x1.4f0978p-11f, DL: SL, VT);
3126
3127 SDValue Mul0 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K0, Flags);
3128 SDValue Exp2_0 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul0, Flags);
3129 SDValue Mul1 = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: AdjustedX, N2: K1, Flags);
3130 SDValue Exp2_1 = DAG.getNode(Opcode: Exp2Op, DL: SL, VT, Operand: Mul1, Flags);
3131
3132 SDValue MulExps = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Exp2_0, N2: Exp2_1, Flags);
3133
3134 SDValue ResultScaleFactor = DAG.getConstantFP(Val: 0x1.9f623ep-107f, DL: SL, VT);
3135 SDValue AdjustedResult =
3136 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: MulExps, N2: ResultScaleFactor, Flags);
3137
3138 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NeedsScaling, N2: AdjustedResult, N3: MulExps,
3139 Flags);
3140}
3141
3142SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
3143 EVT VT = Op.getValueType();
3144
3145 if (VT == MVT::f64)
3146 return lowerFEXPF64(Op, DAG);
3147
3148 SDLoc SL(Op);
3149 SDValue X = Op.getOperand(i: 0);
3150 SDNodeFlags Flags = Op->getFlags();
3151 const bool IsExp10 = Op.getOpcode() == ISD::FEXP10;
3152
3153 // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
3154 // library behavior. Also, is known-not-daz source sufficient?
3155 if (allowApproxFunc(DAG, Flags)) { // TODO: Does this really require fast?
3156 return IsExp10 ? lowerFEXP10Unsafe(X, SL, DAG, Flags)
3157 : lowerFEXPUnsafe(X, SL, DAG, Flags);
3158 }
3159
3160 if (VT.getScalarType() == MVT::f16) {
3161 if (VT.isVector())
3162 return SDValue();
3163
3164 // Nothing in half is a denormal when promoted to f32.
3165 //
3166 // exp(f16 x) ->
3167 // fptrunc (v_exp_f32 (fmul (fpext x), log2e))
3168 //
3169 // exp10(f16 x) ->
3170 // fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
3171 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT: MVT::f32, Operand: X, Flags);
3172 SDValue Lowered = lowerFEXPUnsafeImpl(X: Ext, SL, DAG, Flags, IsExp10);
3173 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Lowered,
3174 N2: DAG.getTargetConstant(Val: 0, DL: SL, VT: MVT::i32), Flags);
3175 }
3176
3177 assert(VT == MVT::f32);
3178
3179 // Algorithm:
3180 //
3181 // e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
3182 //
3183 // x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
3184 // n = 64*m + j, 0 <= j < 64
3185 //
3186 // e^x = 2^((64*m + j + f)/64)
3187 // = (2^m) * (2^(j/64)) * 2^(f/64)
3188 // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
3189 //
3190 // f = x*(64/ln(2)) - n
3191 // r = f*(ln(2)/64) = x - n*(ln(2)/64)
3192 //
3193 // e^x = (2^m) * (2^(j/64)) * e^r
3194 //
3195 // (2^(j/64)) is precomputed
3196 //
3197 // e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3198 // e^r = 1 + q
3199 //
3200 // q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
3201 //
3202 // e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
3203 SDNodeFlags FlagsNoContract = Flags;
3204 FlagsNoContract.setAllowContract(false);
3205
3206 SDValue PH, PL;
3207 if (Subtarget->hasFastFMAF32()) {
3208 const float c_exp = numbers::log2ef;
3209 const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
3210 const float c_exp10 = 0x1.a934f0p+1f;
3211 const float cc_exp10 = 0x1.2f346ep-24f;
3212
3213 SDValue C = DAG.getConstantFP(Val: IsExp10 ? c_exp10 : c_exp, DL: SL, VT);
3214 SDValue CC = DAG.getConstantFP(Val: IsExp10 ? cc_exp10 : cc_exp, DL: SL, VT);
3215
3216 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: X, N2: C, Flags);
3217 SDValue NegPH = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: PH, Flags);
3218 SDValue FMA0 = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: C, N3: NegPH, Flags);
3219 PL = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT, N1: X, N2: CC, N3: FMA0, Flags);
3220 } else {
3221 const float ch_exp = 0x1.714000p+0f;
3222 const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
3223
3224 const float ch_exp10 = 0x1.a92000p+1f;
3225 const float cl_exp10 = 0x1.4f0978p-11f;
3226
3227 SDValue CH = DAG.getConstantFP(Val: IsExp10 ? ch_exp10 : ch_exp, DL: SL, VT);
3228 SDValue CL = DAG.getConstantFP(Val: IsExp10 ? cl_exp10 : cl_exp, DL: SL, VT);
3229
3230 SDValue XAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: X);
3231 SDValue MaskConst = DAG.getConstant(Val: 0xfffff000, DL: SL, VT: MVT::i32);
3232 SDValue XHAsInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: MVT::i32, N1: XAsInt, N2: MaskConst);
3233 SDValue XH = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: XHAsInt);
3234 SDValue XL = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: X, N2: XH, Flags);
3235
3236 PH = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XH, N2: CH, Flags);
3237
3238 SDValue XLCL = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: XL, N2: CL, Flags);
3239 SDValue Mad0 = getMad(DAG, SL, VT, X: XL, Y: CH, C: XLCL, Flags);
3240 PL = getMad(DAG, SL, VT, X: XH, Y: CL, C: Mad0, Flags);
3241 }
3242
3243 SDValue E = DAG.getNode(Opcode: ISD::FROUNDEVEN, DL: SL, VT, Operand: PH, Flags);
3244
3245 // It is unsafe to contract this fsub into the PH multiply.
3246 SDValue PHSubE = DAG.getNode(Opcode: ISD::FSUB, DL: SL, VT, N1: PH, N2: E, Flags: FlagsNoContract);
3247
3248 SDValue A = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: PHSubE, N2: PL, Flags);
3249 SDValue IntE = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: SL, VT: MVT::i32, Operand: E);
3250 SDValue Exp2 = DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: A, Flags);
3251
3252 SDValue R = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT, N1: Exp2, N2: IntE, Flags);
3253
3254 SDValue UnderflowCheckConst =
3255 DAG.getConstantFP(Val: IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, DL: SL, VT);
3256
3257 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3258 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: SL, VT);
3259 SDValue Underflow =
3260 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: UnderflowCheckConst, Cond: ISD::SETOLT);
3261
3262 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Underflow, N2: Zero, N3: R);
3263
3264 if (!Flags.hasNoInfs()) {
3265 SDValue OverflowCheckConst =
3266 DAG.getConstantFP(Val: IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, DL: SL, VT);
3267 SDValue Overflow =
3268 DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: X, RHS: OverflowCheckConst, Cond: ISD::SETOGT);
3269 SDValue Inf =
3270 DAG.getConstantFP(Val: APFloat::getInf(Sem: APFloat::IEEEsingle()), DL: SL, VT);
3271 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Overflow, N2: Inf, N3: R);
3272 }
3273
3274 return R;
3275}
3276
3277// No pow instruction or libcall to fall back on. fmul_legacy returns 0 for a
3278// zero operand even against an infinity or a NaN, so pow(x, 0) and pow(1, y)
3279// fall out as exp2(0) = 1.
3280SDValue AMDGPUTargetLowering::lowerFPOW(SDValue Op, SelectionDAG &DAG) const {
3281 EVT VT = Op.getValueType();
3282 assert(VT == MVT::f32);
3283
3284 SDLoc SL(Op);
3285 SDValue X = Op.getOperand(i: 0);
3286 SDValue Y = Op.getOperand(i: 1);
3287 SDNodeFlags Flags = Op->getFlags();
3288
3289 // log2(0) is -inf, which exp2 turns back into a finite result, so the core
3290 // goes infinite for inputs a ninf fpow still asserts about, like pow(0, 2).
3291 SDNodeFlags CoreFlags = Flags;
3292 CoreFlags.setNoInfs(false);
3293
3294 // Fast expansion: ignores denormals, NaN for a negative base.
3295 if (allowApproxFunc(DAG, Flags)) {
3296 SDValue Log = DAG.getNode(Opcode: AMDGPUISD::LOG, DL: SL, VT, Operand: X, Flags: CoreFlags);
3297 SDValue Mul =
3298 DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL: SL, VT, N1: Y, N2: Log, Flags: CoreFlags);
3299 return DAG.getNode(Opcode: AMDGPUISD::EXP, DL: SL, VT, Operand: Mul, Flags: CoreFlags);
3300 }
3301
3302 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT, Operand: X, Flags);
3303 SDValue Log = DAG.getNode(Opcode: ISD::FLOG2, DL: SL, VT, Operand: Abs, Flags: CoreFlags);
3304 SDValue Mul = DAG.getNode(Opcode: AMDGPUISD::FMUL_LEGACY, DL: SL, VT, N1: Y, N2: Log, Flags: CoreFlags);
3305 SDValue R = DAG.getNode(Opcode: ISD::FEXP2, DL: SL, VT, Operand: Mul, Flags: CoreFlags);
3306
3307 // A base that is never negative needs neither the sign fixup nor the NaN.
3308 if (DAG.computeKnownFPClass(Op: X, InterestedClasses: fcNegative).signBitIsZeroOrNaN())
3309 return R;
3310
3311 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
3312
3313 // Infinities count as integers, and every f32 >= 2^24 in magnitude is even.
3314 SDValue YTrunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: Y);
3315 SDValue YIsInt = DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: YTrunc, RHS: Y, Cond: ISD::SETOEQ);
3316 SDValue YHalf =
3317 DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT, N1: Y, N2: DAG.getConstantFP(Val: 0.5, DL: SL, VT));
3318 SDValue YHalfTrunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT, Operand: YHalf);
3319 SDValue YIsOdd =
3320 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: YIsInt,
3321 N2: DAG.getSetCC(DL: SL, VT: SetCCVT, LHS: YHalfTrunc, RHS: YHalf, Cond: ISD::SETONE));
3322
3323 // pow(-x, odd y) = -pow(x, y). Selecting the copysign lets even y fold it.
3324 R = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: YIsOdd,
3325 N2: DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SL, VT, N1: R, N2: X), N3: R);
3326
3327 if (Flags.hasNoNaNs())
3328 return R;
3329
3330 // A negative finite base to a non-integral power is NaN. -inf is excluded:
3331 // the core already gives pow(+inf, y). So are subnormals when flushed.
3332 FPClassTest NegFiniteMask = fcNegNormal;
3333 if (!DAG.getMachineFunction()
3334 .getDenormalMode(FPType: APFloat::IEEEsingle())
3335 .inputsAreZero())
3336 NegFiniteMask |= fcNegSubnormal;
3337 SDValue XNegFinite =
3338 DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: SL, VT: SetCCVT, N1: X,
3339 N2: DAG.getTargetConstant(Val: NegFiniteMask, DL: SL, VT: MVT::i32));
3340 // Not a SETONE compare: pow(-1, NaN) needs the NaN-true behavior of !SETOEQ.
3341 SDValue NegNonInt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SetCCVT, N1: XNegFinite,
3342 N2: DAG.getNOT(DL: SL, Val: YIsInt, VT: SetCCVT));
3343 SDValue NaN =
3344 DAG.getConstantFP(Val: APFloat::getQNaN(Sem: VT.getFltSemantics()), DL: SL, VT);
3345 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NegNonInt, N2: NaN, N3: R);
3346}
3347
3348static bool isCtlzOpc(unsigned Opc) {
3349 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_POISON;
3350}
3351
3352static bool isCttzOpc(unsigned Opc) {
3353 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_POISON;
3354}
3355
3356SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op,
3357 SelectionDAG &DAG) const {
3358 auto SL = SDLoc(Op);
3359 auto Opc = Op.getOpcode();
3360 auto Arg = Op.getOperand(i: 0u);
3361 auto ResultVT = Op.getValueType();
3362
3363 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3364 return {};
3365
3366 assert(isCtlzOpc(Opc));
3367 assert(ResultVT == Arg.getValueType());
3368
3369 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3370 SDValue NumExtBits = DAG.getConstant(Val: 32u - NumBits, DL: SL, VT: MVT::i32);
3371 SDValue NewOp;
3372
3373 if (Opc == ISD::CTLZ_ZERO_POISON) {
3374 NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3375 NewOp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3376 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3377 } else {
3378 NewOp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i32, Operand: Arg);
3379 NewOp = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, Operand: NewOp);
3380 NewOp = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: NewOp, N2: NumExtBits);
3381 }
3382
3383 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: ResultVT, Operand: NewOp);
3384}
3385
3386SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
3387 SDLoc SL(Op);
3388 SDValue Src = Op.getOperand(i: 0);
3389
3390 assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode()));
3391 bool Ctlz = isCtlzOpc(Opc: Op.getOpcode());
3392 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;
3393
3394 bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_POISON ||
3395 Op.getOpcode() == ISD::CTTZ_ZERO_POISON;
3396 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3397
3398 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3399 // (ctlz hi:lo) -> (umin (ffbh src), 32)
3400 // (cttz hi:lo) -> (umin (ffbl src), 32)
3401 // (ctlz_zero_poison src) -> (ffbh src)
3402 // (cttz_zero_poison src) -> (ffbl src)
3403
3404 // 64-bit scalar version produce 32-bit result
3405 // (ctlz hi:lo) -> (umin (S_FLBIT_I32_B64 src), 64)
3406 // (cttz hi:lo) -> (umin (S_FF1_I32_B64 src), 64)
3407 // (ctlz_zero_poison src) -> (S_FLBIT_I32_B64 src)
3408 // (cttz_zero_poison src) -> (S_FF1_I32_B64 src)
3409 SDValue NewOpr = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Src);
3410 if (!ZeroUndef) {
3411 const SDValue ConstVal = DAG.getConstant(
3412 Val: Op.getValueType().getScalarSizeInBits(), DL: SL, VT: MVT::i32);
3413 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: ConstVal);
3414 }
3415 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: Src.getValueType(), Operand: NewOpr);
3416 }
3417
3418 SDValue Lo, Hi;
3419 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3420
3421 SDValue OprLo = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Lo);
3422 SDValue OprHi = DAG.getNode(Opcode: NewOpc, DL: SL, VT: MVT::i32, Operand: Hi);
3423
3424 // (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64)
3425 // (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64)
3426 // (ctlz_zero_poison hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
3427 // (cttz_zero_poison hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))
3428
3429 unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT;
3430 const SDValue Const32 = DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32);
3431 if (Ctlz)
3432 OprLo = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprLo, N2: Const32);
3433 else
3434 OprHi = DAG.getNode(Opcode: AddOpc, DL: SL, VT: MVT::i32, N1: OprHi, N2: Const32);
3435
3436 SDValue NewOpr;
3437 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: OprLo, N2: OprHi);
3438 if (!ZeroUndef) {
3439 const SDValue Const64 = DAG.getConstant(Val: 64, DL: SL, VT: MVT::i32);
3440 NewOpr = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: NewOpr, N2: Const64);
3441 }
3442
3443 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: MVT::i64, Operand: NewOpr);
3444}
3445
3446SDValue AMDGPUTargetLowering::LowerCTLS(SDValue Op, SelectionDAG &DAG) const {
3447 SDLoc SL(Op);
3448 SDValue Src = Op.getOperand(i: 0);
3449 assert(Src.getValueType() == MVT::i32 && "LowerCTLS only supports i32");
3450 SDValue Ffbh = DAG.getNode(
3451 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3452 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Src);
3453 SDValue Clamped = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: Ffbh,
3454 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3455 return DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: Clamped,
3456 N2: DAG.getAllOnesConstant(DL: SL, VT: MVT::i32));
3457}
3458
3459SDValue AMDGPUTargetLowering::LowerINT_TO_FP16(SDValue Op, SelectionDAG &DAG,
3460 EVT FP16Ty) const {
3461 assert(FP16Ty == MVT::f16 || FP16Ty == MVT::bf16);
3462 SDLoc SL(Op);
3463 SDValue Src = Op.getOperand(i: 0);
3464 SDValue ToF32 = DAG.getNode(Opcode: Op.getOpcode(), DL: SL, VT: MVT::f32, Operand: Src);
3465 SDValue FPRoundFlag = DAG.getIntPtrConstant(Val: 0, DL: SL, /*isTarget=*/true);
3466 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: FP16Ty, N1: ToF32, N2: FPRoundFlag);
3467}
3468
3469SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
3470 bool Signed) const {
3471 // The regular method converting a 64-bit integer to float roughly consists of
3472 // 2 steps: normalization and rounding. In fact, after normalization, the
3473 // conversion from a 64-bit integer to a float is essentially the same as the
3474 // one from a 32-bit integer. The only difference is that it has more
3475 // trailing bits to be rounded. To leverage the native 32-bit conversion, a
3476 // 64-bit integer could be preprocessed and fit into a 32-bit integer then
3477 // converted into the correct float number. The basic steps for the unsigned
3478 // conversion are illustrated in the following pseudo code:
3479 //
3480 // f32 uitofp(i64 u) {
3481 // i32 hi, lo = split(u);
3482 // // Only count the leading zeros in hi as we have native support of the
3483 // // conversion from i32 to f32. If hi is all 0s, the conversion is
3484 // // reduced to a 32-bit one automatically.
3485 // i32 shamt = clz(hi); // Return 32 if hi is all 0s.
3486 // u <<= shamt;
3487 // hi, lo = split(u);
3488 // hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo.
3489 // // convert it as a 32-bit integer and scale the result back.
3490 // return uitofp(hi) * 2^(32 - shamt);
3491 // }
3492 //
3493 // The signed one follows the same principle but uses 'ffbh_i32' to count its
3494 // sign bits instead. If 'ffbh_i32' is not available, its absolute value is
3495 // converted instead followed by negation based its sign bit.
3496
3497 SDLoc SL(Op);
3498 SDValue Src = Op.getOperand(i: 0);
3499
3500 SDValue Lo, Hi;
3501 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3502 SDValue Sign;
3503 SDValue ShAmt;
3504 if (Signed && Subtarget->isGCN()) {
3505 // We also need to consider the sign bit in Lo if Hi has just sign bits,
3506 // i.e. Hi is 0 or -1. However, that only needs to take the MSB into
3507 // account. That is, the maximal shift is
3508 // - 32 if Lo and Hi have opposite signs;
3509 // - 33 if Lo and Hi have the same sign.
3510 //
3511 // Or, MaxShAmt = 33 + OppositeSign, where
3512 //
3513 // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is
3514 // - -1 if Lo and Hi have opposite signs; and
3515 // - 0 otherwise.
3516 //
3517 // All in all, ShAmt is calculated as
3518 //
3519 // umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1.
3520 //
3521 // or
3522 //
3523 // umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31).
3524 //
3525 // to reduce the critical path.
3526 SDValue OppositeSign = DAG.getNode(
3527 Opcode: ISD::SRA, DL: SL, VT: MVT::i32, N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i32, N1: Lo, N2: Hi),
3528 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3529 SDValue MaxShAmt =
3530 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3531 N2: OppositeSign);
3532 // Count the leading sign bits.
3533 ShAmt = DAG.getNode(
3534 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: SL, VT: MVT::i32,
3535 N1: DAG.getTargetConstant(Val: Intrinsic::amdgcn_sffbh, DL: SL, VT: MVT::i32), N2: Hi);
3536 // Different from unsigned conversion, the shift should be one bit less to
3537 // preserve the sign bit.
3538 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: ShAmt,
3539 N2: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32));
3540 ShAmt = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32, N1: ShAmt, N2: MaxShAmt);
3541 } else {
3542 if (Signed) {
3543 // Without 'ffbh_i32', only leading zeros could be counted. Take the
3544 // absolute value first.
3545 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i64, N1: Src,
3546 N2: DAG.getConstant(Val: 63, DL: SL, VT: MVT::i64));
3547 SDValue Abs =
3548 DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64,
3549 N1: DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i64, N1: Src, N2: Sign), N2: Sign);
3550 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Abs, DAG);
3551 }
3552 // Count the leading zeros.
3553 ShAmt = DAG.getNode(Opcode: ISD::CTLZ, DL: SL, VT: MVT::i32, Operand: Hi);
3554 // The shift amount for signed integers is [0, 32].
3555 }
3556 // Normalize the given 64-bit integer.
3557 SDValue Norm = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i64, N1: Src, N2: ShAmt);
3558 // Split it again.
3559 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Norm, DAG);
3560 // Calculate the adjust bit for rounding.
3561 // (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo)
3562 SDValue Adjust = DAG.getNode(Opcode: ISD::UMIN, DL: SL, VT: MVT::i32,
3563 N1: DAG.getConstant(Val: 1, DL: SL, VT: MVT::i32), N2: Lo);
3564 // Get the 32-bit normalized integer.
3565 Norm = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: Hi, N2: Adjust);
3566 // Convert the normalized 32-bit integer into f32.
3567
3568 bool UseLDEXP = isOperationLegal(Op: ISD::FLDEXP, VT: MVT::f32);
3569 unsigned Opc = Signed && UseLDEXP ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3570 SDValue FVal = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::f32, Operand: Norm);
3571
3572 // Finally, need to scale back the converted floating number as the original
3573 // 64-bit integer is converted as a 32-bit one.
3574 ShAmt = DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i32, N1: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32),
3575 N2: ShAmt);
3576 // On GCN, use LDEXP directly.
3577 if (UseLDEXP)
3578 return DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f32, N1: FVal, N2: ShAmt);
3579
3580 // Otherwise, align 'ShAmt' to the exponent part and add it into the exponent
3581 // part directly to emulate the multiplication of 2^ShAmt. That 8-bit
3582 // exponent is enough to avoid overflowing into the sign bit.
3583 SDValue Exp = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32, N1: ShAmt,
3584 N2: DAG.getConstant(Val: 23, DL: SL, VT: MVT::i32));
3585 SDValue IVal =
3586 DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32,
3587 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: FVal), N2: Exp);
3588 if (Signed) {
3589 // Set the sign bit.
3590 Sign = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: MVT::i32,
3591 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MVT::i32, Operand: Sign),
3592 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3593 IVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: MVT::i32, N1: IVal, N2: Sign);
3594 }
3595 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: IVal);
3596}
3597
3598SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
3599 bool Signed) const {
3600 SDLoc SL(Op);
3601 SDValue Src = Op.getOperand(i: 0);
3602
3603 SDValue Lo, Hi;
3604 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: Src, DAG);
3605
3606 SDValue CvtHi = DAG.getNode(Opcode: Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
3607 DL: SL, VT: MVT::f64, Operand: Hi);
3608
3609 SDValue CvtLo = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL: SL, VT: MVT::f64, Operand: Lo);
3610
3611 SDValue LdExp = DAG.getNode(Opcode: ISD::FLDEXP, DL: SL, VT: MVT::f64, N1: CvtHi,
3612 N2: DAG.getConstant(Val: 32, DL: SL, VT: MVT::i32));
3613 // TODO: Should this propagate fast-math-flags?
3614 return DAG.getNode(Opcode: ISD::FADD, DL: SL, VT: MVT::f64, N1: LdExp, N2: CvtLo);
3615}
3616
3617SDValue AMDGPUTargetLowering::lowerINT_TO_FPImpl(SDValue Op, SelectionDAG &DAG,
3618 bool Signed) const {
3619 EVT DestVT = Op.getValueType();
3620 SDValue Src = Op.getOperand(i: 0);
3621 EVT SrcVT = Src.getValueType();
3622 unsigned ExtOpc = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3623 unsigned CvtOpc = Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
3624
3625 if (SrcVT == MVT::i16) {
3626 if (DestVT == MVT::f16)
3627 return Op;
3628
3629 SDLoc DL(Op);
3630 SDValue Ext = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i32, Operand: Src);
3631 return DAG.getNode(Opcode: CvtOpc, DL, VT: DestVT, Operand: Ext);
3632 }
3633
3634 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3635 return LowerINT_TO_FP16(Op, DAG, FP16Ty: DestVT);
3636
3637 if (SrcVT != MVT::i64)
3638 return Op;
3639
3640 if (DestVT == MVT::f32)
3641 return LowerINT_TO_FP32(Op, DAG, Signed);
3642
3643 assert(DestVT == MVT::f64);
3644 return LowerINT_TO_FP64(Op, DAG, Signed);
3645}
3646
3647SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
3648 SelectionDAG &DAG) const {
3649 return lowerINT_TO_FPImpl(Op, DAG, Signed: false);
3650}
3651
3652SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
3653 SelectionDAG &DAG) const {
3654 return lowerINT_TO_FPImpl(Op, DAG, Signed: true);
3655}
3656
3657SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
3658 bool Signed) const {
3659 SDLoc SL(Op);
3660
3661 SDValue Src = Op.getOperand(i: 0);
3662 EVT SrcVT = Src.getValueType();
3663
3664 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3665
3666 // The basic idea of converting a floating point number into a pair of 32-bit
3667 // integers is illustrated as follows:
3668 //
3669 // tf := trunc(val);
3670 // hif := floor(tf * 2^-32);
3671 // lof := tf - hif * 2^32; // lof is always positive due to floor.
3672 // hi := fptoi(hif);
3673 // lo := fptoi(lof);
3674 //
3675 SDValue Trunc = DAG.getNode(Opcode: ISD::FTRUNC, DL: SL, VT: SrcVT, Operand: Src);
3676 SDValue Sign;
3677 if (Signed && SrcVT == MVT::f32) {
3678 // However, a 32-bit floating point number has only 23 bits mantissa and
3679 // it's not enough to hold all the significant bits of `lof` if val is
3680 // negative. To avoid the loss of precision, We need to take the absolute
3681 // value after truncating and flip the result back based on the original
3682 // signedness.
3683 Sign = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: MVT::i32,
3684 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Trunc),
3685 N2: DAG.getConstant(Val: 31, DL: SL, VT: MVT::i32));
3686 Trunc = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: SrcVT, Operand: Trunc);
3687 }
3688
3689 SDValue K0, K1;
3690 if (SrcVT == MVT::f64) {
3691 K0 = DAG.getConstantFP(
3692 Val: llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)), DL: SL,
3693 VT: SrcVT);
3694 K1 = DAG.getConstantFP(
3695 Val: llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)), DL: SL,
3696 VT: SrcVT);
3697 } else {
3698 K0 = DAG.getConstantFP(
3699 Val: llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)), DL: SL, VT: SrcVT);
3700 K1 = DAG.getConstantFP(
3701 Val: llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)), DL: SL, VT: SrcVT);
3702 }
3703 // TODO: Should this propagate fast-math-flags?
3704 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: SL, VT: SrcVT, N1: Trunc, N2: K0);
3705
3706 SDValue FloorMul = DAG.getNode(Opcode: ISD::FFLOOR, DL: SL, VT: SrcVT, Operand: Mul);
3707
3708 SDValue Fma = DAG.getNode(Opcode: ISD::FMA, DL: SL, VT: SrcVT, N1: FloorMul, N2: K1, N3: Trunc);
3709
3710 SDValue Hi = DAG.getNode(Opcode: (Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
3711 : ISD::FP_TO_UINT,
3712 DL: SL, VT: MVT::i32, Operand: FloorMul);
3713 SDValue Lo = DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SL, VT: MVT::i32, Operand: Fma);
3714
3715 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3716 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Lo, Hi}));
3717
3718 if (Signed && SrcVT == MVT::f32) {
3719 assert(Sign);
3720 // Flip the result based on the signedness, which is either all 0s or 1s.
3721 Sign = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64,
3722 Operand: DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {Sign, Sign}));
3723 // r := xor(r, sign) - sign;
3724 Result =
3725 DAG.getNode(Opcode: ISD::SUB, DL: SL, VT: MVT::i64,
3726 N1: DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: MVT::i64, N1: Result, N2: Sign), N2: Sign);
3727 }
3728
3729 return Result;
3730}
3731
3732SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
3733 SDLoc DL(Op);
3734 SDValue N0 = Op.getOperand(i: 0);
3735
3736 // Convert to target node to get known bits
3737 if (N0.getValueType() == MVT::f32)
3738 return DAG.getNode(Opcode: AMDGPUISD::FP_TO_FP16, DL, VT: Op.getValueType(), Operand: N0);
3739
3740 if (Op->getFlags().hasApproximateFuncs()) {
3741 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
3742 return SDValue();
3743 }
3744
3745 return LowerF64ToF16Safe(Src: N0, DL, DAG);
3746}
3747
3748// return node in i32
3749SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
3750 SelectionDAG &DAG) const {
3751 assert(Src.getSimpleValueType() == MVT::f64);
3752
3753 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
3754 // TODO: We can generate better code for True16.
3755 const unsigned ExpMask = 0x7ff;
3756 const unsigned ExpBiasf64 = 1023;
3757 const unsigned ExpBiasf16 = 15;
3758 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
3759 SDValue One = DAG.getConstant(Val: 1, DL, VT: MVT::i32);
3760 SDValue U = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i64, Operand: Src);
3761 SDValue UH = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: U,
3762 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
3763 UH = DAG.getZExtOrTrunc(Op: UH, DL, VT: MVT::i32);
3764 U = DAG.getZExtOrTrunc(Op: U, DL, VT: MVT::i32);
3765 SDValue E = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3766 N2: DAG.getConstant(Val: 20, DL, VT: MVT::i64));
3767 E = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: E,
3768 N2: DAG.getConstant(Val: ExpMask, DL, VT: MVT::i32));
3769 // Subtract the fp64 exponent bias (1023) to get the real exponent and
3770 // add the f16 bias (15) to get the biased exponent for the f16 format.
3771 E = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: E,
3772 N2: DAG.getConstant(Val: -ExpBiasf64 + ExpBiasf16, DL, VT: MVT::i32));
3773
3774 SDValue M = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3775 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i32));
3776 M = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: M,
3777 N2: DAG.getConstant(Val: 0xffe, DL, VT: MVT::i32));
3778
3779 SDValue MaskedSig = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: UH,
3780 N2: DAG.getConstant(Val: 0x1ff, DL, VT: MVT::i32));
3781 MaskedSig = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: MaskedSig, N2: U);
3782
3783 SDValue Lo40Set = DAG.getSelectCC(DL, LHS: MaskedSig, RHS: Zero, True: Zero, False: One, Cond: ISD::SETEQ);
3784 M = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M, N2: Lo40Set);
3785
3786 // (M != 0 ? 0x0200 : 0) | 0x7c00;
3787 SDValue I = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32,
3788 N1: DAG.getSelectCC(DL, LHS: M, RHS: Zero, True: DAG.getConstant(Val: 0x0200, DL, VT: MVT::i32),
3789 False: Zero, Cond: ISD::SETNE), N2: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32));
3790
3791 // N = M | (E << 12);
3792 SDValue N = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3793 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: E,
3794 N2: DAG.getConstant(Val: 12, DL, VT: MVT::i32)));
3795
3796 // B = clamp(1-E, 0, 13);
3797 SDValue OneSubExp = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i32,
3798 N1: One, N2: E);
3799 SDValue B = DAG.getNode(Opcode: ISD::SMAX, DL, VT: MVT::i32, N1: OneSubExp, N2: Zero);
3800 B = DAG.getNode(Opcode: ISD::SMIN, DL, VT: MVT::i32, N1: B,
3801 N2: DAG.getConstant(Val: 13, DL, VT: MVT::i32));
3802
3803 SDValue SigSetHigh = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: M,
3804 N2: DAG.getConstant(Val: 0x1000, DL, VT: MVT::i32));
3805
3806 SDValue D = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: SigSetHigh, N2: B);
3807 SDValue D0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: D, N2: B);
3808 SDValue D1 = DAG.getSelectCC(DL, LHS: D0, RHS: SigSetHigh, True: One, False: Zero, Cond: ISD::SETNE);
3809 D = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: D, N2: D1);
3810
3811 SDValue V = DAG.getSelectCC(DL, LHS: E, RHS: One, True: D, False: N, Cond: ISD::SETLT);
3812 SDValue VLow3 = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: V,
3813 N2: DAG.getConstant(Val: 0x7, DL, VT: MVT::i32));
3814 V = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: V,
3815 N2: DAG.getConstant(Val: 2, DL, VT: MVT::i32));
3816 SDValue V0 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 3, DL, VT: MVT::i32),
3817 True: One, False: Zero, Cond: ISD::SETEQ);
3818 SDValue V1 = DAG.getSelectCC(DL, LHS: VLow3, RHS: DAG.getConstant(Val: 5, DL, VT: MVT::i32),
3819 True: One, False: Zero, Cond: ISD::SETGT);
3820 V1 = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: V0, N2: V1);
3821 V = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i32, N1: V, N2: V1);
3822
3823 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 30, DL, VT: MVT::i32),
3824 True: DAG.getConstant(Val: 0x7c00, DL, VT: MVT::i32), False: V, Cond: ISD::SETGT);
3825 V = DAG.getSelectCC(DL, LHS: E, RHS: DAG.getConstant(Val: 1039, DL, VT: MVT::i32),
3826 True: I, False: V, Cond: ISD::SETEQ);
3827
3828 // Extract the sign bit.
3829 SDValue Sign = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: UH,
3830 N2: DAG.getConstant(Val: 16, DL, VT: MVT::i32));
3831 Sign = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Sign,
3832 N2: DAG.getConstant(Val: 0x8000, DL, VT: MVT::i32));
3833
3834 return DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Sign, N2: V);
3835}
3836
3837SDValue AMDGPUTargetLowering::LowerFP_TO_INT(const SDValue Op,
3838 SelectionDAG &DAG) const {
3839 SDValue Src = Op.getOperand(i: 0);
3840 unsigned OpOpcode = Op.getOpcode();
3841 EVT SrcVT = Src.getValueType();
3842 EVT DestVT = Op.getValueType();
3843
3844 // Will be selected natively
3845 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3846 return Op;
3847
3848 if (SrcVT == MVT::bf16 || (SrcVT == MVT::f16 && DestVT == MVT::i32)) {
3849 SDLoc DL(Op);
3850 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3851 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DestVT, Operand: PromotedSrc);
3852 }
3853
3854 // Promote i16 to i32
3855 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3856 SDLoc DL(Op);
3857
3858 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3859 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FpToInt32);
3860 }
3861
3862 if (DestVT != MVT::i64)
3863 return Op;
3864
3865 if (SrcVT == MVT::f16 ||
3866 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
3867 SDLoc DL(Op);
3868
3869 SDValue FpToInt32 = DAG.getNode(Opcode: OpOpcode, DL, VT: MVT::i32, Operand: Src);
3870 unsigned Ext =
3871 OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3872 return DAG.getNode(Opcode: Ext, DL, VT: MVT::i64, Operand: FpToInt32);
3873 }
3874
3875 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3876 return LowerFP_TO_INT64(Op, DAG, Signed: OpOpcode == ISD::FP_TO_SINT);
3877
3878 return SDValue();
3879}
3880
3881SDValue AMDGPUTargetLowering::LowerFP_TO_INT_SAT(const SDValue Op,
3882 SelectionDAG &DAG) const {
3883 SDValue Src = Op.getOperand(i: 0);
3884 unsigned OpOpcode = Op.getOpcode();
3885 EVT SrcVT = Src.getValueType();
3886 EVT DstVT = Op.getValueType();
3887 SDValue SatVTOp = Op.getNode()->getOperand(Num: 1);
3888 EVT SatVT = cast<VTSDNode>(Val&: SatVTOp)->getVT();
3889 SDLoc DL(Op);
3890
3891 uint64_t DstWidth = DstVT.getScalarSizeInBits();
3892 uint64_t SatWidth = SatVT.getScalarSizeInBits();
3893 assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
3894
3895 // Scalar cases will be selected natively to v_cvt_/s_cvt_ instructions.
3896 // v2f32 -> v2i16 will be selected natively to v_cvt_pk_[iu]16_f32.
3897 if (SatWidth == DstWidth) {
3898 if ((DstVT == MVT::i32 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
3899 (DstVT == MVT::i16 && (SrcVT == MVT::f16 || SrcVT == MVT::f32)) ||
3900 (DstVT == MVT::v2i16 && SrcVT == MVT::v2f32))
3901 return Op;
3902 }
3903
3904 // Vectors can only be selected natively.
3905 if (DstVT.isVector())
3906 return SDValue();
3907
3908 // Perform all saturation at selected width (i16 or i32) and truncate
3909 if (SatWidth < DstWidth && SatWidth <= 32) {
3910 // For f16 conversion with sub-i16 saturation perform saturation
3911 // at i16, if available in the target. This removes the need for extra f16
3912 // to f32 conversion. For all the others use i32.
3913 MVT ResultVT =
3914 Subtarget->has16BitInsts() && SrcVT == MVT::f16 && SatWidth < 16
3915 ? MVT::i16
3916 : MVT::i32;
3917
3918 const SDValue ResultVTOp = DAG.getValueType(ResultVT);
3919 const uint64_t ResultWidth = ResultVT.getScalarSizeInBits();
3920
3921 // First, convert input float into selected integer (i16 or i32)
3922 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: ResultVT, N1: Src, N2: ResultVTOp);
3923 SDValue IntSatVal;
3924
3925 // Then, clamp at the saturation width using either i16 or i32 instructions
3926 if (OpOpcode == ISD::FP_TO_SINT_SAT) {
3927 SDValue MinConst = DAG.getConstant(
3928 Val: APInt::getSignedMaxValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3929 SDValue MaxConst = DAG.getConstant(
3930 Val: APInt::getSignedMinValue(numBits: SatWidth).sext(width: ResultWidth), DL, VT: ResultVT);
3931 SDValue MinVal = DAG.getNode(Opcode: ISD::SMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3932 IntSatVal = DAG.getNode(Opcode: ISD::SMAX, DL, VT: ResultVT, N1: MinVal, N2: MaxConst);
3933 } else {
3934 SDValue MinConst = DAG.getConstant(
3935 Val: APInt::getMaxValue(numBits: SatWidth).zext(width: ResultWidth), DL, VT: ResultVT);
3936 IntSatVal = DAG.getNode(Opcode: ISD::UMIN, DL, VT: ResultVT, N1: FpToInt, N2: MinConst);
3937 }
3938
3939 // Finally, after saturating at i16 or i32 fit into the destination type
3940 return DAG.getExtOrTrunc(IsSigned: OpOpcode == ISD::FP_TO_SINT_SAT, Op: IntSatVal, DL,
3941 VT: DstVT);
3942 }
3943
3944 // SatWidth == DstWidth or SatWidth > 32
3945
3946 // Saturate at i32 for i64 dst and f16/bf16 src (will invoke f16 promotion
3947 // below)
3948 if (DstVT == MVT::i64 &&
3949 (SrcVT == MVT::f16 || SrcVT == MVT::bf16 ||
3950 (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP))) {
3951 const SDValue Int32VTOp = DAG.getValueType(MVT::i32);
3952 return DAG.getNode(Opcode: OpOpcode, DL, VT: DstVT, N1: Src, N2: Int32VTOp);
3953 }
3954
3955 // Promote f16/bf16 src to f32 for i32 conversion
3956 if (DstVT == MVT::i32 && (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
3957 SDValue PromotedSrc = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Src);
3958 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DstVT, N1: PromotedSrc, N2: SatVTOp);
3959 }
3960
3961 // For DstWidth < 16, promote i1 and i8 dst to i16 (if legal) with sub-i16
3962 // saturation. For DstWidth == 16, promote i16 dst to i32 with sub-i32
3963 // saturation; this covers i16.f32 and i16.f64
3964 if (DstWidth < 32) {
3965 // Note: this triggers SatWidth < DstWidth above to generate saturated
3966 // truncate by requesting MVT::i16/i32 destination with SatWidth < 16/32.
3967 MVT PromoteVT =
3968 (DstWidth < 16 && Subtarget->has16BitInsts()) ? MVT::i16 : MVT::i32;
3969 SDValue FpToInt = DAG.getNode(Opcode: OpOpcode, DL, VT: PromoteVT, N1: Src, N2: SatVTOp);
3970 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: FpToInt);
3971 }
3972
3973 // TODO: can we implement i64 dst for f32/f64?
3974
3975 return SDValue();
3976}
3977
3978SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
3979 SelectionDAG &DAG) const {
3980 EVT ExtraVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3981 MVT VT = Op.getSimpleValueType();
3982 MVT ScalarVT = VT.getScalarType();
3983
3984 assert(VT.isVector());
3985
3986 SDValue Src = Op.getOperand(i: 0);
3987 SDLoc DL(Op);
3988
3989 // TODO: Don't scalarize on Evergreen?
3990 unsigned NElts = VT.getVectorNumElements();
3991 SmallVector<SDValue, 8> Args;
3992 DAG.ExtractVectorElements(Op: Src, Args, Start: 0, Count: NElts);
3993
3994 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
3995 for (unsigned I = 0; I < NElts; ++I)
3996 Args[I] = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: ScalarVT, N1: Args[I], N2: VTOp);
3997
3998 return DAG.getBuildVector(VT, DL, Ops: Args);
3999}
4000
4001//===----------------------------------------------------------------------===//
4002// Custom DAG optimizations
4003//===----------------------------------------------------------------------===//
4004
4005static bool isU24(SDValue Op, SelectionDAG &DAG) {
4006 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
4007}
4008
4009static bool isI24(SDValue Op, SelectionDAG &DAG) {
4010 EVT VT = Op.getValueType();
4011 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
4012 // as unsigned 24-bit values.
4013 AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24;
4014}
4015
4016static SDValue simplifyMul24(SDNode *Node24,
4017 TargetLowering::DAGCombinerInfo &DCI) {
4018 SelectionDAG &DAG = DCI.DAG;
4019 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4020 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
4021
4022 SDValue LHS = IsIntrin ? Node24->getOperand(Num: 1) : Node24->getOperand(Num: 0);
4023 SDValue RHS = IsIntrin ? Node24->getOperand(Num: 2) : Node24->getOperand(Num: 1);
4024 unsigned NewOpcode = Node24->getOpcode();
4025 if (IsIntrin) {
4026 unsigned IID = Node24->getConstantOperandVal(Num: 0);
4027 switch (IID) {
4028 case Intrinsic::amdgcn_mul_i24:
4029 NewOpcode = AMDGPUISD::MUL_I24;
4030 break;
4031 case Intrinsic::amdgcn_mul_u24:
4032 NewOpcode = AMDGPUISD::MUL_U24;
4033 break;
4034 case Intrinsic::amdgcn_mulhi_i24:
4035 NewOpcode = AMDGPUISD::MULHI_I24;
4036 break;
4037 case Intrinsic::amdgcn_mulhi_u24:
4038 NewOpcode = AMDGPUISD::MULHI_U24;
4039 break;
4040 default:
4041 llvm_unreachable("Expected 24-bit mul intrinsic");
4042 }
4043 }
4044
4045 APInt Demanded = APInt::getLowBitsSet(numBits: LHS.getValueSizeInBits(), loBitsSet: 24);
4046
4047 if (isNullConstant(V: LHS) || isNullConstant(V: RHS))
4048 return DAG.getConstant(Val: 0, DL: SDLoc(Node24), VT: Node24->getValueType(ResNo: 0));
4049
4050 // First try SimplifyDemandedBits which can simplify the nodes used by our
4051 // operands if this node is the only user.
4052 if (LHS.hasOneUse() && TLI.SimplifyDemandedBits(Op: LHS, DemandedBits: Demanded, DCI))
4053 return SDValue(Node24, 0);
4054 if (RHS.hasOneUse() && TLI.SimplifyDemandedBits(Op: RHS, DemandedBits: Demanded, DCI))
4055 return SDValue(Node24, 0);
4056
4057 // Then try SimplifyMultipleUseDemandedBits which allows the operands to have
4058 // other uses, but will only perform simplifications that involve bypassing
4059 // some nodes for this user.
4060 SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(Op: LHS, DemandedBits: Demanded, DAG);
4061 SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(Op: RHS, DemandedBits: Demanded, DAG);
4062 if (DemandedLHS || DemandedRHS)
4063 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(Node24), VTList: Node24->getVTList(),
4064 N1: DemandedLHS ? DemandedLHS : LHS,
4065 N2: DemandedRHS ? DemandedRHS : RHS);
4066
4067 return SDValue();
4068}
4069
4070template <typename IntTy>
4071static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
4072 uint32_t Width, const SDLoc &DL) {
4073 if (Width + Offset < 32) {
4074 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
4075 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
4076 if constexpr (std::is_signed_v<IntTy>) {
4077 return DAG.getSignedConstant(Val: Result, DL, VT: MVT::i32);
4078 } else {
4079 return DAG.getConstant(Result, DL, MVT::i32);
4080 }
4081 }
4082
4083 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
4084}
4085
4086static bool hasVolatileUser(SDNode *Val) {
4087 for (SDNode *U : Val->users()) {
4088 if (MemSDNode *M = dyn_cast<MemSDNode>(Val: U)) {
4089 if (M->isVolatile())
4090 return true;
4091 }
4092 }
4093
4094 return false;
4095}
4096
4097bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
4098 // i32 vectors are the canonical memory type.
4099 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
4100 return false;
4101
4102 if (!VT.isByteSized())
4103 return false;
4104
4105 unsigned Size = VT.getStoreSize();
4106
4107 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
4108 return false;
4109
4110 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
4111 return false;
4112
4113 return true;
4114}
4115
4116// Replace load of an illegal type with a bitcast from a load of a friendlier
4117// type.
4118SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
4119 DAGCombinerInfo &DCI) const {
4120 if (!DCI.isBeforeLegalize())
4121 return SDValue();
4122
4123 LoadSDNode *LN = cast<LoadSDNode>(Val: N);
4124 if (!LN->isSimple() || !ISD::isNormalLoad(N: LN) || hasVolatileUser(Val: LN))
4125 return SDValue();
4126
4127 SDLoc SL(N);
4128 SelectionDAG &DAG = DCI.DAG;
4129 EVT VT = LN->getMemoryVT();
4130
4131 unsigned Size = VT.getStoreSize();
4132 Align Alignment = LN->getAlign();
4133 if (Alignment < Size && isTypeLegal(VT)) {
4134 unsigned IsFast;
4135 unsigned AS = LN->getAddressSpace();
4136
4137 // Expand unaligned loads earlier than legalization. Due to visitation order
4138 // problems during legalization, the emitted instructions to pack and unpack
4139 // the bytes again are not eliminated in the case of an unaligned copy.
4140 if (!allowsMisalignedMemoryAccesses(
4141 VT, AddrSpace: AS, Alignment, Flags: LN->getMemOperand()->getFlags(), &IsFast)) {
4142 if (VT.isVector())
4143 return SplitVectorLoad(Op: SDValue(LN, 0), DAG);
4144
4145 SDValue Ops[2];
4146 std::tie(args&: Ops[0], args&: Ops[1]) = expandUnalignedLoad(LD: LN, DAG);
4147
4148 return DAG.getMergeValues(Ops, dl: SDLoc(N));
4149 }
4150
4151 if (!IsFast)
4152 return SDValue();
4153 }
4154
4155 if (!shouldCombineMemoryType(VT))
4156 return SDValue();
4157
4158 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4159
4160 SDValue NewLoad
4161 = DAG.getLoad(VT: NewVT, dl: SL, Chain: LN->getChain(),
4162 Ptr: LN->getBasePtr(), MMO: LN->getMemOperand());
4163
4164 SDValue BC = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: NewLoad);
4165 DCI.CombineTo(N, Res0: BC, Res1: NewLoad.getValue(R: 1));
4166 return SDValue(N, 0);
4167}
4168
4169// Replace store of an illegal type with a store of a bitcast to a friendlier
4170// type.
4171SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
4172 DAGCombinerInfo &DCI) const {
4173 if (!DCI.isBeforeLegalize())
4174 return SDValue();
4175
4176 StoreSDNode *SN = cast<StoreSDNode>(Val: N);
4177 if (!SN->isSimple() || !ISD::isNormalStore(N: SN))
4178 return SDValue();
4179
4180 EVT VT = SN->getMemoryVT();
4181 unsigned Size = VT.getStoreSize();
4182
4183 SDLoc SL(N);
4184 SelectionDAG &DAG = DCI.DAG;
4185 Align Alignment = SN->getAlign();
4186 if (Alignment < Size && isTypeLegal(VT)) {
4187 unsigned IsFast;
4188 unsigned AS = SN->getAddressSpace();
4189
4190 // Expand unaligned stores earlier than legalization. Due to visitation
4191 // order problems during legalization, the emitted instructions to pack and
4192 // unpack the bytes again are not eliminated in the case of an unaligned
4193 // copy.
4194 if (!allowsMisalignedMemoryAccesses(
4195 VT, AddrSpace: AS, Alignment, Flags: SN->getMemOperand()->getFlags(), &IsFast)) {
4196 if (VT.isVector())
4197 return SplitVectorStore(Op: SDValue(SN, 0), DAG);
4198
4199 return expandUnalignedStore(ST: SN, DAG);
4200 }
4201
4202 if (!IsFast)
4203 return SDValue();
4204 }
4205
4206 if (!shouldCombineMemoryType(VT))
4207 return SDValue();
4208
4209 EVT NewVT = getEquivalentMemType(Ctx&: *DAG.getContext(), VT);
4210 SDValue Val = SN->getValue();
4211
4212 // DCI.AddToWorklist(Val.getNode());
4213
4214 bool OtherUses = !Val.hasOneUse();
4215 SDValue CastVal = DAG.getBitcast(VT: NewVT, V: Val);
4216 if (OtherUses) {
4217 SDValue CastBack = DAG.getBitcast(VT, V: CastVal);
4218 DAG.ReplaceAllUsesOfValueWith(From: Val, To: CastBack);
4219 }
4220
4221 return DAG.getStore(Chain: SN->getChain(), dl: SL, Val: CastVal,
4222 Ptr: SN->getBasePtr(), MMO: SN->getMemOperand());
4223}
4224
4225// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
4226// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
4227// issues.
4228SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
4229 DAGCombinerInfo &DCI) const {
4230 SelectionDAG &DAG = DCI.DAG;
4231 SDValue N0 = N->getOperand(Num: 0);
4232
4233 // (vt2 (assertzext (truncate vt0:x), vt1)) ->
4234 // (vt2 (truncate (assertzext vt0:x, vt1)))
4235 if (N0.getOpcode() == ISD::TRUNCATE) {
4236 SDValue N1 = N->getOperand(Num: 1);
4237 EVT ExtVT = cast<VTSDNode>(Val&: N1)->getVT();
4238 SDLoc SL(N);
4239
4240 SDValue Src = N0.getOperand(i: 0);
4241 EVT SrcVT = Src.getValueType();
4242 if (SrcVT.bitsGE(VT: ExtVT)) {
4243 SDValue NewInReg = DAG.getNode(Opcode: N->getOpcode(), DL: SL, VT: SrcVT, N1: Src, N2: N1);
4244 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: N->getValueType(ResNo: 0), Operand: NewInReg);
4245 }
4246 }
4247
4248 return SDValue();
4249}
4250
4251SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
4252 SDNode *N, DAGCombinerInfo &DCI) const {
4253 unsigned IID = N->getConstantOperandVal(Num: 0);
4254 switch (IID) {
4255 case Intrinsic::amdgcn_mul_i24:
4256 case Intrinsic::amdgcn_mul_u24:
4257 case Intrinsic::amdgcn_mulhi_i24:
4258 case Intrinsic::amdgcn_mulhi_u24:
4259 return simplifyMul24(Node24: N, DCI);
4260 case Intrinsic::amdgcn_fract:
4261 case Intrinsic::amdgcn_rsq:
4262 case Intrinsic::amdgcn_rcp_legacy:
4263 case Intrinsic::amdgcn_rsq_legacy:
4264 case Intrinsic::amdgcn_rsq_clamp:
4265 case Intrinsic::amdgcn_tanh:
4266 case Intrinsic::amdgcn_prng_b32: {
4267 // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
4268 SDValue Src = N->getOperand(Num: 1);
4269 return Src.isUndef() ? Src : SDValue();
4270 }
4271 case Intrinsic::amdgcn_frexp_exp: {
4272 // frexp_exp (fneg x) -> frexp_exp x
4273 // frexp_exp (fabs x) -> frexp_exp x
4274 // frexp_exp (fneg (fabs x)) -> frexp_exp x
4275 SDValue Src = N->getOperand(Num: 1);
4276 SDValue PeekSign = peekFPSignOps(Val: Src);
4277 if (PeekSign == Src)
4278 return SDValue();
4279 return SDValue(DCI.DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0), Op2: PeekSign),
4280 0);
4281 }
4282 default:
4283 return SDValue();
4284 }
4285}
4286
4287/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
4288/// binary operation \p Opc to it with the corresponding constant operands.
4289SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
4290 DAGCombinerInfo &DCI, const SDLoc &SL,
4291 unsigned Opc, SDValue LHS,
4292 uint32_t ValLo, uint32_t ValHi) const {
4293 SelectionDAG &DAG = DCI.DAG;
4294 SDValue Lo, Hi;
4295 std::tie(args&: Lo, args&: Hi) = split64BitValue(Op: LHS, DAG);
4296
4297 SDValue LoRHS = DAG.getConstant(Val: ValLo, DL: SL, VT: MVT::i32);
4298 SDValue HiRHS = DAG.getConstant(Val: ValHi, DL: SL, VT: MVT::i32);
4299
4300 SDValue LoAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Lo, N2: LoRHS);
4301 SDValue HiAnd = DAG.getNode(Opcode: Opc, DL: SL, VT: MVT::i32, N1: Hi, N2: HiRHS);
4302
4303 // Re-visit the ands. It's possible we eliminated one of them and it could
4304 // simplify the vector.
4305 DCI.AddToWorklist(N: Lo.getNode());
4306 DCI.AddToWorklist(N: Hi.getNode());
4307
4308 SDValue Vec = DAG.getBuildVector(VT: MVT::v2i32, DL: SL, Ops: {LoAnd, HiAnd});
4309 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i64, Operand: Vec);
4310}
4311
4312SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
4313 DAGCombinerInfo &DCI) const {
4314 EVT VT = N->getValueType(ResNo: 0);
4315 SDValue LHS = N->getOperand(Num: 0);
4316 SDValue RHS = N->getOperand(Num: 1);
4317 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4318 SDLoc SL(N);
4319 SelectionDAG &DAG = DCI.DAG;
4320
4321 unsigned RHSVal;
4322 if (CRHS) {
4323 RHSVal = CRHS->getZExtValue();
4324 if (!RHSVal)
4325 return LHS;
4326
4327 switch (LHS->getOpcode()) {
4328 default:
4329 break;
4330 case ISD::ZERO_EXTEND:
4331 case ISD::SIGN_EXTEND:
4332 case ISD::ANY_EXTEND: {
4333 SDValue X = LHS->getOperand(Num: 0);
4334
4335 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
4336 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: MVT::v2i16)) {
4337 // Prefer build_vector as the canonical form if packed types are legal.
4338 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
4339 SDValue Vec = DAG.getBuildVector(
4340 VT: MVT::v2i16, DL: SL,
4341 Ops: {DAG.getConstant(Val: 0, DL: SL, VT: MVT::i16), LHS->getOperand(Num: 0)});
4342 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::i32, Operand: Vec);
4343 }
4344
4345 // shl (ext x) => zext (shl x), if shift does not overflow int
4346 if (VT != MVT::i64)
4347 break;
4348 KnownBits Known = DAG.computeKnownBits(Op: X);
4349 unsigned LZ = Known.countMinLeadingZeros();
4350 if (LZ < RHSVal)
4351 break;
4352 EVT XVT = X.getValueType();
4353 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: XVT, N1: X, N2: SDValue(CRHS, 0));
4354 return DAG.getZExtOrTrunc(Op: Shl, DL: SL, VT);
4355 }
4356 }
4357 }
4358
4359 if (VT.getScalarType() != MVT::i64)
4360 return SDValue();
4361
4362 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4363 // common case, splitting this into a move and a 32-bit shift is faster and
4364 // the same code size.
4365 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4366
4367 EVT ElementType = VT.getScalarType();
4368 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4369 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4370
4371 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4372 return SDValue();
4373 SDValue ShiftAmt;
4374
4375 if (CRHS) {
4376 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4377 VT: TargetType);
4378 } else {
4379 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4380 const SDValue ShiftMask =
4381 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4382 // This AND instruction will clamp out of bounds shift values.
4383 // It will also be removed during later instruction selection.
4384 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4385 }
4386
4387 SDValue Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: LHS);
4388 SDValue NewShift =
4389 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: TargetType, N1: Lo, N2: ShiftAmt, Flags: N->getFlags());
4390
4391 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4392 SDValue Vec;
4393
4394 if (VT.isVector()) {
4395 EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4396 unsigned NElts = TargetType.getVectorNumElements();
4397 SmallVector<SDValue, 8> HiOps;
4398 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4399
4400 DAG.ExtractVectorElements(Op: NewShift, Args&: HiOps, Start: 0, Count: NElts);
4401 for (unsigned I = 0; I != NElts; ++I)
4402 HiAndLoOps[2 * I + 1] = HiOps[I];
4403 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4404 } else {
4405 EVT ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4406 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {Zero, NewShift});
4407 }
4408 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4409}
4410
4411SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
4412 DAGCombinerInfo &DCI) const {
4413 SDValue RHS = N->getOperand(Num: 1);
4414 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4415 EVT VT = N->getValueType(ResNo: 0);
4416 SDValue LHS = N->getOperand(Num: 0);
4417 SelectionDAG &DAG = DCI.DAG;
4418 SDLoc SL(N);
4419
4420 if (VT.getScalarType() != MVT::i64)
4421 return SDValue();
4422
4423 // For C >= 32
4424 // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
4425
4426 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4427 // common case, splitting this into a move and a 32-bit shift is faster and
4428 // the same code size.
4429 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4430
4431 EVT ElementType = VT.getScalarType();
4432 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4433 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4434
4435 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4436 return SDValue();
4437
4438 SDValue ShiftFullAmt =
4439 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4440 SDValue ShiftAmt;
4441 if (CRHS) {
4442 unsigned RHSVal = CRHS->getZExtValue();
4443 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4444 VT: TargetType);
4445 } else if (Known.getMinValue().getZExtValue() ==
4446 (ElementType.getSizeInBits() - 1)) {
4447 ShiftAmt = ShiftFullAmt;
4448 } else {
4449 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4450 const SDValue ShiftMask =
4451 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4452 // This AND instruction will clamp out of bounds shift values.
4453 // It will also be removed during later instruction selection.
4454 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4455 }
4456
4457 EVT ConcatType;
4458 SDValue Hi;
4459 SDLoc LHSSL(LHS);
4460 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4461 if (VT.isVector()) {
4462 unsigned NElts = TargetType.getVectorNumElements();
4463 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4464 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4465 SmallVector<SDValue, 8> HiOps(NElts);
4466 SmallVector<SDValue, 16> HiAndLoOps;
4467
4468 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, Start: 0, Count: NElts * 2);
4469 for (unsigned I = 0; I != NElts; ++I) {
4470 HiOps[I] = HiAndLoOps[2 * I + 1];
4471 }
4472 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4473 } else {
4474 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4475 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4476 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4477 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4478 }
4479
4480 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
4481 SDValue NewShift, HiShift;
4482 if (KnownLHS.isNegative()) {
4483 HiShift = DAG.getAllOnesConstant(DL: SL, VT: TargetType);
4484 NewShift =
4485 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4486 } else if (CRHS &&
4487 CRHS->getZExtValue() == (ElementType.getSizeInBits() - 1)) {
4488 NewShift = HiShift =
4489 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4490 } else {
4491 Hi = DAG.getFreeze(V: Hi);
4492 HiShift = DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftFullAmt);
4493 NewShift =
4494 DAG.getNode(Opcode: ISD::SRA, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4495 }
4496
4497 SDValue Vec;
4498 if (VT.isVector()) {
4499 unsigned NElts = TargetType.getVectorNumElements();
4500 SmallVector<SDValue, 8> HiOps;
4501 SmallVector<SDValue, 8> LoOps;
4502 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
4503
4504 DAG.ExtractVectorElements(Op: HiShift, Args&: HiOps, Start: 0, Count: NElts);
4505 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4506 for (unsigned I = 0; I != NElts; ++I) {
4507 HiAndLoOps[2 * I + 1] = HiOps[I];
4508 HiAndLoOps[2 * I] = LoOps[I];
4509 }
4510 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4511 } else {
4512 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, HiShift});
4513 }
4514 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4515}
4516
4517SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
4518 DAGCombinerInfo &DCI) const {
4519 SDValue RHS = N->getOperand(Num: 1);
4520 ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Val&: RHS);
4521 EVT VT = N->getValueType(ResNo: 0);
4522 SDValue LHS = N->getOperand(Num: 0);
4523 SelectionDAG &DAG = DCI.DAG;
4524 SDLoc SL(N);
4525 unsigned RHSVal;
4526
4527 if (CRHS) {
4528 RHSVal = CRHS->getZExtValue();
4529
4530 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
4531 // this improves the ability to match BFE patterns in isel.
4532 if (LHS.getOpcode() == ISD::AND) {
4533 if (auto *Mask = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
4534 unsigned MaskIdx, MaskLen;
4535 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4536 MaskIdx == RHSVal) {
4537 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT,
4538 N1: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 0),
4539 N2: N->getOperand(Num: 1)),
4540 N2: DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: LHS.getOperand(i: 1),
4541 N2: N->getOperand(Num: 1)));
4542 }
4543 }
4544 }
4545 }
4546
4547 if (VT.getScalarType() != MVT::i64)
4548 return SDValue();
4549
4550 // for C >= 32
4551 // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
4552
4553 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
4554 // common case, splitting this into a move and a 32-bit shift is faster and
4555 // the same code size.
4556 KnownBits Known = DAG.computeKnownBits(Op: RHS);
4557
4558 EVT ElementType = VT.getScalarType();
4559 EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4560 EVT TargetType = VT.changeElementType(Context&: *DAG.getContext(), EltVT: TargetScalarType);
4561
4562 if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
4563 return SDValue();
4564
4565 SDValue ShiftAmt;
4566 if (CRHS) {
4567 ShiftAmt = DAG.getConstant(Val: RHSVal - TargetScalarType.getSizeInBits(), DL: SL,
4568 VT: TargetType);
4569 } else {
4570 SDValue TruncShiftAmt = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: TargetType, Operand: RHS);
4571 const SDValue ShiftMask =
4572 DAG.getConstant(Val: TargetScalarType.getSizeInBits() - 1, DL: SL, VT: TargetType);
4573 // This AND instruction will clamp out of bounds shift values.
4574 // It will also be removed during later instruction selection.
4575 ShiftAmt = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: TargetType, N1: TruncShiftAmt, N2: ShiftMask);
4576 }
4577
4578 const SDValue Zero = DAG.getConstant(Val: 0, DL: SL, VT: TargetScalarType);
4579 EVT ConcatType;
4580 SDValue Hi;
4581 SDLoc LHSSL(LHS);
4582 // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
4583 if (VT.isVector()) {
4584 unsigned NElts = TargetType.getVectorNumElements();
4585 ConcatType = TargetType.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
4586 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4587 SmallVector<SDValue, 8> HiOps(NElts);
4588 SmallVector<SDValue, 16> HiAndLoOps;
4589
4590 DAG.ExtractVectorElements(Op: SplitLHS, Args&: HiAndLoOps, /*Start=*/0, Count: NElts * 2);
4591 for (unsigned I = 0; I != NElts; ++I)
4592 HiOps[I] = HiAndLoOps[2 * I + 1];
4593 Hi = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: LHSSL, VT: TargetType, Ops: HiOps);
4594 } else {
4595 const SDValue One = DAG.getConstant(Val: 1, DL: LHSSL, VT: TargetScalarType);
4596 ConcatType = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TargetType, NumElements: 2);
4597 SDValue SplitLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: LHSSL, VT: ConcatType, Operand: LHS);
4598 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: LHSSL, VT: TargetType, N1: SplitLHS, N2: One);
4599 }
4600
4601 SDValue NewShift =
4602 DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: TargetType, N1: Hi, N2: ShiftAmt, Flags: N->getFlags());
4603
4604 SDValue Vec;
4605 if (VT.isVector()) {
4606 unsigned NElts = TargetType.getVectorNumElements();
4607 SmallVector<SDValue, 8> LoOps;
4608 SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
4609
4610 DAG.ExtractVectorElements(Op: NewShift, Args&: LoOps, Start: 0, Count: NElts);
4611 for (unsigned I = 0; I != NElts; ++I)
4612 HiAndLoOps[2 * I] = LoOps[I];
4613 Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: ConcatType, Ops: HiAndLoOps);
4614 } else {
4615 Vec = DAG.getBuildVector(VT: ConcatType, DL: SL, Ops: {NewShift, Zero});
4616 }
4617 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Vec);
4618}
4619
4620SDValue AMDGPUTargetLowering::performTruncateCombine(
4621 SDNode *N, DAGCombinerInfo &DCI) const {
4622 SDLoc SL(N);
4623 SelectionDAG &DAG = DCI.DAG;
4624 EVT VT = N->getValueType(ResNo: 0);
4625 SDValue Src = N->getOperand(Num: 0);
4626
4627 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
4628 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
4629 SDValue Vec = Src.getOperand(i: 0);
4630 if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
4631 SDValue Elt0 = Vec.getOperand(i: 0);
4632 EVT EltVT = Elt0.getValueType();
4633 if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
4634 if (EltVT.isFloatingPoint()) {
4635 Elt0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SL,
4636 VT: EltVT.changeTypeToInteger(), Operand: Elt0);
4637 }
4638
4639 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Elt0);
4640 }
4641 }
4642 }
4643
4644 // Equivalent of above for accessing the high element of a vector as an
4645 // integer operation.
4646 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
4647 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
4648 if (auto *K = isConstOrConstSplat(N: Src.getOperand(i: 1))) {
4649 SDValue BV = stripBitcast(Val: Src.getOperand(i: 0));
4650 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
4651 EVT SrcEltVT = BV.getOperand(i: 0).getValueType();
4652 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
4653 unsigned BitIndex = K->getZExtValue();
4654 unsigned PartIndex = BitIndex / SrcEltSize;
4655
4656 if (PartIndex * SrcEltSize == BitIndex &&
4657 PartIndex < BV.getNumOperands()) {
4658 if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
4659 SDValue SrcElt =
4660 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: SrcEltVT.changeTypeToInteger(),
4661 Operand: BV.getOperand(i: PartIndex));
4662 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: SrcElt);
4663 }
4664 }
4665 }
4666 }
4667 }
4668
4669 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
4670 //
4671 // i16 (trunc (srl i64:x, K)), K <= 16 ->
4672 // i16 (trunc (srl (i32 (trunc x), K)))
4673 if (VT.getScalarSizeInBits() < 32) {
4674 EVT SrcVT = Src.getValueType();
4675 if (SrcVT.getScalarSizeInBits() > 32 &&
4676 (Src.getOpcode() == ISD::SRL ||
4677 Src.getOpcode() == ISD::SRA ||
4678 Src.getOpcode() == ISD::SHL)) {
4679 SDValue Amt = Src.getOperand(i: 1);
4680 KnownBits Known = DAG.computeKnownBits(Op: Amt);
4681
4682 // - For left shifts, do the transform as long as the shift
4683 // amount is still legal for i32, so when ShiftAmt < 32 (<= 31)
4684 // - For right shift, do it if ShiftAmt <= (32 - Size) to avoid
4685 // losing information stored in the high bits when truncating.
4686 const unsigned MaxCstSize =
4687 (Src.getOpcode() == ISD::SHL) ? 31 : (32 - VT.getScalarSizeInBits());
4688 if (Known.getMaxValue().ule(RHS: MaxCstSize)) {
4689 EVT MidVT = VT.isVector() ?
4690 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i32,
4691 NumElements: VT.getVectorNumElements()) : MVT::i32;
4692
4693 EVT NewShiftVT = getShiftAmountTy(LHSTy: MidVT, DL: DAG.getDataLayout());
4694 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MidVT,
4695 Operand: Src.getOperand(i: 0));
4696 DCI.AddToWorklist(N: Trunc.getNode());
4697
4698 if (Amt.getValueType() != NewShiftVT) {
4699 Amt = DAG.getZExtOrTrunc(Op: Amt, DL: SL, VT: NewShiftVT);
4700 DCI.AddToWorklist(N: Amt.getNode());
4701 }
4702
4703 SDValue ShrunkShift = DAG.getNode(Opcode: Src.getOpcode(), DL: SL, VT: MidVT,
4704 N1: Trunc, N2: Amt);
4705 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: ShrunkShift);
4706 }
4707 }
4708 }
4709
4710 return SDValue();
4711}
4712
4713// We need to specifically handle i64 mul here to avoid unnecessary conversion
4714// instructions. If we only match on the legalized i64 mul expansion,
4715// SimplifyDemandedBits will be unable to remove them because there will be
4716// multiple uses due to the separate mul + mulh[su].
4717static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
4718 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
4719 if (Size <= 32) {
4720 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4721 return DAG.getNode(Opcode: MulOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4722 }
4723
4724 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4725 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
4726
4727 SDValue MulLo = DAG.getNode(Opcode: MulLoOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4728 SDValue MulHi = DAG.getNode(Opcode: MulHiOpc, DL: SL, VT: MVT::i32, N1: N0, N2: N1);
4729
4730 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SL, VT: MVT::i64, N1: MulLo, N2: MulHi);
4731}
4732
4733/// If \p V is an add of a constant 1, returns the other operand. Otherwise
4734/// return SDValue().
4735static SDValue getAddOneOp(const SDNode *V) {
4736 if (V->getOpcode() != ISD::ADD)
4737 return SDValue();
4738
4739 return isOneConstant(V: V->getOperand(Num: 1)) ? V->getOperand(Num: 0) : SDValue();
4740}
4741
4742SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
4743 DAGCombinerInfo &DCI) const {
4744 assert(N->getOpcode() == ISD::MUL);
4745 EVT VT = N->getValueType(ResNo: 0);
4746
4747 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4748 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4749 // unnecessarily). isDivergent() is used as an approximation of whether the
4750 // value is in an SGPR.
4751 if (!N->isDivergent())
4752 return SDValue();
4753
4754 unsigned Size = VT.getSizeInBits();
4755 if (VT.isVector() || Size > 64)
4756 return SDValue();
4757
4758 SelectionDAG &DAG = DCI.DAG;
4759 SDLoc DL(N);
4760
4761 SDValue N0 = N->getOperand(Num: 0);
4762 SDValue N1 = N->getOperand(Num: 1);
4763
4764 // Undo InstCombine canonicalize X * (Y + 1) -> X * Y + X to enable mad
4765 // matching.
4766
4767 // mul x, (add y, 1) -> add (mul x, y), x
4768 auto IsFoldableAdd = [](SDValue V) -> SDValue {
4769 SDValue AddOp = getAddOneOp(V: V.getNode());
4770 if (!AddOp)
4771 return SDValue();
4772
4773 if (V.hasOneUse() || all_of(Range: V->users(), P: [](const SDNode *U) -> bool {
4774 return U->getOpcode() == ISD::MUL;
4775 }))
4776 return AddOp;
4777
4778 return SDValue();
4779 };
4780
4781 // FIXME: The selection pattern is not properly checking for commuted
4782 // operands, so we have to place the mul in the LHS
4783 if (SDValue MulOper = IsFoldableAdd(N0)) {
4784 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1, N2: MulOper);
4785 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N1);
4786 }
4787
4788 if (SDValue MulOper = IsFoldableAdd(N1)) {
4789 SDValue MulVal = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N0, N2: MulOper);
4790 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: MulVal, N2: N0);
4791 }
4792
4793 // There are i16 integer mul/mad.
4794 if (isTypeLegal(VT: MVT::i16) && VT.getScalarType().bitsLE(VT: MVT::i16))
4795 return SDValue();
4796
4797 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4798 // in the source into any_extends if the result of the mul is truncated. Since
4799 // we can assume the high bits are whatever we want, use the underlying value
4800 // to avoid the unknown high bits from interfering.
4801 if (N0.getOpcode() == ISD::ANY_EXTEND)
4802 N0 = N0.getOperand(i: 0);
4803
4804 if (N1.getOpcode() == ISD::ANY_EXTEND)
4805 N1 = N1.getOperand(i: 0);
4806
4807 SDValue Mul;
4808
4809 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4810 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4811 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4812 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: false);
4813 } else if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4814 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4815 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4816 Mul = getMul24(DAG, SL: DL, N0, N1, Size, Signed: true);
4817 } else {
4818 return SDValue();
4819 }
4820
4821 // We need to use sext even for MUL_U24, because MUL_U24 is used
4822 // for signed multiply of 8 and 16-bit types.
4823 return DAG.getSExtOrTrunc(Op: Mul, DL, VT);
4824}
4825
4826SDValue
4827AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N,
4828 DAGCombinerInfo &DCI) const {
4829 if (N->getValueType(ResNo: 0) != MVT::i32)
4830 return SDValue();
4831
4832 SelectionDAG &DAG = DCI.DAG;
4833 SDLoc DL(N);
4834
4835 bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
4836 SDValue N0 = N->getOperand(Num: 0);
4837 SDValue N1 = N->getOperand(Num: 1);
4838
4839 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
4840 // in the source into any_extends if the result of the mul is truncated. Since
4841 // we can assume the high bits are whatever we want, use the underlying value
4842 // to avoid the unknown high bits from interfering.
4843 if (N0.getOpcode() == ISD::ANY_EXTEND)
4844 N0 = N0.getOperand(i: 0);
4845 if (N1.getOpcode() == ISD::ANY_EXTEND)
4846 N1 = N1.getOperand(i: 0);
4847
4848 // Try to use two fast 24-bit multiplies (one for each half of the result)
4849 // instead of one slow extending multiply.
4850 unsigned LoOpcode = 0;
4851 unsigned HiOpcode = 0;
4852 if (Signed) {
4853 if (Subtarget->hasMulI24() && isI24(Op: N0, DAG) && isI24(Op: N1, DAG)) {
4854 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4855 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4856 LoOpcode = AMDGPUISD::MUL_I24;
4857 HiOpcode = AMDGPUISD::MULHI_I24;
4858 }
4859 } else {
4860 if (Subtarget->hasMulU24() && isU24(Op: N0, DAG) && isU24(Op: N1, DAG)) {
4861 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4862 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4863 LoOpcode = AMDGPUISD::MUL_U24;
4864 HiOpcode = AMDGPUISD::MULHI_U24;
4865 }
4866 }
4867 if (!LoOpcode)
4868 return SDValue();
4869
4870 SDValue Lo = DAG.getNode(Opcode: LoOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4871 SDValue Hi = DAG.getNode(Opcode: HiOpcode, DL, VT: MVT::i32, N1: N0, N2: N1);
4872 DCI.CombineTo(N, Res0: Lo, Res1: Hi);
4873 return SDValue(N, 0);
4874}
4875
4876SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
4877 DAGCombinerInfo &DCI) const {
4878 EVT VT = N->getValueType(ResNo: 0);
4879
4880 if (!Subtarget->hasMulI24() || VT.isVector())
4881 return SDValue();
4882
4883 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4884 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4885 // unnecessarily). isDivergent() is used as an approximation of whether the
4886 // value is in an SGPR.
4887 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4888 // valu op anyway)
4889 if (Subtarget->hasSMulHi() && !N->isDivergent())
4890 return SDValue();
4891
4892 SelectionDAG &DAG = DCI.DAG;
4893 SDLoc DL(N);
4894
4895 SDValue N0 = N->getOperand(Num: 0);
4896 SDValue N1 = N->getOperand(Num: 1);
4897
4898 if (!isI24(Op: N0, DAG) || !isI24(Op: N1, DAG))
4899 return SDValue();
4900
4901 N0 = DAG.getSExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4902 N1 = DAG.getSExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4903
4904 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_I24, DL, VT: MVT::i32, N1: N0, N2: N1);
4905 DCI.AddToWorklist(N: Mulhi.getNode());
4906 return DAG.getSExtOrTrunc(Op: Mulhi, DL, VT);
4907}
4908
4909SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
4910 DAGCombinerInfo &DCI) const {
4911 EVT VT = N->getValueType(ResNo: 0);
4912
4913 if (VT.isVector() || VT.getSizeInBits() > 32 || !Subtarget->hasMulU24())
4914 return SDValue();
4915
4916 // Don't generate 24-bit multiplies on values that are in SGPRs, since
4917 // we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
4918 // unnecessarily). isDivergent() is used as an approximation of whether the
4919 // value is in an SGPR.
4920 // This doesn't apply if no s_mul_hi is available (since we'll end up with a
4921 // valu op anyway)
4922 if (!N->isDivergent() && Subtarget->hasSMulHi())
4923 return SDValue();
4924
4925 SelectionDAG &DAG = DCI.DAG;
4926 SDLoc DL(N);
4927
4928 SDValue N0 = N->getOperand(Num: 0);
4929 SDValue N1 = N->getOperand(Num: 1);
4930
4931 if (!isU24(Op: N0, DAG) || !isU24(Op: N1, DAG))
4932 return SDValue();
4933
4934 N0 = DAG.getZExtOrTrunc(Op: N0, DL, VT: MVT::i32);
4935 N1 = DAG.getZExtOrTrunc(Op: N1, DL, VT: MVT::i32);
4936
4937 SDValue Mulhi = DAG.getNode(Opcode: AMDGPUISD::MULHI_U24, DL, VT: MVT::i32, N1: N0, N2: N1);
4938 DCI.AddToWorklist(N: Mulhi.getNode());
4939 return DAG.getZExtOrTrunc(Op: Mulhi, DL, VT);
4940}
4941
4942SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
4943 SDValue Op,
4944 const SDLoc &DL,
4945 unsigned Opc) const {
4946 EVT VT = Op.getValueType();
4947 if (VT.bitsGT(VT: MVT::i32))
4948 return SDValue();
4949
4950 if (VT != MVT::i32)
4951 Op = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: Op);
4952
4953 SDValue FFBX = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, Operand: Op);
4954 if (VT != MVT::i32)
4955 FFBX = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FFBX);
4956
4957 return FFBX;
4958}
4959
4960// The native instructions return -1 on 0 input. Optimize out a select that
4961// produces -1 on 0.
4962//
4963// TODO: If zero is not undef, we could also do this if the output is compared
4964// against the bitwidth.
4965//
4966// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
4967SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
4968 SDValue LHS, SDValue RHS,
4969 DAGCombinerInfo &DCI) const {
4970 if (!isNullConstant(V: Cond.getOperand(i: 1)))
4971 return SDValue();
4972
4973 SelectionDAG &DAG = DCI.DAG;
4974 ISD::CondCode CCOpcode = Cond.getOperandCondCode(i: 2);
4975 SDValue CmpLHS = Cond.getOperand(i: 0);
4976
4977 // select (setcc x, 0, eq), -1, (ctlz_zero_poison x) -> ffbh_u32 x
4978 // select (setcc x, 0, eq), -1, (cttz_zero_poison x) -> ffbl_u32 x
4979 if (CCOpcode == ISD::SETEQ &&
4980 (isCtlzOpc(Opc: RHS.getOpcode()) || isCttzOpc(Opc: RHS.getOpcode())) &&
4981 RHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: LHS)) {
4982 unsigned Opc =
4983 isCttzOpc(Opc: RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4984 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4985 }
4986
4987 // select (setcc x, 0, ne), (ctlz_zero_poison x), -1 -> ffbh_u32 x
4988 // select (setcc x, 0, ne), (cttz_zero_poison x), -1 -> ffbl_u32 x
4989 if (CCOpcode == ISD::SETNE &&
4990 (isCtlzOpc(Opc: LHS.getOpcode()) || isCttzOpc(Opc: LHS.getOpcode())) &&
4991 LHS.getOperand(i: 0) == CmpLHS && isAllOnesConstant(V: RHS)) {
4992 unsigned Opc =
4993 isCttzOpc(Opc: LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4994
4995 return getFFBX_U32(DAG, Op: CmpLHS, DL: SL, Opc);
4996 }
4997
4998 return SDValue();
4999}
5000
5001static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
5002 unsigned Op,
5003 const SDLoc &SL,
5004 SDValue Cond,
5005 SDValue N1,
5006 SDValue N2) {
5007 SelectionDAG &DAG = DCI.DAG;
5008 EVT VT = N1.getValueType();
5009
5010 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: Cond,
5011 N2: N1.getOperand(i: 0), N3: N2.getOperand(i: 0));
5012 DCI.AddToWorklist(N: NewSelect.getNode());
5013 return DAG.getNode(Opcode: Op, DL: SL, VT, Operand: NewSelect);
5014}
5015
5016// Pull a free FP operation out of a select so it may fold into uses.
5017//
5018// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
5019// select c, (fneg x), k -> fneg (select c, x, (fneg k))
5020//
5021// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
5022// select c, (fabs x), +k -> fabs (select c, x, k)
5023SDValue
5024AMDGPUTargetLowering::foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
5025 SDValue N) const {
5026 SelectionDAG &DAG = DCI.DAG;
5027 SDValue Cond = N.getOperand(i: 0);
5028 SDValue LHS = N.getOperand(i: 1);
5029 SDValue RHS = N.getOperand(i: 2);
5030
5031 EVT VT = N.getValueType();
5032 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
5033 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
5034 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5035 return SDValue();
5036
5037 return distributeOpThroughSelect(DCI, Op: LHS.getOpcode(),
5038 SL: SDLoc(N), Cond, N1: LHS, N2: RHS);
5039 }
5040
5041 bool Inv = false;
5042 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
5043 std::swap(a&: LHS, b&: RHS);
5044 Inv = true;
5045 }
5046
5047 // TODO: Support vector constants.
5048 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(Val&: RHS);
5049 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS &&
5050 !selectSupportsSourceMods(N: N.getNode())) {
5051 SDLoc SL(N);
5052 // If one side is an fneg/fabs and the other is a constant, we can push the
5053 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
5054 SDValue NewLHS = LHS.getOperand(i: 0);
5055 SDValue NewRHS = RHS;
5056
5057 // Careful: if the neg can be folded up, don't try to pull it back down.
5058 bool ShouldFoldNeg = true;
5059
5060 if (NewLHS.hasOneUse()) {
5061 unsigned Opc = NewLHS.getOpcode();
5062 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(N: NewLHS.getNode()))
5063 ShouldFoldNeg = false;
5064 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
5065 ShouldFoldNeg = false;
5066 }
5067
5068 if (ShouldFoldNeg) {
5069 if (LHS.getOpcode() == ISD::FABS && CRHS->isNegative())
5070 return SDValue();
5071
5072 // We're going to be forced to use a source modifier anyway, there's no
5073 // point to pulling the negate out unless we can get a size reduction by
5074 // negating the constant.
5075 //
5076 // TODO: Generalize to use getCheaperNegatedExpression which doesn't know
5077 // about cheaper constants.
5078 if (NewLHS.getOpcode() == ISD::FABS &&
5079 getConstantNegateCost(C: CRHS) != NegatibleCost::Cheaper)
5080 return SDValue();
5081
5082 if (!AMDGPUTargetLowering::allUsesHaveSourceMods(N: N.getNode()))
5083 return SDValue();
5084
5085 if (LHS.getOpcode() == ISD::FNEG)
5086 NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5087
5088 if (Inv)
5089 std::swap(a&: NewLHS, b&: NewRHS);
5090
5091 SDValue NewSelect = DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT,
5092 N1: Cond, N2: NewLHS, N3: NewRHS);
5093 DCI.AddToWorklist(N: NewSelect.getNode());
5094 return DAG.getNode(Opcode: LHS.getOpcode(), DL: SL, VT, Operand: NewSelect);
5095 }
5096 }
5097
5098 return SDValue();
5099}
5100
5101SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
5102 DAGCombinerInfo &DCI) const {
5103 if (SDValue Folded = foldFreeOpFromSelect(DCI, N: SDValue(N, 0)))
5104 return Folded;
5105
5106 SDValue Cond = N->getOperand(Num: 0);
5107 if (Cond.getOpcode() != ISD::SETCC)
5108 return SDValue();
5109
5110 EVT VT = N->getValueType(ResNo: 0);
5111 SDValue LHS = Cond.getOperand(i: 0);
5112 SDValue RHS = Cond.getOperand(i: 1);
5113 SDValue CC = Cond.getOperand(i: 2);
5114
5115 SDValue True = N->getOperand(Num: 1);
5116 SDValue False = N->getOperand(Num: 2);
5117
5118 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
5119 SelectionDAG &DAG = DCI.DAG;
5120 if (DAG.isConstantValueOfAnyType(N: True) &&
5121 !DAG.isConstantValueOfAnyType(N: False)) {
5122 // Swap cmp + select pair to move constant to false input.
5123 // This will allow using VOPC cndmasks more often.
5124 // select (setcc x, y), k, x -> select (setccinv x, y), x, k
5125
5126 SDLoc SL(N);
5127 ISD::CondCode NewCC =
5128 getSetCCInverse(Operation: cast<CondCodeSDNode>(Val&: CC)->get(), Type: LHS.getValueType());
5129
5130 SDValue NewCond = DAG.getSetCC(DL: SL, VT: Cond.getValueType(), LHS, RHS, Cond: NewCC);
5131 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT, N1: NewCond, N2: False, N3: True);
5132 }
5133
5134 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
5135 SDValue MinMax = combineFMinMaxLegacy(DL: SDLoc(N), VT, LHS, RHS, True, False,
5136 CC, Flags: N->getFlags(), DCI);
5137 // Revisit this node so we can catch min3/max3/med3 patterns.
5138 //DCI.AddToWorklist(MinMax.getNode());
5139 return MinMax;
5140 }
5141 }
5142
5143 // There's no reason to not do this if the condition has other uses.
5144 return performCtlz_CttzCombine(SL: SDLoc(N), Cond, LHS: True, RHS: False, DCI);
5145}
5146
5147static bool isInv2Pi(const APFloat &APF) {
5148 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
5149 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
5150 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
5151
5152 return APF.bitwiseIsEqual(RHS: KF16) ||
5153 APF.bitwiseIsEqual(RHS: KF32) ||
5154 APF.bitwiseIsEqual(RHS: KF64);
5155}
5156
5157// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
5158// additional cost to negate them.
5159TargetLowering::NegatibleCost
5160AMDGPUTargetLowering::getConstantNegateCost(const ConstantFPSDNode *C) const {
5161 if (C->isZero())
5162 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5163
5164 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(APF: C->getValueAPF()))
5165 return C->isNegative() ? NegatibleCost::Cheaper : NegatibleCost::Expensive;
5166
5167 return NegatibleCost::Neutral;
5168}
5169
5170bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
5171 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N))
5172 return getConstantNegateCost(C) == NegatibleCost::Expensive;
5173 return false;
5174}
5175
5176static unsigned inverseMinMax(unsigned Opc) {
5177 switch (Opc) {
5178 case ISD::FMAXNUM:
5179 return ISD::FMINNUM;
5180 case ISD::FMINNUM:
5181 return ISD::FMAXNUM;
5182 case ISD::FMAXNUM_IEEE:
5183 return ISD::FMINNUM_IEEE;
5184 case ISD::FMINNUM_IEEE:
5185 return ISD::FMAXNUM_IEEE;
5186 case ISD::FMAXIMUM:
5187 return ISD::FMINIMUM;
5188 case ISD::FMINIMUM:
5189 return ISD::FMAXIMUM;
5190 case ISD::FMAXIMUMNUM:
5191 return ISD::FMINIMUMNUM;
5192 case ISD::FMINIMUMNUM:
5193 return ISD::FMAXIMUMNUM;
5194 case AMDGPUISD::FMAX_LEGACY:
5195 return AMDGPUISD::FMIN_LEGACY;
5196 case AMDGPUISD::FMIN_LEGACY:
5197 return AMDGPUISD::FMAX_LEGACY;
5198 default:
5199 llvm_unreachable("invalid min/max opcode");
5200 }
5201}
5202
5203/// \return true if it's profitable to try to push an fneg into its source
5204/// instruction.
5205bool AMDGPUTargetLowering::shouldFoldFNegIntoSrc(SDNode *N, SDValue N0) {
5206 // If the input has multiple uses and we can either fold the negate down, or
5207 // the other uses cannot, give up. This both prevents unprofitable
5208 // transformations and infinite loops: we won't repeatedly try to fold around
5209 // a negate that has no 'good' form.
5210 if (N0.hasOneUse()) {
5211 // This may be able to fold into the source, but at a code size cost. Don't
5212 // fold if the fold into the user is free.
5213 if (allUsesHaveSourceMods(N, CostThreshold: 0))
5214 return false;
5215 } else {
5216 if (fnegFoldsIntoOp(N: N0.getNode()) &&
5217 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N: N0.getNode())))
5218 return false;
5219 }
5220
5221 return true;
5222}
5223
5224SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
5225 DAGCombinerInfo &DCI) const {
5226 SelectionDAG &DAG = DCI.DAG;
5227 SDValue N0 = N->getOperand(Num: 0);
5228 EVT VT = N->getValueType(ResNo: 0);
5229
5230 unsigned Opc = N0.getOpcode();
5231
5232 if (!shouldFoldFNegIntoSrc(N, N0))
5233 return SDValue();
5234
5235 bool MayIgnoreSignedZeroForAllUses =
5236 N0->getFlags().hasNoSignedZeros() ||
5237 (N0.hasOneUse() && N->getFlags().hasNoSignedZeros());
5238
5239 SDLoc SL(N);
5240 switch (Opc) {
5241 case ISD::FADD: {
5242 if (!MayIgnoreSignedZeroForAllUses)
5243 return SDValue();
5244
5245 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
5246 SDValue LHS = N0.getOperand(i: 0);
5247 SDValue RHS = N0.getOperand(i: 1);
5248
5249 if (LHS.getOpcode() != ISD::FNEG)
5250 LHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5251 else
5252 LHS = LHS.getOperand(i: 0);
5253
5254 if (RHS.getOpcode() != ISD::FNEG)
5255 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5256 else
5257 RHS = RHS.getOperand(i: 0);
5258
5259 SDValue Res = DAG.getNode(Opcode: ISD::FADD, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5260 if (Res.getOpcode() != ISD::FADD)
5261 return SDValue(); // Op got folded away.
5262 if (!N0.hasOneUse())
5263 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5264 return Res;
5265 }
5266 case ISD::FMUL:
5267 case AMDGPUISD::FMUL_LEGACY: {
5268 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
5269 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
5270 SDValue LHS = N0.getOperand(i: 0);
5271 SDValue RHS = N0.getOperand(i: 1);
5272
5273 if (LHS.getOpcode() == ISD::FNEG)
5274 LHS = LHS.getOperand(i: 0);
5275 else if (RHS.getOpcode() == ISD::FNEG)
5276 RHS = RHS.getOperand(i: 0);
5277 else
5278 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5279
5280 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: RHS, Flags: N0->getFlags());
5281 if (Res.getOpcode() != Opc)
5282 return SDValue(); // Op got folded away.
5283 if (!N0.hasOneUse())
5284 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5285 return Res;
5286 }
5287 case ISD::FMA:
5288 case ISD::FMAD: {
5289 // TODO: handle llvm.amdgcn.fma.legacy
5290 if (!MayIgnoreSignedZeroForAllUses)
5291 return SDValue();
5292
5293 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
5294 SDValue LHS = N0.getOperand(i: 0);
5295 SDValue MHS = N0.getOperand(i: 1);
5296 SDValue RHS = N0.getOperand(i: 2);
5297
5298 if (LHS.getOpcode() == ISD::FNEG)
5299 LHS = LHS.getOperand(i: 0);
5300 else if (MHS.getOpcode() == ISD::FNEG)
5301 MHS = MHS.getOperand(i: 0);
5302 else
5303 MHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: MHS);
5304
5305 if (RHS.getOpcode() != ISD::FNEG)
5306 RHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5307 else
5308 RHS = RHS.getOperand(i: 0);
5309
5310 SDValue Res = DAG.getNode(Opcode: Opc, DL: SL, VT, N1: LHS, N2: MHS, N3: RHS);
5311 if (Res.getOpcode() != Opc)
5312 return SDValue(); // Op got folded away.
5313 if (!N0.hasOneUse())
5314 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5315 return Res;
5316 }
5317 case ISD::FMAXNUM:
5318 case ISD::FMINNUM:
5319 case ISD::FMAXNUM_IEEE:
5320 case ISD::FMINNUM_IEEE:
5321 case ISD::FMINIMUM:
5322 case ISD::FMAXIMUM:
5323 case ISD::FMINIMUMNUM:
5324 case ISD::FMAXIMUMNUM:
5325 case AMDGPUISD::FMAX_LEGACY:
5326 case AMDGPUISD::FMIN_LEGACY: {
5327 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
5328 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
5329 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
5330 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
5331
5332 SDValue LHS = N0.getOperand(i: 0);
5333 SDValue RHS = N0.getOperand(i: 1);
5334
5335 // 0 doesn't have a negated inline immediate.
5336 // TODO: This constant check should be generalized to other operations.
5337 if (isConstantCostlierToNegate(N: RHS))
5338 return SDValue();
5339
5340 // Swapping min<->max flips which operand a signed zero tie selects.
5341 if ((Opc == AMDGPUISD::FMIN_LEGACY || Opc == AMDGPUISD::FMAX_LEGACY) &&
5342 !canIgnoreLegacyMinMaxTies(DAG, Flags: N0->getFlags(), LHS, RHS))
5343 return SDValue();
5344
5345 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: LHS);
5346 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: RHS);
5347 unsigned Opposite = inverseMinMax(Opc);
5348
5349 SDValue Res = DAG.getNode(Opcode: Opposite, DL: SL, VT, N1: NegLHS, N2: NegRHS, Flags: N0->getFlags());
5350 if (Res.getOpcode() != Opposite)
5351 return SDValue(); // Op got folded away.
5352 if (!N0.hasOneUse())
5353 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res));
5354 return Res;
5355 }
5356 case AMDGPUISD::FMED3: {
5357 // med3 sorts a NaN input as smaller than everything regardless of its sign,
5358 // so negating all operands does not sign-flip the median when an input may
5359 // be NaN.
5360 if (!N0->getFlags().hasNoNaNs())
5361 return SDValue();
5362
5363 SDValue Ops[3];
5364 for (unsigned I = 0; I < 3; ++I)
5365 Ops[I] = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N0->getOperand(Num: I), Flags: N0->getFlags());
5366
5367 SDValue Res = DAG.getNode(Opcode: AMDGPUISD::FMED3, DL: SL, VT, Ops, Flags: N0->getFlags());
5368 if (Res.getOpcode() != AMDGPUISD::FMED3)
5369 return SDValue(); // Op got folded away.
5370
5371 if (!N0.hasOneUse()) {
5372 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Res);
5373 DAG.ReplaceAllUsesWith(From: N0, To: Neg);
5374
5375 for (SDNode *U : Neg->users())
5376 DCI.AddToWorklist(N: U);
5377 }
5378
5379 return Res;
5380 }
5381 case ISD::FP_EXTEND:
5382 case ISD::FTRUNC:
5383 case ISD::FRINT:
5384 case ISD::FNEARBYINT: // XXX - Should fround be handled?
5385 case ISD::FROUNDEVEN:
5386 case ISD::FSIN:
5387 case ISD::FCANONICALIZE:
5388 case AMDGPUISD::RCP:
5389 case AMDGPUISD::RCP_LEGACY:
5390 case AMDGPUISD::RCP_IFLAG:
5391 case AMDGPUISD::SIN_HW: {
5392 SDValue CvtSrc = N0.getOperand(i: 0);
5393 if (CvtSrc.getOpcode() == ISD::FNEG) {
5394 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
5395 // (fneg (rcp (fneg x))) -> (rcp x)
5396 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: CvtSrc.getOperand(i: 0));
5397 }
5398
5399 if (!N0.hasOneUse())
5400 return SDValue();
5401
5402 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
5403 // (fneg (rcp x)) -> (rcp (fneg x))
5404 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5405 return DAG.getNode(Opcode: Opc, DL: SL, VT, Operand: Neg, Flags: N0->getFlags());
5406 }
5407 case ISD::FP_ROUND: {
5408 SDValue CvtSrc = N0.getOperand(i: 0);
5409
5410 if (CvtSrc.getOpcode() == ISD::FNEG) {
5411 // (fneg (fp_round (fneg x))) -> (fp_round x)
5412 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT,
5413 N1: CvtSrc.getOperand(i: 0), N2: N0.getOperand(i: 1));
5414 }
5415
5416 if (!N0.hasOneUse())
5417 return SDValue();
5418
5419 // (fneg (fp_round x)) -> (fp_round (fneg x))
5420 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc);
5421 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT, N1: Neg, N2: N0.getOperand(i: 1));
5422 }
5423 case ISD::FP16_TO_FP: {
5424 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
5425 // f16, but legalization of f16 fneg ends up pulling it out of the source.
5426 // Put the fneg back as a legal source operation that can be matched later.
5427 SDLoc SL(N);
5428
5429 SDValue Src = N0.getOperand(i: 0);
5430 EVT SrcVT = Src.getValueType();
5431
5432 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
5433 SDValue IntFNeg = DAG.getNode(Opcode: ISD::XOR, DL: SL, VT: SrcVT, N1: Src,
5434 N2: DAG.getConstant(Val: 0x8000, DL: SL, VT: SrcVT));
5435 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFNeg);
5436 }
5437 case ISD::SELECT: {
5438 // fneg (select c, a, b) -> select c, (fneg a), (fneg b)
5439 // TODO: Invert conditions of foldFreeOpFromSelect
5440 return SDValue();
5441 }
5442 case ISD::BITCAST: {
5443 SDLoc SL(N);
5444 SDValue BCSrc = N0.getOperand(i: 0);
5445 if (BCSrc.getOpcode() == ISD::BUILD_VECTOR) {
5446 SDValue HighBits = BCSrc.getOperand(i: BCSrc.getNumOperands() - 1);
5447 if (VT != MVT::f64 || HighBits.getValueType().getSizeInBits() != 32 ||
5448 !fnegFoldsIntoOp(N: HighBits.getNode()))
5449 return SDValue();
5450
5451 // f64 fneg only really needs to operate on the high half of of the
5452 // register, so try to force it to an f32 operation to help make use of
5453 // source modifiers.
5454 //
5455 //
5456 // fneg (f64 (bitcast (build_vector x, y))) ->
5457 // f64 (bitcast (build_vector (bitcast i32:x to f32),
5458 // (fneg (bitcast i32:y to f32)))
5459
5460 SDValue CastHi = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: HighBits);
5461 SDValue NegHi = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: CastHi);
5462 SDValue CastBack =
5463 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: HighBits.getValueType(), Operand: NegHi);
5464
5465 SmallVector<SDValue, 8> Ops(BCSrc->ops());
5466 Ops.back() = CastBack;
5467 DCI.AddToWorklist(N: NegHi.getNode());
5468 SDValue Build =
5469 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: BCSrc.getValueType(), Ops);
5470 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT, Operand: Build);
5471
5472 if (!N0.hasOneUse())
5473 DAG.ReplaceAllUsesWith(From: N0, To: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Result));
5474 return Result;
5475 }
5476
5477 if (BCSrc.getOpcode() == ISD::SELECT && VT == MVT::f32 &&
5478 BCSrc.hasOneUse()) {
5479 // fneg (bitcast (f32 (select cond, i32:lhs, i32:rhs))) ->
5480 // select cond, (bitcast i32:lhs to f32), (bitcast i32:rhs to f32)
5481
5482 // TODO: Cast back result for multiple uses is beneficial in some cases.
5483
5484 SDValue LHS =
5485 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 1));
5486 SDValue RHS =
5487 DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MVT::f32, Operand: BCSrc.getOperand(i: 2));
5488
5489 SDValue NegLHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: LHS);
5490 SDValue NegRHS = DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT: MVT::f32, Operand: RHS);
5491
5492 return DAG.getNode(Opcode: ISD::SELECT, DL: SL, VT: MVT::f32, N1: BCSrc.getOperand(i: 0), N2: NegLHS,
5493 N3: NegRHS);
5494 }
5495
5496 return SDValue();
5497 }
5498 default:
5499 return SDValue();
5500 }
5501}
5502
5503SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
5504 DAGCombinerInfo &DCI) const {
5505 SelectionDAG &DAG = DCI.DAG;
5506 SDValue N0 = N->getOperand(Num: 0);
5507
5508 if (!N0.hasOneUse())
5509 return SDValue();
5510
5511 switch (N0.getOpcode()) {
5512 case ISD::FP16_TO_FP: {
5513 assert(!isTypeLegal(MVT::f16) && "should only see if f16 is illegal");
5514 SDLoc SL(N);
5515 SDValue Src = N0.getOperand(i: 0);
5516 EVT SrcVT = Src.getValueType();
5517
5518 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
5519 SDValue IntFAbs = DAG.getNode(Opcode: ISD::AND, DL: SL, VT: SrcVT, N1: Src,
5520 N2: DAG.getConstant(Val: 0x7fff, DL: SL, VT: SrcVT));
5521 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: SL, VT: N->getValueType(ResNo: 0), Operand: IntFAbs);
5522 }
5523 case ISD::FP_ROUND: {
5524 SDLoc SL(N);
5525 SDValue CvtSrc = N0.getOperand(i: 0);
5526
5527 // fabs (fp_round x) -> fp_round (fabs x)
5528 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: SL, VT: CvtSrc.getValueType(), Operand: CvtSrc,
5529 Flags: N->getFlags());
5530 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SL, VT: N->getValueType(ResNo: 0), N1: Abs,
5531 N2: N0.getOperand(i: 1), Flags: N0->getFlags());
5532 }
5533 default:
5534 return SDValue();
5535 }
5536}
5537
5538SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
5539 DAGCombinerInfo &DCI) const {
5540 const auto *CFP = dyn_cast<ConstantFPSDNode>(Val: N->getOperand(Num: 0));
5541 if (!CFP)
5542 return SDValue();
5543
5544 std::optional<APFloat> Result = AMDGPU::evaluateRcp(Val: CFP->getValueAPF());
5545 if (!Result)
5546 return SDValue();
5547
5548 return DCI.DAG.getConstantFP(Val: *Result, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
5549}
5550
5551bool AMDGPUTargetLowering::isInt64ImmLegal(SDNode *N, SelectionDAG &DAG) const {
5552 if (!Subtarget->isGCN())
5553 return false;
5554
5555 ConstantSDNode *SDConstant = dyn_cast<ConstantSDNode>(Val: N);
5556 ConstantFPSDNode *SDFPConstant = dyn_cast<ConstantFPSDNode>(Val: N);
5557 auto &ST = DAG.getSubtarget<GCNSubtarget>();
5558 const auto *TII = ST.getInstrInfo();
5559
5560 if (!ST.hasVMovB64Inst() || (!SDConstant && !SDFPConstant))
5561 return false;
5562
5563 if (ST.has64BitLiterals())
5564 return true;
5565
5566 if (SDConstant) {
5567 const APInt &APVal = SDConstant->getAPIntValue();
5568 return isUInt<32>(x: APVal.getZExtValue()) || TII->isInlineConstant(Imm: APVal);
5569 }
5570
5571 APInt Val = SDFPConstant->getValueAPF().bitcastToAPInt();
5572 return isUInt<32>(x: Val.getZExtValue()) || TII->isInlineConstant(Imm: Val);
5573}
5574
5575SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
5576 DAGCombinerInfo &DCI) const {
5577 SelectionDAG &DAG = DCI.DAG;
5578 SDLoc DL(N);
5579
5580 switch(N->getOpcode()) {
5581 default:
5582 break;
5583 case ISD::BITCAST: {
5584 EVT DestVT = N->getValueType(ResNo: 0);
5585
5586 // Push casts through vector builds. This helps avoid emitting a large
5587 // number of copies when materializing floating point vector constants.
5588 //
5589 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
5590 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
5591 if (DestVT.isVector()) {
5592 SDValue Src = N->getOperand(Num: 0);
5593 if (Src.getOpcode() == ISD::BUILD_VECTOR &&
5594 (DCI.getDAGCombineLevel() < AfterLegalizeDAG ||
5595 isOperationLegal(Op: ISD::BUILD_VECTOR, VT: DestVT))) {
5596 EVT SrcVT = Src.getValueType();
5597 unsigned NElts = DestVT.getVectorNumElements();
5598
5599 if (SrcVT.getVectorNumElements() == NElts) {
5600 EVT DestEltVT = DestVT.getVectorElementType();
5601
5602 SmallVector<SDValue, 8> CastedElts;
5603 SDLoc SL(N);
5604 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
5605 SDValue Elt = Src.getOperand(i: I);
5606 CastedElts.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: DestEltVT, Operand: Elt));
5607 }
5608
5609 return DAG.getBuildVector(VT: DestVT, DL: SL, Ops: CastedElts);
5610 }
5611 }
5612 }
5613
5614 if (DestVT.getSizeInBits() != 64 || !DestVT.isVector())
5615 break;
5616
5617 // Fold bitcasts of constants.
5618 //
5619 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
5620 // TODO: Generalize and move to DAGCombiner
5621 SDValue Src = N->getOperand(Num: 0);
5622 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Src)) {
5623 SDLoc SL(N);
5624 if (isInt64ImmLegal(N: C, DAG))
5625 break;
5626 uint64_t CVal = C->getZExtValue();
5627 SDValue BV = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5628 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5629 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5630 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: BV);
5631 }
5632
5633 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Src)) {
5634 const APInt &Val = C->getValueAPF().bitcastToAPInt();
5635 SDLoc SL(N);
5636 if (isInt64ImmLegal(N: C, DAG))
5637 break;
5638 uint64_t CVal = Val.getZExtValue();
5639 SDValue Vec = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SL, VT: MVT::v2i32,
5640 N1: DAG.getConstant(Val: Lo_32(Value: CVal), DL: SL, VT: MVT::i32),
5641 N2: DAG.getConstant(Val: Hi_32(Value: CVal), DL: SL, VT: MVT::i32));
5642
5643 return DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: DestVT, Operand: Vec);
5644 }
5645
5646 break;
5647 }
5648 case ISD::SHL:
5649 case ISD::SRA:
5650 case ISD::SRL: {
5651 // Range metadata can be invalidated when loads are converted to legal types
5652 // (e.g. v2i64 -> v4i32).
5653 // Try to convert vector shl/sra/srl before type legalization so that range
5654 // metadata can be utilized.
5655 if (!(N->getValueType(ResNo: 0).isVector() &&
5656 DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
5657 DCI.getDAGCombineLevel() < AfterLegalizeDAG)
5658 break;
5659 if (N->getOpcode() == ISD::SHL)
5660 return performShlCombine(N, DCI);
5661 if (N->getOpcode() == ISD::SRA)
5662 return performSraCombine(N, DCI);
5663 return performSrlCombine(N, DCI);
5664 }
5665 case ISD::TRUNCATE:
5666 return performTruncateCombine(N, DCI);
5667 case ISD::MUL:
5668 return performMulCombine(N, DCI);
5669 case AMDGPUISD::MUL_U24:
5670 case AMDGPUISD::MUL_I24: {
5671 if (SDValue Simplified = simplifyMul24(Node24: N, DCI))
5672 return Simplified;
5673 break;
5674 }
5675 case AMDGPUISD::MULHI_I24:
5676 case AMDGPUISD::MULHI_U24:
5677 return simplifyMul24(Node24: N, DCI);
5678 case ISD::SMUL_LOHI:
5679 case ISD::UMUL_LOHI:
5680 return performMulLoHiCombine(N, DCI);
5681 case ISD::MULHS:
5682 return performMulhsCombine(N, DCI);
5683 case ISD::MULHU:
5684 return performMulhuCombine(N, DCI);
5685 case ISD::SELECT:
5686 return performSelectCombine(N, DCI);
5687 case ISD::FNEG:
5688 return performFNegCombine(N, DCI);
5689 case ISD::FABS:
5690 return performFAbsCombine(N, DCI);
5691 case AMDGPUISD::BFE_I32:
5692 case AMDGPUISD::BFE_U32: {
5693 assert(N->getValueType(0) == MVT::i32 &&
5694 "BFE_I32/BFE_U32 is a 32-bit operation");
5695 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2));
5696 if (!Width)
5697 break;
5698
5699 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
5700 if (WidthVal == 0)
5701 return DAG.getConstant(Val: 0, DL, VT: MVT::i32);
5702
5703 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
5704 if (!Offset)
5705 break;
5706
5707 SDValue BitsFrom = N->getOperand(Num: 0);
5708 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
5709
5710 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
5711
5712 if (OffsetVal == 0) {
5713 // This is already sign / zero extended, so try to fold away extra BFEs.
5714 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: WidthVal);
5715 if (Signed) {
5716 if (DAG.ComputeNumSignBits(Op: BitsFrom) >= 32 - WidthVal + 1)
5717 return BitsFrom;
5718
5719 // This is a sign_extend_inreg. Replace it to take advantage of existing
5720 // DAG Combines. If not eliminated, we will match back to BFE during
5721 // selection.
5722
5723 // TODO: The sext_inreg of extended types ends, although we can could
5724 // handle them in a single BFE.
5725 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32, N1: BitsFrom,
5726 N2: DAG.getValueType(SmallVT));
5727 }
5728
5729 if (DAG.MaskedValueIsZero(Op: BitsFrom,
5730 Mask: APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - WidthVal)))
5731 return BitsFrom;
5732
5733 return DAG.getZeroExtendInReg(Op: BitsFrom, DL, VT: SmallVT);
5734 }
5735
5736 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(Val&: BitsFrom)) {
5737 if (Signed) {
5738 return constantFoldBFE<int32_t>(DAG,
5739 Src0: CVal->getSExtValue(),
5740 Offset: OffsetVal,
5741 Width: WidthVal,
5742 DL);
5743 }
5744
5745 return constantFoldBFE<uint32_t>(DAG,
5746 Src0: CVal->getZExtValue(),
5747 Offset: OffsetVal,
5748 Width: WidthVal,
5749 DL);
5750 }
5751
5752 if ((OffsetVal + WidthVal) >= 32 &&
5753 !(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
5754 SDValue ShiftVal = DAG.getConstant(Val: OffsetVal, DL, VT: MVT::i32);
5755 return DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32,
5756 N1: BitsFrom, N2: ShiftVal);
5757 }
5758
5759 if (BitsFrom.hasOneUse()) {
5760 APInt Demanded = APInt::getBitsSet(numBits: 32,
5761 loBit: OffsetVal,
5762 hiBit: OffsetVal + WidthVal);
5763
5764 KnownBits Known;
5765 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
5766 !DCI.isBeforeLegalizeOps());
5767 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5768 if (TLI.ShrinkDemandedConstant(Op: BitsFrom, DemandedBits: Demanded, TLO) ||
5769 TLI.SimplifyDemandedBits(Op: BitsFrom, DemandedBits: Demanded, Known, TLO)) {
5770 DCI.CommitTargetLoweringOpt(TLO);
5771 }
5772 }
5773
5774 break;
5775 }
5776 case ISD::LOAD:
5777 return performLoadCombine(N, DCI);
5778 case ISD::STORE:
5779 return performStoreCombine(N, DCI);
5780 case AMDGPUISD::RCP:
5781 case AMDGPUISD::RCP_IFLAG:
5782 return performRcpCombine(N, DCI);
5783 case ISD::AssertZext:
5784 case ISD::AssertSext:
5785 return performAssertSZExtCombine(N, DCI);
5786 case ISD::INTRINSIC_WO_CHAIN:
5787 return performIntrinsicWOChainCombine(N, DCI);
5788 case AMDGPUISD::FMAD_FTZ: {
5789 SDValue N0 = N->getOperand(Num: 0);
5790 SDValue N1 = N->getOperand(Num: 1);
5791 SDValue N2 = N->getOperand(Num: 2);
5792 EVT VT = N->getValueType(ResNo: 0);
5793
5794 // FMAD_FTZ is a FMAD + flush denormals to zero.
5795 // We flush the inputs, the intermediate step, and the output.
5796 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Val&: N0);
5797 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
5798 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
5799 if (N0CFP && N1CFP && N2CFP) {
5800 const auto FTZ = [](const APFloat &V) {
5801 if (V.isDenormal()) {
5802 APFloat Zero(V.getSemantics(), 0);
5803 return V.isNegative() ? -Zero : Zero;
5804 }
5805 return V;
5806 };
5807
5808 APFloat V0 = FTZ(N0CFP->getValueAPF());
5809 APFloat V1 = FTZ(N1CFP->getValueAPF());
5810 APFloat V2 = FTZ(N2CFP->getValueAPF());
5811 V0.multiply(RHS: V1, RM: APFloat::rmNearestTiesToEven);
5812 V0 = FTZ(V0);
5813 V0.add(RHS: V2, RM: APFloat::rmNearestTiesToEven);
5814 return DAG.getConstantFP(Val: FTZ(V0), DL, VT);
5815 }
5816 break;
5817 }
5818 }
5819 return SDValue();
5820}
5821
5822bool AMDGPUTargetLowering::SimplifyDemandedBitsForTargetNode(
5823 SDValue Op, const APInt &OriginalDemandedBits,
5824 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
5825 unsigned Depth) const {
5826 switch (Op.getOpcode()) {
5827 case ISD::INTRINSIC_WO_CHAIN: {
5828 switch (Op.getConstantOperandVal(i: 0)) {
5829 case Intrinsic::amdgcn_readfirstlane:
5830 case Intrinsic::amdgcn_readlane:
5831 case Intrinsic::amdgcn_wwm: {
5832 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5833 DemandedElts: OriginalDemandedElts, Known, TLO, Depth: Depth + 1))
5834 return true;
5835 break;
5836 }
5837 case Intrinsic::amdgcn_set_inactive:
5838 case Intrinsic::amdgcn_set_inactive_chain_arg: {
5839 // The result is operand 1 in active lanes and operand 2 in inactive
5840 // lanes, so the known bits are the intersection of both operands.
5841 KnownBits KnownValue, KnownInactive;
5842 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: OriginalDemandedBits,
5843 DemandedElts: OriginalDemandedElts, Known&: KnownValue, TLO,
5844 Depth: Depth + 1))
5845 return true;
5846 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), DemandedBits: OriginalDemandedBits,
5847 DemandedElts: OriginalDemandedElts, Known&: KnownInactive, TLO,
5848 Depth: Depth + 1))
5849 return true;
5850 Known = KnownValue.intersectWith(RHS: KnownInactive);
5851 break;
5852 }
5853 default:
5854 break;
5855 }
5856 break;
5857 }
5858 default:
5859 break;
5860 }
5861
5862 return false;
5863}
5864
5865//===----------------------------------------------------------------------===//
5866// Helper functions
5867//===----------------------------------------------------------------------===//
5868
5869SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
5870 const TargetRegisterClass *RC,
5871 Register Reg, EVT VT,
5872 const SDLoc &SL,
5873 bool RawReg) const {
5874 MachineFunction &MF = DAG.getMachineFunction();
5875 MachineRegisterInfo &MRI = MF.getRegInfo();
5876 Register VReg;
5877
5878 if (!MRI.isLiveIn(Reg)) {
5879 VReg = MRI.createVirtualRegister(RegClass: RC);
5880 MRI.addLiveIn(Reg, vreg: VReg);
5881 } else {
5882 VReg = MRI.getLiveInVirtReg(PReg: Reg);
5883 }
5884
5885 if (RawReg)
5886 return DAG.getRegister(Reg: VReg, VT);
5887
5888 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SL, Reg: VReg, VT);
5889}
5890
5891// This may be called multiple times, and nothing prevents creating multiple
5892// objects at the same offset. See if we already defined this object.
5893static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
5894 int64_t Offset) {
5895 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
5896 if (MFI.getObjectOffset(ObjectIdx: I) == Offset) {
5897 assert(MFI.getObjectSize(I) == Size);
5898 return I;
5899 }
5900 }
5901
5902 return MFI.CreateFixedObject(Size, SPOffset: Offset, IsImmutable: true);
5903}
5904
5905SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
5906 EVT VT,
5907 const SDLoc &SL,
5908 int64_t Offset) const {
5909 MachineFunction &MF = DAG.getMachineFunction();
5910 MachineFrameInfo &MFI = MF.getFrameInfo();
5911 int FI = getOrCreateFixedStackObject(MFI, Size: VT.getStoreSize(), Offset);
5912
5913 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
5914 SDValue Ptr = DAG.getFrameIndex(FI, VT: MVT::i32);
5915
5916 return DAG.getLoad(VT, dl: SL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: SrcPtrInfo, Alignment: Align(4),
5917 MMOFlags: MachineMemOperand::MODereferenceable |
5918 MachineMemOperand::MOInvariant);
5919}
5920
5921SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
5922 const SDLoc &SL,
5923 SDValue Chain,
5924 SDValue ArgVal,
5925 int64_t Offset) const {
5926 MachineFunction &MF = DAG.getMachineFunction();
5927 MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
5928 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
5929
5930 SDValue Ptr = DAG.getConstant(Val: Offset, DL: SL, VT: MVT::i32);
5931 // Stores to the argument stack area are relative to the stack pointer.
5932 SDValue SP =
5933 DAG.getCopyFromReg(Chain, dl: SL, Reg: Info->getStackPtrOffsetReg(), VT: MVT::i32);
5934 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: MVT::i32, N1: SP, N2: Ptr);
5935 SDValue Store = DAG.getStore(Chain, dl: SL, Val: ArgVal, Ptr, PtrInfo: DstInfo, Alignment: Align(4),
5936 MMOFlags: MachineMemOperand::MODereferenceable);
5937 return Store;
5938}
5939
5940SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
5941 const TargetRegisterClass *RC,
5942 EVT VT, const SDLoc &SL,
5943 const ArgDescriptor &Arg) const {
5944 assert(Arg && "Attempting to load missing argument");
5945
5946 SDValue V = Arg.isRegister() ?
5947 CreateLiveInRegister(DAG, RC, Reg: Arg.getRegister(), VT, SL) :
5948 loadStackInputValue(DAG, VT, SL, Offset: Arg.getStackOffset());
5949
5950 if (!Arg.isMasked())
5951 return V;
5952
5953 unsigned Mask = Arg.getMask();
5954 unsigned Shift = llvm::countr_zero<unsigned>(Val: Mask);
5955 V = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT, N1: V,
5956 N2: DAG.getShiftAmountConstant(Val: Shift, VT, DL: SL));
5957 return DAG.getNode(Opcode: ISD::AND, DL: SL, VT, N1: V,
5958 N2: DAG.getConstant(Val: Mask >> Shift, DL: SL, VT));
5959}
5960
5961uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5962 uint64_t ExplicitKernArgSize, const ImplicitParameter Param) const {
5963 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
5964 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr();
5965 uint64_t ArgOffset =
5966 alignTo(Size: ExplicitKernArgSize, A: Alignment) + ExplicitArgOffset;
5967 switch (Param) {
5968 case FIRST_IMPLICIT:
5969 return ArgOffset;
5970 case PRIVATE_BASE:
5971 return ArgOffset + AMDGPU::ImplicitArg::PRIVATE_BASE_OFFSET;
5972 case SHARED_BASE:
5973 return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET;
5974 case QUEUE_PTR:
5975 return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET;
5976 }
5977 llvm_unreachable("unexpected implicit parameter type");
5978}
5979
5980uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
5981 const MachineFunction &MF, const ImplicitParameter Param) const {
5982 const AMDGPUMachineFunctionInfo *MFI =
5983 MF.getInfo<AMDGPUMachineFunctionInfo>();
5984 return getImplicitParameterOffset(ExplicitKernArgSize: MFI->getExplicitKernArgSize(), Param);
5985}
5986
5987SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
5988 SelectionDAG &DAG, int Enabled,
5989 int &RefinementSteps,
5990 bool &UseOneConstNR,
5991 bool Reciprocal) const {
5992 EVT VT = Operand.getValueType();
5993
5994 if (VT == MVT::f32) {
5995 RefinementSteps = 0;
5996 return DAG.getNode(Opcode: AMDGPUISD::RSQ, DL: SDLoc(Operand), VT, Operand);
5997 }
5998
5999 // TODO: There is also f64 rsq instruction, but the documentation is less
6000 // clear on its precision.
6001
6002 return SDValue();
6003}
6004
6005SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
6006 SelectionDAG &DAG, int Enabled,
6007 int &RefinementSteps) const {
6008 EVT VT = Operand.getValueType();
6009
6010 if (VT == MVT::f32) {
6011 // Reciprocal, < 1 ulp error.
6012 //
6013 // This reciprocal approximation converges to < 0.5 ulp error with one
6014 // newton rhapson performed with two fused multiple adds (FMAs).
6015
6016 RefinementSteps = 0;
6017 return DAG.getNode(Opcode: AMDGPUISD::RCP, DL: SDLoc(Operand), VT, Operand);
6018 }
6019
6020 // TODO: There is also f64 rcp instruction, but the documentation is less
6021 // clear on its precision.
6022
6023 return SDValue();
6024}
6025
6026static unsigned workitemIntrinsicDim(unsigned ID) {
6027 switch (ID) {
6028 case Intrinsic::amdgcn_workitem_id_x:
6029 return 0;
6030 case Intrinsic::amdgcn_workitem_id_y:
6031 return 1;
6032 case Intrinsic::amdgcn_workitem_id_z:
6033 return 2;
6034 default:
6035 llvm_unreachable("not a workitem intrinsic");
6036 }
6037}
6038
6039void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
6040 const SDValue Op, KnownBits &Known,
6041 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
6042
6043 Known.resetAll(); // Don't know anything.
6044
6045 unsigned Opc = Op.getOpcode();
6046
6047 switch (Opc) {
6048 default:
6049 break;
6050 case AMDGPUISD::CARRY:
6051 case AMDGPUISD::BORROW: {
6052 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 31);
6053 break;
6054 }
6055
6056 case AMDGPUISD::BFE_I32:
6057 case AMDGPUISD::BFE_U32: {
6058 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6059 if (!CWidth)
6060 return;
6061
6062 uint32_t Width = CWidth->getZExtValue() & 0x1f;
6063
6064 if (Opc == AMDGPUISD::BFE_U32)
6065 Known.Zero = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - Width);
6066
6067 break;
6068 }
6069 case AMDGPUISD::FP_TO_FP16: {
6070 unsigned BitWidth = Known.getBitWidth();
6071
6072 // High bits are zero.
6073 Known.Zero = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 16);
6074 break;
6075 }
6076 case AMDGPUISD::MUL_U24:
6077 case AMDGPUISD::MUL_I24: {
6078 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6079
6080 // TODO: SelfMultiply can be poison, but not undef.
6081 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
6082
6083 // For a self-multiply both operands are the same value, so RHSKnown also
6084 // describes the LHS; avoid the duplicated query.
6085 KnownBits LHSKnown =
6086 SelfMultiply ? RHSKnown
6087 : DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6088
6089 if (SelfMultiply)
6090 SelfMultiply &= DAG.isGuaranteedNotToBeUndefOrPoison(
6091 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
6092 Depth: Depth + 1);
6093
6094 Known = computeKnownBitsForMul24(LHSKnown, RHSKnown,
6095 BitWidth: Op.getScalarValueSizeInBits(),
6096 IsSigned: Opc == AMDGPUISD::MUL_I24, SelfMultiply);
6097 break;
6098 }
6099 case AMDGPUISD::PERM: {
6100 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6101 if (!CMask)
6102 return;
6103
6104 KnownBits LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6105 KnownBits RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6106 unsigned Sel = CMask->getZExtValue();
6107
6108 for (unsigned I = 0; I < 32; I += 8) {
6109 unsigned SelBits = Sel & 0xff;
6110 if (SelBits < 4) {
6111 SelBits *= 8;
6112 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6113 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6114 } else if (SelBits < 7) {
6115 SelBits = (SelBits & 3) * 8;
6116 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
6117 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
6118 } else if (SelBits == 0x0c) {
6119 Known.Zero |= 0xFFull << I;
6120 } else if (SelBits > 0x0c) {
6121 Known.One |= 0xFFull << I;
6122 }
6123 Sel >>= 8;
6124 }
6125 break;
6126 }
6127 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
6128 Known.Zero.setHighBits(24);
6129 break;
6130 }
6131 case AMDGPUISD::BUFFER_LOAD_USHORT: {
6132 Known.Zero.setHighBits(16);
6133 break;
6134 }
6135 case AMDGPUISD::LDS: {
6136 auto *GA = cast<GlobalAddressSDNode>(Val: Op.getOperand(i: 0).getNode());
6137 Align Alignment = GA->getGlobal()->getPointerAlignment(DL: DAG.getDataLayout());
6138
6139 Known.Zero.setHighBits(16);
6140 Known.Zero.setLowBits(Log2(A: Alignment));
6141 break;
6142 }
6143 case AMDGPUISD::SMIN3:
6144 case AMDGPUISD::SMAX3:
6145 case AMDGPUISD::SMED3:
6146 case AMDGPUISD::UMIN3:
6147 case AMDGPUISD::UMAX3:
6148 case AMDGPUISD::UMED3: {
6149 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6150 if (Known2.isUnknown())
6151 break;
6152
6153 KnownBits Known1 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6154 if (Known1.isUnknown())
6155 break;
6156
6157 KnownBits Known0 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6158 if (Known0.isUnknown())
6159 break;
6160
6161 // TODO: Handle LeadZero/LeadOne from UMIN/UMAX handling.
6162 Known.Zero = Known0.Zero & Known1.Zero & Known2.Zero;
6163 Known.One = Known0.One & Known1.One & Known2.One;
6164 break;
6165 }
6166 case ISD::INTRINSIC_WO_CHAIN: {
6167 unsigned IID = Op.getConstantOperandVal(i: 0);
6168 switch (IID) {
6169 case Intrinsic::amdgcn_workitem_id_x:
6170 case Intrinsic::amdgcn_workitem_id_y:
6171 case Intrinsic::amdgcn_workitem_id_z: {
6172 unsigned MaxValue = Subtarget->getMaxWorkitemID(
6173 Kernel: DAG.getMachineFunction().getFunction(), Dimension: workitemIntrinsicDim(ID: IID));
6174 Known.Zero.setHighBits(llvm::countl_zero(Val: MaxValue));
6175 break;
6176 }
6177 case Intrinsic::amdgcn_readfirstlane:
6178 case Intrinsic::amdgcn_readlane:
6179 // Result is the data operand's value from some lane.
6180 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6181 break;
6182 default:
6183 break;
6184 }
6185 }
6186 }
6187}
6188
6189unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
6190 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6191 unsigned Depth) const {
6192 switch (Op.getOpcode()) {
6193 case AMDGPUISD::BFE_I32: {
6194 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6195 if (!Width)
6196 return 1;
6197
6198 unsigned SignBits = 32 - (Width->getZExtValue() & 0x1f) + 1;
6199 if (!isNullConstant(V: Op.getOperand(i: 1)))
6200 return SignBits;
6201
6202 // TODO: Could probably figure something out with non-0 offsets.
6203 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6204 return std::max(a: SignBits, b: Op0SignBits);
6205 }
6206
6207 case AMDGPUISD::BFE_U32: {
6208 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
6209 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
6210 }
6211
6212 case AMDGPUISD::CARRY:
6213 case AMDGPUISD::BORROW:
6214 return 31;
6215 case AMDGPUISD::BUFFER_LOAD_BYTE:
6216 return 25;
6217 case AMDGPUISD::BUFFER_LOAD_SHORT:
6218 return 17;
6219 case AMDGPUISD::BUFFER_LOAD_UBYTE:
6220 return 24;
6221 case AMDGPUISD::BUFFER_LOAD_USHORT:
6222 return 16;
6223 case AMDGPUISD::FP_TO_FP16:
6224 return 16;
6225 case AMDGPUISD::SMIN3:
6226 case AMDGPUISD::SMAX3:
6227 case AMDGPUISD::SMED3:
6228 case AMDGPUISD::UMIN3:
6229 case AMDGPUISD::UMAX3:
6230 case AMDGPUISD::UMED3: {
6231 unsigned Tmp2 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6232 if (Tmp2 == 1)
6233 return 1; // Early out.
6234
6235 unsigned Tmp1 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6236 if (Tmp1 == 1)
6237 return 1; // Early out.
6238
6239 unsigned Tmp0 = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6240 if (Tmp0 == 1)
6241 return 1; // Early out.
6242
6243 return std::min(l: {Tmp0, Tmp1, Tmp2});
6244 }
6245 default:
6246 return 1;
6247 }
6248}
6249
6250unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
6251 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
6252 const MachineRegisterInfo &MRI, unsigned Depth) const {
6253 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
6254 if (!MI)
6255 return 1;
6256
6257 // TODO: Check range metadata on MMO.
6258 switch (MI->getOpcode()) {
6259 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
6260 return 25;
6261 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
6262 return 17;
6263 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
6264 return 24;
6265 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
6266 return 16;
6267 case AMDGPU::G_AMDGPU_SMED3:
6268 case AMDGPU::G_AMDGPU_UMED3: {
6269 auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs();
6270 unsigned Tmp2 = Analysis.computeNumSignBits(R: Src2, DemandedElts, Depth: Depth + 1);
6271 if (Tmp2 == 1)
6272 return 1;
6273 unsigned Tmp1 = Analysis.computeNumSignBits(R: Src1, DemandedElts, Depth: Depth + 1);
6274 if (Tmp1 == 1)
6275 return 1;
6276 unsigned Tmp0 = Analysis.computeNumSignBits(R: Src0, DemandedElts, Depth: Depth + 1);
6277 if (Tmp0 == 1)
6278 return 1;
6279 return std::min(l: {Tmp0, Tmp1, Tmp2});
6280 }
6281 default:
6282 return 1;
6283 }
6284}
6285
6286bool AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode(
6287 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6288 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
6289 unsigned Opcode = Op.getOpcode();
6290 switch (Opcode) {
6291 case AMDGPUISD::BFE_I32:
6292 case AMDGPUISD::BFE_U32:
6293 return false;
6294 }
6295 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
6296 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
6297}
6298
6299bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
6300 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
6301 unsigned Depth) const {
6302 unsigned Opcode = Op.getOpcode();
6303 switch (Opcode) {
6304 case AMDGPUISD::FMIN_LEGACY:
6305 case AMDGPUISD::FMAX_LEGACY:
6306 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6307 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6308 case AMDGPUISD::FMUL_LEGACY:
6309 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
6310 if (SNaN)
6311 return true;
6312 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6313 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6314 }
6315 case AMDGPUISD::FMED3:
6316 case AMDGPUISD::FMIN3:
6317 case AMDGPUISD::FMAX3:
6318 case AMDGPUISD::FMINIMUM3:
6319 case AMDGPUISD::FMAXIMUM3:
6320 case AMDGPUISD::FMAD_FTZ: {
6321 if (SNaN)
6322 return true;
6323 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1) &&
6324 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6325 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6326 }
6327 case AMDGPUISD::CVT_F32_UBYTE0:
6328 case AMDGPUISD::CVT_F32_UBYTE1:
6329 case AMDGPUISD::CVT_F32_UBYTE2:
6330 case AMDGPUISD::CVT_F32_UBYTE3:
6331 return true;
6332
6333 case AMDGPUISD::RCP:
6334 case AMDGPUISD::RSQ:
6335 case AMDGPUISD::RCP_LEGACY:
6336 case AMDGPUISD::RSQ_CLAMP: {
6337 if (SNaN)
6338 return true;
6339
6340 // TODO: Need is known positive check.
6341 return false;
6342 }
6343 case ISD::FLDEXP:
6344 case AMDGPUISD::FRACT: {
6345 if (SNaN)
6346 return true;
6347 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6348 }
6349 case AMDGPUISD::DIV_SCALE:
6350 case AMDGPUISD::DIV_FMAS:
6351 case AMDGPUISD::DIV_FIXUP:
6352 // TODO: Refine on operands.
6353 return SNaN;
6354 case AMDGPUISD::SIN_HW:
6355 case AMDGPUISD::COS_HW: {
6356 // TODO: Need check for infinity
6357 return SNaN;
6358 }
6359 case ISD::INTRINSIC_WO_CHAIN: {
6360 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
6361 // TODO: Handle more intrinsics
6362 switch (IntrinsicID) {
6363 case Intrinsic::amdgcn_cubeid:
6364 case Intrinsic::amdgcn_cvt_off_f32_i4:
6365 return true;
6366
6367 case Intrinsic::amdgcn_frexp_mant: {
6368 if (SNaN)
6369 return true;
6370 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1);
6371 }
6372 case Intrinsic::amdgcn_cvt_pkrtz: {
6373 if (SNaN)
6374 return true;
6375 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6376 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1);
6377 }
6378 case Intrinsic::amdgcn_rcp:
6379 case Intrinsic::amdgcn_rsq:
6380 case Intrinsic::amdgcn_rcp_legacy:
6381 case Intrinsic::amdgcn_rsq_legacy:
6382 case Intrinsic::amdgcn_rsq_clamp:
6383 case Intrinsic::amdgcn_tanh: {
6384 if (SNaN)
6385 return true;
6386
6387 // TODO: Need is known positive check.
6388 return false;
6389 }
6390 case Intrinsic::amdgcn_trig_preop:
6391 case Intrinsic::amdgcn_fdot2:
6392 // TODO: Refine on operand
6393 return SNaN;
6394 case Intrinsic::amdgcn_fma_legacy:
6395 if (SNaN)
6396 return true;
6397 return DAG.isKnownNeverNaN(Op: Op.getOperand(i: 1), SNaN, Depth: Depth + 1) &&
6398 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 2), SNaN, Depth: Depth + 1) &&
6399 DAG.isKnownNeverNaN(Op: Op.getOperand(i: 3), SNaN, Depth: Depth + 1);
6400 default:
6401 return false;
6402 }
6403 }
6404 default:
6405 return false;
6406 }
6407}
6408
6409bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
6410 Register N0, Register N1) const {
6411 return MRI.hasOneNonDBGUse(RegNo: N0); // FIXME: handle regbanks
6412}
6413