1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error(reason: "Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::FCANONICALIZE:
74 R = SoftenFloatRes_FCANONICALIZE(N); break;
75 case ISD::STRICT_FMINNUM:
76 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
77 case ISD::STRICT_FMAXNUM:
78 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
79 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
80 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
81 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
82 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
83 case ISD::STRICT_FADD:
84 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
85 case ISD::STRICT_FACOS:
86 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
87 case ISD::STRICT_FASIN:
88 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
89 case ISD::STRICT_FATAN:
90 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
91 case ISD::STRICT_FATAN2:
92 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
93 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
94 case ISD::STRICT_FCEIL:
95 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
96 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
97 case ISD::STRICT_FCOS:
98 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
99 case ISD::STRICT_FCOSH:
100 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
101 case ISD::STRICT_FDIV:
102 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
103 case ISD::STRICT_FEXP:
104 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
105 case ISD::STRICT_FEXP2:
106 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
107 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
108 case ISD::STRICT_FFLOOR:
109 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
110 case ISD::STRICT_FLOG:
111 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
112 case ISD::STRICT_FLOG2:
113 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
114 case ISD::STRICT_FLOG10:
115 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
116 case ISD::STRICT_FMA:
117 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
118 case ISD::STRICT_FMUL:
119 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
120 case ISD::STRICT_FNEARBYINT:
121 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
122 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
123 case ISD::STRICT_FP_EXTEND:
124 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
125 case ISD::STRICT_FP_ROUND:
126 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
127 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
128 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
129 case ISD::STRICT_FPOW:
130 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
131 case ISD::STRICT_FPOWI:
132 case ISD::FPOWI:
133 case ISD::FLDEXP:
134 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
135 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
136 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
137 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
138 case ISD::STRICT_FREM:
139 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
140 case ISD::STRICT_FRINT:
141 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
142 case ISD::STRICT_FROUND:
143 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
144 case ISD::STRICT_FROUNDEVEN:
145 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
146 case ISD::STRICT_FSIN:
147 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
148 case ISD::STRICT_FSINH:
149 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
150 case ISD::STRICT_FSQRT:
151 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
152 case ISD::STRICT_FSUB:
153 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
154 case ISD::STRICT_FTAN:
155 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
156 case ISD::STRICT_FTANH:
157 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
158 case ISD::STRICT_FTRUNC:
159 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
160 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
161 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
162 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
163 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
164 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
165 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
166 case ISD::STRICT_SINT_TO_FP:
167 case ISD::STRICT_UINT_TO_FP:
168 case ISD::SINT_TO_FP:
169 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
170 case ISD::POISON:
171 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
172 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
173 case ISD::AssertNoFPClass: R = GetSoftenedFloat(Op: N->getOperand(Num: 0)); break;
174 case ISD::VECREDUCE_FADD:
175 case ISD::VECREDUCE_FMUL:
176 case ISD::VECREDUCE_FMIN:
177 case ISD::VECREDUCE_FMAX:
178 case ISD::VECREDUCE_FMAXIMUM:
179 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
180 case ISD::VECREDUCE_SEQ_FADD:
181 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
182 // clang-format on
183 }
184
185 // If R is null, the sub-method took care of registering the result.
186 if (R.getNode()) {
187 assert(R.getNode() != N);
188 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
189 }
190}
191
192SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
193 bool IsStrict = N->isStrictFPOpcode();
194 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
195 unsigned Offset = IsStrict ? 1 : 0;
196 assert(N->getNumOperands() == (1 + Offset) &&
197 "Unexpected number of operands!");
198 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
199 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
200 TargetLowering::MakeLibCallOptions CallOptions;
201 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
202 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
203 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
204 CallOptions, dl: SDLoc(N),
205 Chain);
206 if (IsStrict)
207 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
208 return Tmp.first;
209}
210
211SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
212 bool IsStrict = N->isStrictFPOpcode();
213 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
214 unsigned Offset = IsStrict ? 1 : 0;
215 assert(N->getNumOperands() == (2 + Offset) &&
216 "Unexpected number of operands!");
217 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
218 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
219 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
220 TargetLowering::MakeLibCallOptions CallOptions;
221 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
222 N->getOperand(Num: 1 + Offset).getValueType() };
223 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
224 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
225 CallOptions, dl: SDLoc(N),
226 Chain);
227 if (IsStrict)
228 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
229 return Tmp.first;
230}
231
232SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
233 return BitConvertToInteger(Op: N->getOperand(Num: 0));
234}
235
236SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
237 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
238 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
239 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
240}
241
242SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
243 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
244 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
245 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
246 return NewFence;
247}
248
249SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
250 unsigned ResNo) {
251 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
252 return BitConvertToInteger(Op);
253}
254
255SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
256 // Convert the inputs to integers, and build a new pair out of them.
257 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
258 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
259 VT: N->getValueType(ResNo: 0)),
260 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
261 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
262}
263
264SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
265 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
266 // In ppcf128, the high 64 bits are always first in memory regardless
267 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
268 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
269 // way. However, APInt's are serialized in an Endian-sensitive fashion,
270 // so on big-Endian targets, the two doubles are output in the wrong
271 // order. Fix this by manually flipping the order of the high 64 bits
272 // and the low 64 bits here.
273 if (DAG.getDataLayout().isBigEndian() &&
274 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
275 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
276 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
277 APInt Val(128, words);
278 return DAG.getConstant(Val, DL: SDLoc(CN),
279 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
280 VT: CN->getValueType(ResNo: 0)));
281 } else {
282 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
283 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
284 VT: CN->getValueType(ResNo: 0)));
285 }
286}
287
288SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
289 SDValue Src = N->getOperand(Num: 0);
290 assert(Src.getValueType() == MVT::ppcf128 &&
291 "In floats only ppcf128 can be extracted by element!");
292 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(N),
293 VT: N->getValueType(ResNo: 0).changeTypeToInteger(),
294 N1: DAG.getBitcast(VT: MVT::i128, V: Src), N2: N->getOperand(Num: 1));
295}
296
297SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
298 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
299 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
300 VT: NewOp.getValueType().getVectorElementType(),
301 N1: NewOp, N2: N->getOperand(Num: 1));
302}
303
304SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
305 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
306 unsigned Size = NVT.getSizeInBits();
307
308 // Mask = ~(1 << (Size-1))
309 APInt API = APInt::getAllOnes(numBits: Size);
310 API.clearBit(BitPosition: Size - 1);
311 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
312 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
313 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
314}
315
316SDValue DAGTypeLegalizer::SoftenFloatRes_FCANONICALIZE(SDNode *N) {
317 SDLoc dl(N);
318
319 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
320 // suggested in
321 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
322 // It uses strict_fp operations even outside a strict_fp context in order
323 // to guarantee that the canonicalization is not optimized away by later
324 // passes. The result chain introduced by that is intentionally ignored
325 // since no ordering requirement is intended here.
326
327 // Create strict multiplication by 1.0.
328 SDValue Operand = N->getOperand(Num: 0);
329 EVT VT = Operand.getValueType();
330 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
331 SDValue Chain = DAG.getEntryNode();
332 // Propagate existing flags on canonicalize, and additionally set
333 // NoFPExcept.
334 SDNodeFlags CanonicalizeFlags = N->getFlags();
335 CanonicalizeFlags.setNoFPExcept(true);
336 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
337 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
338 return BitConvertToInteger(Op: Mul);
339}
340
341SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
342 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
343 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
344 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
345 Call_F32: RTLIB::FMIN_F32,
346 Call_F64: RTLIB::FMIN_F64,
347 Call_F80: RTLIB::FMIN_F80,
348 Call_F128: RTLIB::FMIN_F128,
349 Call_PPCF128: RTLIB::FMIN_PPCF128));
350}
351
352SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
353 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
354 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
355 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
356 Call_F32: RTLIB::FMAX_F32,
357 Call_F64: RTLIB::FMAX_F64,
358 Call_F80: RTLIB::FMAX_F80,
359 Call_F128: RTLIB::FMAX_F128,
360 Call_PPCF128: RTLIB::FMAX_PPCF128));
361}
362
363SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
364 return SoftenFloatRes_Binary(
365 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_NUM_F32,
366 Call_F64: RTLIB::FMINIMUM_NUM_F64, Call_F80: RTLIB::FMINIMUM_NUM_F80,
367 Call_F128: RTLIB::FMINIMUM_NUM_F128, Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128));
368}
369
370SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
371 return SoftenFloatRes_Binary(
372 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_NUM_F32,
373 Call_F64: RTLIB::FMAXIMUM_NUM_F64, Call_F80: RTLIB::FMAXIMUM_NUM_F80,
374 Call_F128: RTLIB::FMAXIMUM_NUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128));
375}
376
377SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
378 return SoftenFloatRes_Binary(
379 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_F32,
380 Call_F64: RTLIB::FMINIMUM_F64, Call_F80: RTLIB::FMINIMUM_F80,
381 Call_F128: RTLIB::FMINIMUM_F128, Call_PPCF128: RTLIB::FMINIMUM_PPCF128));
382}
383
384SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
385 return SoftenFloatRes_Binary(
386 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_F32,
387 Call_F64: RTLIB::FMAXIMUM_F64, Call_F80: RTLIB::FMAXIMUM_F80,
388 Call_F128: RTLIB::FMAXIMUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_PPCF128));
389}
390
391SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
392 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
393 Call_F32: RTLIB::ADD_F32,
394 Call_F64: RTLIB::ADD_F64,
395 Call_F80: RTLIB::ADD_F80,
396 Call_F128: RTLIB::ADD_F128,
397 Call_PPCF128: RTLIB::ADD_PPCF128));
398}
399
400SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
401 return SoftenFloatRes_Unary(
402 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64,
403 Call_F80: RTLIB::ACOS_F80, Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128));
404}
405
406SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
407 return SoftenFloatRes_Unary(
408 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64,
409 Call_F80: RTLIB::ASIN_F80, Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128));
410}
411
412SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
413 return SoftenFloatRes_Unary(
414 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64,
415 Call_F80: RTLIB::ATAN_F80, Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128));
416}
417
418SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
419 return SoftenFloatRes_Binary(
420 N,
421 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN2_F32, Call_F64: RTLIB::ATAN2_F64,
422 Call_F80: RTLIB::ATAN2_F80, Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128));
423}
424
425SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
426 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
427 Call_F32: RTLIB::CBRT_F32,
428 Call_F64: RTLIB::CBRT_F64,
429 Call_F80: RTLIB::CBRT_F80,
430 Call_F128: RTLIB::CBRT_F128,
431 Call_PPCF128: RTLIB::CBRT_PPCF128));
432}
433
434SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
435 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
436 Call_F32: RTLIB::CEIL_F32,
437 Call_F64: RTLIB::CEIL_F64,
438 Call_F80: RTLIB::CEIL_F80,
439 Call_F128: RTLIB::CEIL_F128,
440 Call_PPCF128: RTLIB::CEIL_PPCF128));
441}
442
443SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
444 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
445 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
446 SDLoc dl(N);
447
448 EVT LVT = LHS.getValueType();
449 EVT RVT = RHS.getValueType();
450
451 unsigned LSize = LVT.getSizeInBits();
452 unsigned RSize = RVT.getSizeInBits();
453
454 // First get the sign bit of second operand.
455 SDValue SignBit = DAG.getNode(
456 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
457 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
458 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
459 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
460
461 // Shift right or sign-extend it if the two operands have different types.
462 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
463 if (SizeDiff > 0) {
464 SignBit =
465 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
466 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
467 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
468 DL: DAG.getDataLayout())));
469 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
470 } else if (SizeDiff < 0) {
471 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
472 SignBit =
473 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
474 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
475 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
476 DL: DAG.getDataLayout())));
477 }
478
479 // Clear the sign bit of the first operand.
480 SDValue Mask = DAG.getNode(
481 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
482 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
483 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
484 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
485 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
486
487 // Or the value with the sign bit.
488 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
489}
490
491SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
492 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
493 Call_F32: RTLIB::COS_F32,
494 Call_F64: RTLIB::COS_F64,
495 Call_F80: RTLIB::COS_F80,
496 Call_F128: RTLIB::COS_F128,
497 Call_PPCF128: RTLIB::COS_PPCF128));
498}
499
500SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
501 return SoftenFloatRes_Unary(
502 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64,
503 Call_F80: RTLIB::COSH_F80, Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128));
504}
505
506SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
507 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
508 Call_F32: RTLIB::DIV_F32,
509 Call_F64: RTLIB::DIV_F64,
510 Call_F80: RTLIB::DIV_F80,
511 Call_F128: RTLIB::DIV_F128,
512 Call_PPCF128: RTLIB::DIV_PPCF128));
513}
514
515SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
516 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
517 Call_F32: RTLIB::EXP_F32,
518 Call_F64: RTLIB::EXP_F64,
519 Call_F80: RTLIB::EXP_F80,
520 Call_F128: RTLIB::EXP_F128,
521 Call_PPCF128: RTLIB::EXP_PPCF128));
522}
523
524SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
525 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
526 Call_F32: RTLIB::EXP2_F32,
527 Call_F64: RTLIB::EXP2_F64,
528 Call_F80: RTLIB::EXP2_F80,
529 Call_F128: RTLIB::EXP2_F128,
530 Call_PPCF128: RTLIB::EXP2_PPCF128));
531}
532
533SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
534 return SoftenFloatRes_Unary(
535 N,
536 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64,
537 Call_F80: RTLIB::EXP10_F80, Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128));
538}
539
540SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
541 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
542 Call_F32: RTLIB::FLOOR_F32,
543 Call_F64: RTLIB::FLOOR_F64,
544 Call_F80: RTLIB::FLOOR_F80,
545 Call_F128: RTLIB::FLOOR_F128,
546 Call_PPCF128: RTLIB::FLOOR_PPCF128));
547}
548
549SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
550 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
551 Call_F32: RTLIB::LOG_F32,
552 Call_F64: RTLIB::LOG_F64,
553 Call_F80: RTLIB::LOG_F80,
554 Call_F128: RTLIB::LOG_F128,
555 Call_PPCF128: RTLIB::LOG_PPCF128));
556}
557
558SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
559 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
560 Call_F32: RTLIB::LOG2_F32,
561 Call_F64: RTLIB::LOG2_F64,
562 Call_F80: RTLIB::LOG2_F80,
563 Call_F128: RTLIB::LOG2_F128,
564 Call_PPCF128: RTLIB::LOG2_PPCF128));
565}
566
567SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
568 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
569 Call_F32: RTLIB::LOG10_F32,
570 Call_F64: RTLIB::LOG10_F64,
571 Call_F80: RTLIB::LOG10_F80,
572 Call_F128: RTLIB::LOG10_F128,
573 Call_PPCF128: RTLIB::LOG10_PPCF128));
574}
575
576SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
577 bool IsStrict = N->isStrictFPOpcode();
578 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
579 unsigned Offset = IsStrict ? 1 : 0;
580 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
581 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
582 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
583 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
584 TargetLowering::MakeLibCallOptions CallOptions;
585 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
586 N->getOperand(Num: 1 + Offset).getValueType(),
587 N->getOperand(Num: 2 + Offset).getValueType() };
588 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
589 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
590 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
591 Call_F32: RTLIB::FMA_F32,
592 Call_F64: RTLIB::FMA_F64,
593 Call_F80: RTLIB::FMA_F80,
594 Call_F128: RTLIB::FMA_F128,
595 Call_PPCF128: RTLIB::FMA_PPCF128),
596 RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
597 if (IsStrict)
598 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
599 return Tmp.first;
600}
601
602SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
603 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
604 Call_F32: RTLIB::MUL_F32,
605 Call_F64: RTLIB::MUL_F64,
606 Call_F80: RTLIB::MUL_F80,
607 Call_F128: RTLIB::MUL_F128,
608 Call_PPCF128: RTLIB::MUL_PPCF128));
609}
610
611SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
612 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
613 Call_F32: RTLIB::NEARBYINT_F32,
614 Call_F64: RTLIB::NEARBYINT_F64,
615 Call_F80: RTLIB::NEARBYINT_F80,
616 Call_F128: RTLIB::NEARBYINT_F128,
617 Call_PPCF128: RTLIB::NEARBYINT_PPCF128));
618}
619
620SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
621 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
622 SDLoc dl(N);
623
624 // Expand Y = FNEG(X) -> Y = X ^ sign mask
625 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
626 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
627 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
628}
629
630SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
631 bool IsStrict = N->isStrictFPOpcode();
632 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
633 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
634
635 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
636
637 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
638 // -> f32, so proceed in two stages. Also, it's entirely possible for both
639 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
640 // than FP16_TO_FP.
641 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
642 N->getValueType(ResNo: 0) != MVT::f32) {
643 if (IsStrict) {
644 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
645 ResultTys: { MVT::f32, MVT::Other }, Ops: { Chain, Op });
646 Chain = Op.getValue(R: 1);
647 } else {
648 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT: MVT::f32, Operand: Op);
649 }
650 }
651
652 if (Op.getValueType() == MVT::bf16) {
653 // FIXME: Need ReplaceValueWith on chain in strict case
654 return SoftenFloatRes_BF16_TO_FP(N);
655 }
656
657 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
658 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
659 TargetLowering::MakeLibCallOptions CallOptions;
660 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
661 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
662 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
663 CallOptions, dl: SDLoc(N),
664 Chain);
665 if (IsStrict)
666 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
667 return Tmp.first;
668}
669
670// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
671// nodes?
672SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
673 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
674 SDValue Op = N->getOperand(Num: 0);
675 TargetLowering::MakeLibCallOptions CallOptions;
676 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
677 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
678 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
679 CallOptions, dl: SDLoc(N)).first;
680 if (N->getValueType(ResNo: 0) == MVT::f32)
681 return Res32;
682
683 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
684 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: MVT::f32, RetVT: N->getValueType(ResNo: 0));
685 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
686 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
687}
688
689// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
690// nodes?
691SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
692 assert(N->getValueType(0) == MVT::f32 &&
693 "Can only soften BF16_TO_FP with f32 result");
694 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
695 SDValue Op = N->getOperand(Num: 0);
696 SDLoc DL(N);
697 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT,
698 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op));
699 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
700 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
701 return Res;
702}
703
704SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
705 bool IsStrict = N->isStrictFPOpcode();
706 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
707 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
708 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
709 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
710 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
711 TargetLowering::MakeLibCallOptions CallOptions;
712 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
713 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
714 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
715 CallOptions, dl: SDLoc(N),
716 Chain);
717 if (IsStrict)
718 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
719 return Tmp.first;
720}
721
722SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
723 return SoftenFloatRes_Binary(N, LC: RTLIB::getPOW(RetVT: N->getValueType(ResNo: 0)));
724}
725
726SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
727 bool IsStrict = N->isStrictFPOpcode();
728 unsigned Offset = IsStrict ? 1 : 0;
729 bool IsPowI =
730 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
731 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
732
733 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0))
734 : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0));
735 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
736 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
737 // Some targets don't have a powi libcall; use pow instead.
738 // FIXME: Implement this if some target needs it.
739 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fpowi to fpow");
740 if (IsStrict)
741 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
742 return DAG.getPOISON(VT: NVT);
743 }
744
745 if (DAG.getLibInfo().getIntSize() !=
746 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
747 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
748 // would use the wrong type for the argument.
749 DAG.getContext()->emitError(ErrorStr: "powi exponent does not match sizeof(int)");
750 if (IsStrict)
751 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
752 return DAG.getPOISON(VT: NVT);
753 }
754
755 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
756 N->getOperand(Num: 1 + Offset) };
757 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
758 TargetLowering::MakeLibCallOptions CallOptions;
759 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
760 N->getOperand(Num: 1 + Offset).getValueType() };
761 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
762 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
763 CallOptions, dl: SDLoc(N),
764 Chain);
765 if (IsStrict)
766 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
767 return Tmp.first;
768}
769
770SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
771 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
772 EVT VT0 = N->getValueType(ResNo: 0);
773 EVT VT1 = N->getValueType(ResNo: 1);
774 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT0);
775 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
776 SDLoc DL(N);
777
778 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
779 // If the exponent does not match with sizeof(int) a libcall would use the
780 // wrong type for the argument.
781 // TODO: Should be able to handle mismatches.
782 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
783 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
784 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
785 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
786 }
787
788 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
789
790 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
791 TargetLowering::MakeLibCallOptions CallOptions;
792 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
793 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
794 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
795
796 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
797 // but we only really need to handle the 0th one for softening anyway.
798 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0)
799 .setOpsTypeOverrides(CallOpsTypeOverrides);
800
801 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, RetVT: NVT0, Ops, CallOptions, dl: DL,
802 /*Chain=*/SDValue());
803 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
804 auto PtrInfo =
805 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
806
807 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
808
809 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
810 return ReturnVal;
811}
812
813bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
814 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
815 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
816 EVT VT = N->getValueType(ResNo: 0);
817
818 assert(VT == N->getValueType(1) &&
819 "expected both return values to have the same type");
820
821 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
822 if (LCImpl == RTLIB::Unsupported)
823 return false;
824
825 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
826
827 SDLoc DL(N);
828
829 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(Op: N->getOperand(Num: 0))};
830 SmallVector<EVT, 3> OpsVT = {VT};
831
832 std::array<SDValue, 2> StackSlots;
833 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
834 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
835 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
836 if (ResNum == CallRetResNo)
837 continue;
838 SDValue StackSlot = DAG.CreateStackTemporary(VT: NVT);
839 Ops.push_back(Elt: StackSlot);
840 OpsVT.push_back(Elt: StackSlot.getValueType());
841 StackSlots[ResNum] = StackSlot;
842 CallOpsTypeOverrides.push_back(Elt: PointerTy);
843 }
844
845 TargetLowering::MakeLibCallOptions CallOptions;
846 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
847 // but since both returns have the same type it should be okay.
848 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT)
849 .setOpsTypeOverrides(CallOpsTypeOverrides);
850
851 auto [ReturnVal, Chain] =
852 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: DL,
853 /*Chain=*/SDValue());
854
855 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
856 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
857 auto PtrInfo =
858 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
859 return DAG.getLoad(VT: NVT, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
860 };
861
862 for (auto [ResNum, SlackSlot] : enumerate(First&: StackSlots)) {
863 if (CallRetResNo == ResNum) {
864 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: ReturnVal);
865 continue;
866 }
867 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: CreateStackLoad(SlackSlot));
868 }
869
870 return true;
871}
872
873SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
874 EVT VT = N->getValueType(ResNo: 0);
875 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(RetVT: VT)))
876 return SDValue();
877
878 // Fall back on softening the separate sin and cos calls if available.
879 RTLIB::Libcall SinLC = RTLIB::getSIN(RetVT: VT);
880 RTLIB::Libcall CosLC = RTLIB::getCOS(RetVT: VT);
881
882 SDValue SoftSin, SoftCos;
883 if (DAG.getLibcalls().getLibcallImpl(Call: SinLC) == RTLIB::Unsupported ||
884 DAG.getLibcalls().getLibcallImpl(Call: CosLC) == RTLIB::Unsupported) {
885 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fsincos");
886
887 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
888 SoftSin = SoftCos = DAG.getPOISON(VT: NVT);
889 } else {
890 SoftSin = SoftenFloatRes_Unary(N, LC: SinLC);
891 SoftCos = SoftenFloatRes_Unary(N, LC: CosLC);
892 }
893
894 SetSoftenedFloat(Op: SDValue(N, 0), Result: SoftSin);
895 SetSoftenedFloat(Op: SDValue(N, 1), Result: SoftCos);
896 return SDValue();
897}
898
899SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
900 EVT VT = N->getValueType(ResNo: 0);
901 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT),
902 /*CallRetResNo=*/0))
903 return SDValue();
904
905 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
906 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fmodf");
907 SDValue Poison = DAG.getPOISON(VT: NVT);
908 SetSoftenedFloat(Op: SDValue(N, 0), Result: Poison);
909 SetSoftenedFloat(Op: SDValue(N, 1), Result: Poison);
910 return SDValue();
911}
912
913SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
914 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
915 Call_F32: RTLIB::REM_F32,
916 Call_F64: RTLIB::REM_F64,
917 Call_F80: RTLIB::REM_F80,
918 Call_F128: RTLIB::REM_F128,
919 Call_PPCF128: RTLIB::REM_PPCF128));
920}
921
922SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
923 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
924 Call_F32: RTLIB::RINT_F32,
925 Call_F64: RTLIB::RINT_F64,
926 Call_F80: RTLIB::RINT_F80,
927 Call_F128: RTLIB::RINT_F128,
928 Call_PPCF128: RTLIB::RINT_PPCF128));
929}
930
931SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
932 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
933 Call_F32: RTLIB::ROUND_F32,
934 Call_F64: RTLIB::ROUND_F64,
935 Call_F80: RTLIB::ROUND_F80,
936 Call_F128: RTLIB::ROUND_F128,
937 Call_PPCF128: RTLIB::ROUND_PPCF128));
938}
939
940SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
941 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
942 Call_F32: RTLIB::ROUNDEVEN_F32,
943 Call_F64: RTLIB::ROUNDEVEN_F64,
944 Call_F80: RTLIB::ROUNDEVEN_F80,
945 Call_F128: RTLIB::ROUNDEVEN_F128,
946 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128));
947}
948
949SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
950 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
951 Call_F32: RTLIB::SIN_F32,
952 Call_F64: RTLIB::SIN_F64,
953 Call_F80: RTLIB::SIN_F80,
954 Call_F128: RTLIB::SIN_F128,
955 Call_PPCF128: RTLIB::SIN_PPCF128));
956}
957
958SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
959 return SoftenFloatRes_Unary(
960 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64,
961 Call_F80: RTLIB::SINH_F80, Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128));
962}
963
964SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
965 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
966 Call_F32: RTLIB::SQRT_F32,
967 Call_F64: RTLIB::SQRT_F64,
968 Call_F80: RTLIB::SQRT_F80,
969 Call_F128: RTLIB::SQRT_F128,
970 Call_PPCF128: RTLIB::SQRT_PPCF128));
971}
972
973SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
974 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
975 Call_F32: RTLIB::SUB_F32,
976 Call_F64: RTLIB::SUB_F64,
977 Call_F80: RTLIB::SUB_F80,
978 Call_F128: RTLIB::SUB_F128,
979 Call_PPCF128: RTLIB::SUB_PPCF128));
980}
981
982SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
983 return SoftenFloatRes_Unary(
984 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64,
985 Call_F80: RTLIB::TAN_F80, Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128));
986}
987
988SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
989 return SoftenFloatRes_Unary(
990 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64,
991 Call_F80: RTLIB::TANH_F80, Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128));
992}
993
994SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
995 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
996 Call_F32: RTLIB::TRUNC_F32,
997 Call_F64: RTLIB::TRUNC_F64,
998 Call_F80: RTLIB::TRUNC_F80,
999 Call_F128: RTLIB::TRUNC_F128,
1000 Call_PPCF128: RTLIB::TRUNC_PPCF128));
1001}
1002
1003SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
1004 LoadSDNode *L = cast<LoadSDNode>(Val: N);
1005 EVT VT = N->getValueType(ResNo: 0);
1006 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1007 SDLoc dl(N);
1008
1009 auto MMOFlags =
1010 L->getMemOperand()->getFlags() &
1011 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
1012 SDValue NewL;
1013 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
1014 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: NVT, dl,
1015 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
1016 PtrInfo: L->getPointerInfo(), MemVT: NVT, Alignment: L->getBaseAlign(), MMOFlags,
1017 AAInfo: L->getAAInfo());
1018 // Legalized the chain result - switch anything that used the old chain to
1019 // use the new one.
1020 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
1021 return NewL;
1022 }
1023
1024 // Do a non-extending load followed by FP_EXTEND.
1025 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
1026 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
1027 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(), Alignment: L->getBaseAlign(),
1028 MMOFlags, AAInfo: L->getAAInfo());
1029 // Legalized the chain result - switch anything that used the old chain to
1030 // use the new one.
1031 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
1032 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
1033 return BitConvertToInteger(Op: ExtendNode);
1034}
1035
1036SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
1037 AtomicSDNode *L = cast<AtomicSDNode>(Val: N);
1038 EVT VT = N->getValueType(ResNo: 0);
1039 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1040 SDLoc dl(N);
1041
1042 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
1043 SDValue NewL =
1044 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
1045 Ops: {L->getChain(), L->getBasePtr()}, MMO: L->getMemOperand());
1046
1047 // Legalized the chain result - switch anything that used the old chain to
1048 // use the new one.
1049 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
1050 return NewL;
1051 }
1052
1053 report_fatal_error(reason: "softening fp extending atomic load not handled");
1054}
1055
1056SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
1057 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
1058 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
1059 return DAG.getSelect(DL: SDLoc(N),
1060 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
1061}
1062
1063SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
1064 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
1065 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
1066 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
1067 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
1068 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
1069}
1070
1071SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
1072 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
1073 VT: N->getValueType(ResNo: 0)));
1074}
1075
1076SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
1077 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
1078 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
1079 EVT VT = N->getValueType(ResNo: 0);
1080 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1081 SDLoc dl(N);
1082
1083 SDValue NewVAARG;
1084 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
1085 Align: N->getConstantOperandVal(Num: 3));
1086
1087 // Legalized the chain result - switch anything that used the old chain to
1088 // use the new one.
1089 if (N != NewVAARG.getValue(R: 1).getNode())
1090 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
1091 return NewVAARG;
1092}
1093
1094SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1095 bool IsStrict = N->isStrictFPOpcode();
1096 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1097 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1098 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
1099 EVT RVT = N->getValueType(ResNo: 0);
1100 EVT NVT = EVT();
1101 SDLoc dl(N);
1102
1103 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1104 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1105 // match. Look for an appropriate libcall.
1106 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1107 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1108 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1109 NVT = (MVT::SimpleValueType)t;
1110 // The source needs to big enough to hold the operand.
1111 if (NVT.bitsGE(VT: SVT))
1112 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
1113 }
1114 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1115
1116 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1117 // Sign/zero extend the argument if the libcall takes a larger type.
1118 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
1119 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
1120 TargetLowering::MakeLibCallOptions CallOptions;
1121 CallOptions.setIsSigned(Signed);
1122 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1123 std::pair<SDValue, SDValue> Tmp =
1124 TLI.makeLibCall(DAG, LC, RetVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT),
1125 Ops: Op, CallOptions, dl, Chain);
1126
1127 if (IsStrict)
1128 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1129 return Tmp.first;
1130}
1131
1132SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1133 // Expand and soften recursively.
1134 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
1135 return SDValue();
1136}
1137
1138SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1139 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
1140 return SDValue();
1141}
1142
1143//===----------------------------------------------------------------------===//
1144// Convert Float Operand to Integer
1145//===----------------------------------------------------------------------===//
1146
1147bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1148 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1149 SDValue Res = SDValue();
1150
1151 switch (N->getOpcode()) {
1152 default:
1153#ifndef NDEBUG
1154 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1155 N->dump(&DAG); dbgs() << "\n";
1156#endif
1157 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
1158
1159 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1160 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1161 case ISD::STRICT_FP_TO_FP16:
1162 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1163 case ISD::FP_TO_BF16:
1164 case ISD::STRICT_FP_TO_BF16:
1165 case ISD::STRICT_FP_ROUND:
1166 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1167 case ISD::STRICT_FP_TO_SINT:
1168 case ISD::STRICT_FP_TO_UINT:
1169 case ISD::FP_TO_SINT:
1170 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1171 case ISD::FP_TO_SINT_SAT:
1172 case ISD::FP_TO_UINT_SAT:
1173 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1174 case ISD::STRICT_LROUND:
1175 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1176 case ISD::STRICT_LLROUND:
1177 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1178 case ISD::STRICT_LRINT:
1179 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1180 case ISD::STRICT_LLRINT:
1181 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1182 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1183 case ISD::STRICT_FSETCC:
1184 case ISD::STRICT_FSETCCS:
1185 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1186 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1187 case ISD::ATOMIC_STORE:
1188 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1189 break;
1190 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1191 case ISD::FAKE_USE:
1192 Res = SoftenFloatOp_FAKE_USE(N);
1193 break;
1194 case ISD::STACKMAP:
1195 Res = SoftenFloatOp_STACKMAP(N, OpNo);
1196 break;
1197 case ISD::PATCHPOINT:
1198 Res = SoftenFloatOp_PATCHPOINT(N, OpNo);
1199 break;
1200 }
1201
1202 // If the result is null, the sub-method took care of registering results etc.
1203 if (!Res.getNode()) return false;
1204
1205 // If the result is N, the sub-method updated N in place. Tell the legalizer
1206 // core about this to re-analyze.
1207 if (Res.getNode() == N)
1208 return true;
1209
1210 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1211 "Invalid operand softening");
1212
1213 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1214 return false;
1215}
1216
1217SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1218 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
1219
1220 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
1221}
1222
1223SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1224 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1225 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1226 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1227 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1228 N->getOpcode() == ISD::FP_TO_BF16 ||
1229 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1230 N->getOpcode() == ISD::STRICT_FP_ROUND);
1231
1232 bool IsStrict = N->isStrictFPOpcode();
1233 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1234 EVT SVT = Op.getValueType();
1235 EVT RVT = N->getValueType(ResNo: 0);
1236 EVT FloatRVT = RVT;
1237 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1238 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1239 FloatRVT = MVT::f16;
1240 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1241 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1242 FloatRVT = MVT::bf16;
1243
1244 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
1245 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1246
1247 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1248 Op = GetSoftenedFloat(Op);
1249 TargetLowering::MakeLibCallOptions CallOptions;
1250 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1251 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op,
1252 CallOptions, dl: SDLoc(N),
1253 Chain);
1254 if (IsStrict) {
1255 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1256 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1257 return SDValue();
1258 }
1259 return Tmp.first;
1260}
1261
1262SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1263 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1264 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1265
1266 EVT VT = NewLHS.getValueType();
1267 NewLHS = GetSoftenedFloat(Op: NewLHS);
1268 NewRHS = GetSoftenedFloat(Op: NewRHS);
1269 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1270 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1271
1272 // If softenSetCCOperands returned a scalar, we need to compare the result
1273 // against zero to select between true and false values.
1274 if (!NewRHS.getNode()) {
1275 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1276 CCCode = ISD::SETNE;
1277 }
1278
1279 // Update N to have the operands specified.
1280 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1281 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1282 Op5: N->getOperand(Num: 4)),
1283 0);
1284}
1285
1286// Even if the result type is legal, no libcall may exactly match. (e.g. We
1287// don't have FP-i8 conversions) This helper method looks for an appropriate
1288// promoted libcall.
1289static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1290 bool Signed) {
1291 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1292 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1293 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1294 ++IntVT) {
1295 Promoted = (MVT::SimpleValueType)IntVT;
1296 // The type needs to big enough to hold the result.
1297 if (Promoted.bitsGE(VT: RetVT))
1298 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1299 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1300 }
1301 return LC;
1302}
1303
1304SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1305 bool IsStrict = N->isStrictFPOpcode();
1306 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1307 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1308
1309 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1310 EVT SVT = Op.getValueType();
1311 EVT RVT = N->getValueType(ResNo: 0);
1312 EVT NVT = EVT();
1313 SDLoc dl(N);
1314
1315 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1316 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1317 // match, eg. we don't have fp -> i8 conversions.
1318 // Look for an appropriate libcall.
1319 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1320 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1321 "Unsupported FP_TO_XINT!");
1322
1323 Op = GetSoftenedFloat(Op);
1324 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1325 TargetLowering::MakeLibCallOptions CallOptions;
1326 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1327 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1328 CallOptions, dl, Chain);
1329
1330 // Truncate the result if the libcall returns a larger type.
1331 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1332
1333 if (!IsStrict)
1334 return Res;
1335
1336 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1337 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1338 return SDValue();
1339}
1340
1341SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1342 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1343 return Res;
1344}
1345
1346SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1347 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1348 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1349
1350 EVT VT = NewLHS.getValueType();
1351 NewLHS = GetSoftenedFloat(Op: NewLHS);
1352 NewRHS = GetSoftenedFloat(Op: NewRHS);
1353 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1354 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1355
1356 // If softenSetCCOperands returned a scalar, we need to compare the result
1357 // against zero to select between true and false values.
1358 if (!NewRHS.getNode()) {
1359 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1360 CCCode = ISD::SETNE;
1361 }
1362
1363 // Update N to have the operands specified.
1364 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1365 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1366 Op5: DAG.getCondCode(Cond: CCCode)),
1367 0);
1368}
1369
1370SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1371 bool IsStrict = N->isStrictFPOpcode();
1372 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1373 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1374 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1375 ISD::CondCode CCCode =
1376 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1377
1378 EVT VT = Op0.getValueType();
1379 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1380 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1381 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1382 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1383
1384 // Update N to have the operands specified.
1385 if (NewRHS.getNode()) {
1386 if (IsStrict)
1387 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1388 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1389 else
1390 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1391 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1392 }
1393
1394 // Otherwise, softenSetCCOperands returned a scalar, use it.
1395 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1396 "Unexpected setcc expansion!");
1397
1398 if (IsStrict) {
1399 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1400 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1401 return SDValue();
1402 }
1403 return NewLHS;
1404}
1405
1406SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1407 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1408 assert(OpNo == 1 && "Can only soften the stored value!");
1409 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1410 SDValue Val = ST->getValue();
1411 SDLoc dl(N);
1412
1413 if (ST->isTruncatingStore())
1414 // Do an FP_ROUND followed by a non-truncating store.
1415 Val = BitConvertToInteger(
1416 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1417 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1418 else
1419 Val = GetSoftenedFloat(Op: Val);
1420
1421 return DAG.getStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(),
1422 MMO: ST->getMemOperand());
1423}
1424
1425SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1426 assert(OpNo == 1 && "Can only soften the stored value!");
1427 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
1428 SDValue Val = ST->getVal();
1429 EVT VT = Val.getValueType();
1430 SDLoc dl(N);
1431
1432 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1433
1434 SDValue NewVal = GetSoftenedFloat(Op: Val);
1435 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: VT, Chain: ST->getChain(), Ptr: NewVal,
1436 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
1437}
1438
1439SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1440 SDValue LHS = N->getOperand(Num: 0);
1441 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1442 SDLoc dl(N);
1443
1444 EVT LVT = LHS.getValueType();
1445 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1446 EVT RVT = RHS.getValueType();
1447
1448 unsigned LSize = LVT.getSizeInBits();
1449 unsigned RSize = RVT.getSizeInBits();
1450
1451 // Shift right or sign-extend it if the two operands have different types.
1452 int SizeDiff = RSize - LSize;
1453 if (SizeDiff > 0) {
1454 RHS =
1455 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1456 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1457 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1458 DL: DAG.getDataLayout())));
1459 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1460 } else if (SizeDiff < 0) {
1461 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1462 RHS =
1463 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1464 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1465 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1466 DL: DAG.getDataLayout())));
1467 }
1468
1469 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1470 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1471}
1472
1473SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1474 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1475 bool IsStrict = N->isStrictFPOpcode();
1476 unsigned Offset = IsStrict ? 1 : 0;
1477 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1478 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1479 TargetLowering::MakeLibCallOptions CallOptions;
1480 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1481 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
1482 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1483 CallOptions, dl: SDLoc(N),
1484 Chain);
1485 if (IsStrict) {
1486 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1487 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1488 return SDValue();
1489 }
1490
1491 return Tmp.first;
1492}
1493
1494SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1495 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1496 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1497 Call_F32: RTLIB::LROUND_F32,
1498 Call_F64: RTLIB::LROUND_F64,
1499 Call_F80: RTLIB::LROUND_F80,
1500 Call_F128: RTLIB::LROUND_F128,
1501 Call_PPCF128: RTLIB::LROUND_PPCF128));
1502}
1503
1504SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1505 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1506 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1507 Call_F32: RTLIB::LLROUND_F32,
1508 Call_F64: RTLIB::LLROUND_F64,
1509 Call_F80: RTLIB::LLROUND_F80,
1510 Call_F128: RTLIB::LLROUND_F128,
1511 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1512}
1513
1514SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1515 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1516 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1517 Call_F32: RTLIB::LRINT_F32,
1518 Call_F64: RTLIB::LRINT_F64,
1519 Call_F80: RTLIB::LRINT_F80,
1520 Call_F128: RTLIB::LRINT_F128,
1521 Call_PPCF128: RTLIB::LRINT_PPCF128));
1522}
1523
1524SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1525 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1526 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1527 Call_F32: RTLIB::LLRINT_F32,
1528 Call_F64: RTLIB::LLRINT_F64,
1529 Call_F80: RTLIB::LLRINT_F80,
1530 Call_F128: RTLIB::LLRINT_F128,
1531 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1532}
1533
1534SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1535 SDValue Op1 = BitConvertToInteger(Op: N->getOperand(Num: 1));
1536 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1537 N1: N->getOperand(Num: 0), N2: Op1);
1538}
1539
1540SDValue DAGTypeLegalizer::SoftenFloatOp_STACKMAP(SDNode *N, unsigned OpNo) {
1541 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
1542 SmallVector<SDValue> NewOps(N->ops());
1543 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1544 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1545}
1546
1547SDValue DAGTypeLegalizer::SoftenFloatOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
1548 assert(OpNo >= 7);
1549 SmallVector<SDValue> NewOps(N->ops());
1550 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1551 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1552}
1553
1554//===----------------------------------------------------------------------===//
1555// Float Result Expansion
1556//===----------------------------------------------------------------------===//
1557
1558/// ExpandFloatResult - This method is called when the specified result of the
1559/// specified node is found to need expansion. At this point, the node may also
1560/// have invalid operands or may have other results that need promotion, we just
1561/// know that (at least) one result needs expansion.
1562void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1563 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1564 SDValue Lo, Hi;
1565 Lo = Hi = SDValue();
1566
1567 // See if the target wants to custom expand this node.
1568 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1569 return;
1570
1571 switch (N->getOpcode()) {
1572 default:
1573#ifndef NDEBUG
1574 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1575 N->dump(&DAG); dbgs() << "\n";
1576#endif
1577 report_fatal_error(reason: "Do not know how to expand the result of this "
1578 "operator!");
1579 // clang-format off
1580 case ISD::POISON:
1581 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1582 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1583 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1584
1585 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1586 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1587 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1588 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1589 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1590 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1591
1592 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1593 case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
1594 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1595 case ISD::STRICT_FMINNUM:
1596 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1597 case ISD::STRICT_FMAXNUM:
1598 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1599 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1600 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1601 case ISD::STRICT_FADD:
1602 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1603 case ISD::STRICT_FACOS:
1604 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1605 case ISD::STRICT_FASIN:
1606 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1607 case ISD::STRICT_FATAN:
1608 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1609 case ISD::STRICT_FATAN2:
1610 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1611 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1612 case ISD::STRICT_FCEIL:
1613 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1614 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1615 case ISD::STRICT_FCOS:
1616 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1617 case ISD::STRICT_FCOSH:
1618 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1619 case ISD::STRICT_FDIV:
1620 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1621 case ISD::STRICT_FEXP:
1622 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1623 case ISD::STRICT_FEXP2:
1624 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1625 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1626 case ISD::STRICT_FFLOOR:
1627 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1628 case ISD::STRICT_FLOG:
1629 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1630 case ISD::STRICT_FLOG2:
1631 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1632 case ISD::STRICT_FLOG10:
1633 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1634 case ISD::STRICT_FMA:
1635 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1636 case ISD::STRICT_FMUL:
1637 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1638 case ISD::STRICT_FNEARBYINT:
1639 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1640 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1641 case ISD::STRICT_FP_EXTEND:
1642 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1643 case ISD::STRICT_FPOW:
1644 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1645 case ISD::STRICT_FPOWI:
1646 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1647 case ISD::FLDEXP:
1648 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1649 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1650 case ISD::STRICT_FRINT:
1651 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1652 case ISD::STRICT_FROUND:
1653 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1654 case ISD::STRICT_FROUNDEVEN:
1655 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1656 case ISD::STRICT_FSIN:
1657 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1658 case ISD::STRICT_FSINH:
1659 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1660 case ISD::STRICT_FSQRT:
1661 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1662 case ISD::STRICT_FSUB:
1663 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1664 case ISD::STRICT_FTAN:
1665 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1666 case ISD::STRICT_FTANH:
1667 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1668 case ISD::STRICT_FTRUNC:
1669 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1670 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1671 case ISD::STRICT_SINT_TO_FP:
1672 case ISD::STRICT_UINT_TO_FP:
1673 case ISD::SINT_TO_FP:
1674 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1675 case ISD::STRICT_FREM:
1676 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1677 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1678 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1679 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1680 // clang-format on
1681 }
1682
1683 // If Lo/Hi is null, the sub-method took care of registering results etc.
1684 if (Lo.getNode())
1685 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1686}
1687
1688void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1689 SDValue &Hi) {
1690 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1691 assert(NVT.getSizeInBits() == 64 &&
1692 "Do not know how to expand this float constant!");
1693 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1694 SDLoc dl(N);
1695 const fltSemantics &Sem = NVT.getFltSemantics();
1696 Lo = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 64)), DL: dl, VT: NVT);
1697 Hi = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 0)), DL: dl, VT: NVT);
1698}
1699
1700void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1701 SDValue &Lo, SDValue &Hi) {
1702 bool IsStrict = N->isStrictFPOpcode();
1703 unsigned Offset = IsStrict ? 1 : 0;
1704 SDValue Op = N->getOperand(Num: 0 + Offset);
1705 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1706 TargetLowering::MakeLibCallOptions CallOptions;
1707 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1708 Ops: Op, CallOptions, dl: SDLoc(N),
1709 Chain);
1710 if (IsStrict)
1711 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1712 GetPairElements(Pair: Tmp.first, Lo, Hi);
1713}
1714
1715void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1716 SDValue &Lo, SDValue &Hi) {
1717 bool IsStrict = N->isStrictFPOpcode();
1718 unsigned Offset = IsStrict ? 1 : 0;
1719 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1720 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1721 TargetLowering::MakeLibCallOptions CallOptions;
1722 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1723 Ops, CallOptions, dl: SDLoc(N),
1724 Chain);
1725 if (IsStrict)
1726 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1727 GetPairElements(Pair: Tmp.first, Lo, Hi);
1728}
1729
1730void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1731 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT: N->getValueType(ResNo: 0)),
1732 /*CallRetResNo=*/0);
1733}
1734
1735void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1736 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(RetVT: N->getValueType(ResNo: 0)));
1737}
1738
1739void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1740 ExpandFloatRes_UnaryWithTwoFPResults(N,
1741 LC: RTLIB::getSINCOSPI(RetVT: N->getValueType(ResNo: 0)));
1742}
1743
1744void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1745 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1746 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1747 SmallVector<SDValue> Results;
1748 TLI.expandMultipleResultFPLibCall(DAG, LC, Node: N, Results, CallRetResNo);
1749 for (auto [ResNo, Res] : enumerate(First&: Results)) {
1750 SDValue Lo, Hi;
1751 GetPairElements(Pair: Res, Lo, Hi);
1752 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1753 }
1754}
1755
1756void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1757 SDValue &Hi) {
1758 assert(N->getValueType(0) == MVT::ppcf128 &&
1759 "Logic only correct for ppcf128!");
1760 SDLoc dl(N);
1761 SDValue Tmp;
1762 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1763 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1764 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1765 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1766 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1767 Cond: ISD::SETEQ);
1768}
1769
1770void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1771 SDValue &Hi) {
1772 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1773 Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
1774 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
1775 Call_PPCF128: RTLIB::FMIN_PPCF128), Lo, Hi);
1776}
1777
1778void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1779 SDValue &Hi) {
1780 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1781 Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
1782 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
1783 Call_PPCF128: RTLIB::FMAX_PPCF128), Lo, Hi);
1784}
1785
1786void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1787 SDValue &Hi) {
1788 ExpandFloatRes_Binary(
1789 N,
1790 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_NUM_F32,
1791 Call_F64: RTLIB::FMINIMUM_NUM_F64, Call_F80: RTLIB::FMINIMUM_NUM_F80,
1792 Call_F128: RTLIB::FMINIMUM_NUM_F128, Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128),
1793 Lo, Hi);
1794}
1795
1796void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1797 SDValue &Hi) {
1798 ExpandFloatRes_Binary(
1799 N,
1800 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_NUM_F32,
1801 Call_F64: RTLIB::FMAXIMUM_NUM_F64, Call_F80: RTLIB::FMAXIMUM_NUM_F80,
1802 Call_F128: RTLIB::FMAXIMUM_NUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128),
1803 Lo, Hi);
1804}
1805
1806void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1807 SDValue &Hi) {
1808 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1809 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1810 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1811 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1812}
1813
1814void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1815 SDValue &Hi) {
1816 ExpandFloatRes_Unary(N,
1817 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ACOS_F32,
1818 Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
1819 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128),
1820 Lo, Hi);
1821}
1822
1823void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1824 SDValue &Hi) {
1825 ExpandFloatRes_Unary(N,
1826 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ASIN_F32,
1827 Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
1828 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128),
1829 Lo, Hi);
1830}
1831
1832void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1833 SDValue &Hi) {
1834 ExpandFloatRes_Unary(N,
1835 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN_F32,
1836 Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
1837 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128),
1838 Lo, Hi);
1839}
1840
1841void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1842 SDValue &Hi) {
1843 ExpandFloatRes_Binary(N,
1844 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN2_F32,
1845 Call_F64: RTLIB::ATAN2_F64, Call_F80: RTLIB::ATAN2_F80,
1846 Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128),
1847 Lo, Hi);
1848}
1849
1850void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1851 SDValue &Hi) {
1852 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::CBRT_F32,
1853 Call_F64: RTLIB::CBRT_F64, Call_F80: RTLIB::CBRT_F80,
1854 Call_F128: RTLIB::CBRT_F128,
1855 Call_PPCF128: RTLIB::CBRT_PPCF128), Lo, Hi);
1856}
1857
1858void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1859 SDValue &Lo, SDValue &Hi) {
1860 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1861 Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
1862 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
1863 Call_PPCF128: RTLIB::CEIL_PPCF128), Lo, Hi);
1864}
1865
1866void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1867 SDValue &Lo, SDValue &Hi) {
1868 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1869 Call_F32: RTLIB::COPYSIGN_F32,
1870 Call_F64: RTLIB::COPYSIGN_F64,
1871 Call_F80: RTLIB::COPYSIGN_F80,
1872 Call_F128: RTLIB::COPYSIGN_F128,
1873 Call_PPCF128: RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1874}
1875
1876void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1877 SDValue &Lo, SDValue &Hi) {
1878 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1879 Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
1880 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
1881 Call_PPCF128: RTLIB::COS_PPCF128), Lo, Hi);
1882}
1883
1884void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1885 SDValue &Hi) {
1886 ExpandFloatRes_Unary(N,
1887 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::COSH_F32,
1888 Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
1889 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128),
1890 Lo, Hi);
1891}
1892
1893void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1894 SDValue &Hi) {
1895 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1896 Call_F32: RTLIB::DIV_F32,
1897 Call_F64: RTLIB::DIV_F64,
1898 Call_F80: RTLIB::DIV_F80,
1899 Call_F128: RTLIB::DIV_F128,
1900 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1901}
1902
1903void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1904 SDValue &Lo, SDValue &Hi) {
1905 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1906 Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64,
1907 Call_F80: RTLIB::EXP_F80, Call_F128: RTLIB::EXP_F128,
1908 Call_PPCF128: RTLIB::EXP_PPCF128), Lo, Hi);
1909}
1910
1911void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1912 SDValue &Lo, SDValue &Hi) {
1913 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1914 Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64,
1915 Call_F80: RTLIB::EXP2_F80, Call_F128: RTLIB::EXP2_F128,
1916 Call_PPCF128: RTLIB::EXP2_PPCF128), Lo, Hi);
1917}
1918
1919void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1920 SDValue &Hi) {
1921 ExpandFloatRes_Unary(N,
1922 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32,
1923 Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
1924 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128),
1925 Lo, Hi);
1926}
1927
1928void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1929 SDValue &Lo, SDValue &Hi) {
1930 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1931 Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
1932 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
1933 Call_PPCF128: RTLIB::FLOOR_PPCF128), Lo, Hi);
1934}
1935
1936void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1937 SDValue &Lo, SDValue &Hi) {
1938 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1939 Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64,
1940 Call_F80: RTLIB::LOG_F80, Call_F128: RTLIB::LOG_F128,
1941 Call_PPCF128: RTLIB::LOG_PPCF128), Lo, Hi);
1942}
1943
1944void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1945 SDValue &Lo, SDValue &Hi) {
1946 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1947 Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64,
1948 Call_F80: RTLIB::LOG2_F80, Call_F128: RTLIB::LOG2_F128,
1949 Call_PPCF128: RTLIB::LOG2_PPCF128), Lo, Hi);
1950}
1951
1952void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1953 SDValue &Lo, SDValue &Hi) {
1954 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1955 Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64,
1956 Call_F80: RTLIB::LOG10_F80, Call_F128: RTLIB::LOG10_F128,
1957 Call_PPCF128: RTLIB::LOG10_PPCF128), Lo, Hi);
1958}
1959
1960void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1961 SDValue &Hi) {
1962 bool IsStrict = N->isStrictFPOpcode();
1963 unsigned Offset = IsStrict ? 1 : 0;
1964 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1965 N->getOperand(Num: 2 + Offset) };
1966 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1967 TargetLowering::MakeLibCallOptions CallOptions;
1968 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1969 Call_F32: RTLIB::FMA_F32,
1970 Call_F64: RTLIB::FMA_F64,
1971 Call_F80: RTLIB::FMA_F80,
1972 Call_F128: RTLIB::FMA_F128,
1973 Call_PPCF128: RTLIB::FMA_PPCF128),
1974 RetVT: N->getValueType(ResNo: 0), Ops, CallOptions,
1975 dl: SDLoc(N), Chain);
1976 if (IsStrict)
1977 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1978 GetPairElements(Pair: Tmp.first, Lo, Hi);
1979}
1980
1981void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1982 SDValue &Hi) {
1983 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1984 Call_F32: RTLIB::MUL_F32,
1985 Call_F64: RTLIB::MUL_F64,
1986 Call_F80: RTLIB::MUL_F80,
1987 Call_F128: RTLIB::MUL_F128,
1988 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1989}
1990
1991void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1992 SDValue &Lo, SDValue &Hi) {
1993 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1994 Call_F32: RTLIB::NEARBYINT_F32,
1995 Call_F64: RTLIB::NEARBYINT_F64,
1996 Call_F80: RTLIB::NEARBYINT_F80,
1997 Call_F128: RTLIB::NEARBYINT_F128,
1998 Call_PPCF128: RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1999}
2000
2001void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
2002 SDValue &Hi) {
2003 SDLoc dl(N);
2004 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
2005 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
2006 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
2007}
2008
2009void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
2010 SDValue &Hi) {
2011 // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
2012 SDLoc dl(N);
2013 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
2014}
2015
2016void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
2017 SDValue &Hi) {
2018 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
2019 SDLoc dl(N);
2020 bool IsStrict = N->isStrictFPOpcode();
2021
2022 SDValue Chain;
2023 if (IsStrict) {
2024 // If the expanded type is the same as the input type, just bypass the node.
2025 if (NVT == N->getOperand(Num: 1).getValueType()) {
2026 Hi = N->getOperand(Num: 1);
2027 Chain = N->getOperand(Num: 0);
2028 } else {
2029 // Other we need to extend.
2030 Hi = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: { NVT, MVT::Other },
2031 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
2032 Chain = Hi.getValue(R: 1);
2033 }
2034 } else {
2035 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
2036 }
2037
2038 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2039
2040 if (IsStrict)
2041 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2042}
2043
2044void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
2045 SDValue &Lo, SDValue &Hi) {
2046 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2047 Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64,
2048 Call_F80: RTLIB::POW_F80, Call_F128: RTLIB::POW_F128,
2049 Call_PPCF128: RTLIB::POW_PPCF128), Lo, Hi);
2050}
2051
2052void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
2053 SDValue &Lo, SDValue &Hi) {
2054 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
2055}
2056
2057void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
2058 SDValue &Hi) {
2059 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
2060}
2061
2062void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
2063 SDValue &Lo, SDValue &Hi) {
2064 assert(N->getValueType(0) == MVT::ppcf128 &&
2065 "Logic only correct for ppcf128!");
2066
2067 SDLoc dl(N);
2068 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
2069 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
2070 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
2071}
2072
2073void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
2074 SDValue &Lo, SDValue &Hi) {
2075 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2076 Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
2077 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
2078 Call_PPCF128: RTLIB::REM_PPCF128), Lo, Hi);
2079}
2080
2081void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
2082 SDValue &Lo, SDValue &Hi) {
2083 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2084 Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
2085 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
2086 Call_PPCF128: RTLIB::RINT_PPCF128), Lo, Hi);
2087}
2088
2089void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
2090 SDValue &Lo, SDValue &Hi) {
2091 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2092 Call_F32: RTLIB::ROUND_F32,
2093 Call_F64: RTLIB::ROUND_F64,
2094 Call_F80: RTLIB::ROUND_F80,
2095 Call_F128: RTLIB::ROUND_F128,
2096 Call_PPCF128: RTLIB::ROUND_PPCF128), Lo, Hi);
2097}
2098
2099void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
2100 SDValue &Lo, SDValue &Hi) {
2101 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2102 Call_F32: RTLIB::ROUNDEVEN_F32,
2103 Call_F64: RTLIB::ROUNDEVEN_F64,
2104 Call_F80: RTLIB::ROUNDEVEN_F80,
2105 Call_F128: RTLIB::ROUNDEVEN_F128,
2106 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
2107}
2108
2109void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
2110 SDValue &Lo, SDValue &Hi) {
2111 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2112 Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
2113 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
2114 Call_PPCF128: RTLIB::SIN_PPCF128), Lo, Hi);
2115}
2116
2117void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
2118 SDValue &Hi) {
2119 ExpandFloatRes_Unary(N,
2120 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::SINH_F32,
2121 Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
2122 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128),
2123 Lo, Hi);
2124}
2125
2126void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
2127 SDValue &Lo, SDValue &Hi) {
2128 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2129 Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
2130 Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
2131 Call_PPCF128: RTLIB::SQRT_PPCF128), Lo, Hi);
2132}
2133
2134void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
2135 SDValue &Hi) {
2136 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2137 Call_F32: RTLIB::SUB_F32,
2138 Call_F64: RTLIB::SUB_F64,
2139 Call_F80: RTLIB::SUB_F80,
2140 Call_F128: RTLIB::SUB_F128,
2141 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
2142}
2143
2144void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
2145 SDValue &Hi) {
2146 ExpandFloatRes_Unary(N,
2147 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TAN_F32,
2148 Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
2149 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128),
2150 Lo, Hi);
2151}
2152
2153void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
2154 SDValue &Hi) {
2155 ExpandFloatRes_Unary(N,
2156 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TANH_F32,
2157 Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
2158 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128),
2159 Lo, Hi);
2160}
2161
2162void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
2163 SDValue &Lo, SDValue &Hi) {
2164 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2165 Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
2166 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
2167 Call_PPCF128: RTLIB::TRUNC_PPCF128), Lo, Hi);
2168}
2169
2170void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
2171 SDValue &Hi) {
2172 if (ISD::isNormalLoad(N)) {
2173 ExpandRes_NormalLoad(N, Lo, Hi);
2174 return;
2175 }
2176
2177 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2178 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
2179 SDValue Chain = LD->getChain();
2180 SDValue Ptr = LD->getBasePtr();
2181 SDLoc dl(N);
2182
2183 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
2184 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2185 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2186
2187 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
2188 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
2189
2190 // Remember the chain.
2191 Chain = Hi.getValue(R: 1);
2192
2193 // The low part is zero.
2194 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2195
2196 // Modified the chain - switch anything that used the old chain to use the
2197 // new one.
2198 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
2199}
2200
2201void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2202 SDValue &Hi) {
2203 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2204 EVT VT = N->getValueType(ResNo: 0);
2205 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2206 bool Strict = N->isStrictFPOpcode();
2207 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
2208 EVT SrcVT = Src.getValueType();
2209 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2210 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2211 SDLoc dl(N);
2212 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
2213
2214 // TODO: Any other flags to propagate?
2215 SDNodeFlags Flags;
2216 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2217
2218 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2219 // When promoting partial word types to i32 we must honor the signedness,
2220 // though.
2221 if (SrcVT.bitsLE(VT: MVT::i32)) {
2222 // The integer can be represented exactly in an f64.
2223 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2224 if (Strict) {
2225 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
2226 Ops: {Chain, Src}, Flags);
2227 Chain = Hi.getValue(R: 1);
2228 } else
2229 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
2230 } else {
2231 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2232 if (SrcVT.bitsLE(VT: MVT::i64)) {
2233 Src = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
2234 VT: MVT::i64, Operand: Src);
2235 LC = RTLIB::SINTTOFP_I64_PPCF128;
2236 } else if (SrcVT.bitsLE(VT: MVT::i128)) {
2237 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i128, Operand: Src);
2238 LC = RTLIB::SINTTOFP_I128_PPCF128;
2239 }
2240 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2241
2242 TargetLowering::MakeLibCallOptions CallOptions;
2243 CallOptions.setIsSigned(true);
2244 std::pair<SDValue, SDValue> Tmp =
2245 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
2246 if (Strict)
2247 Chain = Tmp.second;
2248 GetPairElements(Pair: Tmp.first, Lo, Hi);
2249 }
2250
2251 // No need to complement for unsigned 32-bit integers
2252 if (isSigned || SrcVT.bitsLE(VT: MVT::i32)) {
2253 if (Strict)
2254 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2255
2256 return;
2257 }
2258
2259 // Unsigned - fix up the SINT_TO_FP value just calculated.
2260 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2261 // keep semantics correctness if the integer is not exactly representable
2262 // here. See ExpandLegalINT_TO_FP.
2263 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
2264 SrcVT = Src.getValueType();
2265
2266 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2267 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2268 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2269 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2270 ArrayRef<uint64_t> Parts;
2271
2272 switch (SrcVT.getSimpleVT().SimpleTy) {
2273 default:
2274 llvm_unreachable("Unsupported UINT_TO_FP!");
2275 case MVT::i32:
2276 Parts = TwoE32;
2277 break;
2278 case MVT::i64:
2279 Parts = TwoE64;
2280 break;
2281 case MVT::i128:
2282 Parts = TwoE128;
2283 break;
2284 }
2285
2286 // TODO: Are there other fast-math-flags to propagate to this FADD?
2287 SDValue NewLo = DAG.getConstantFP(
2288 Val: APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), DL: dl, VT: MVT::ppcf128);
2289 if (Strict) {
2290 Lo = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other),
2291 Ops: {Chain, Hi, NewLo}, Flags);
2292 Chain = Lo.getValue(R: 1);
2293 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2294 } else
2295 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
2296 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
2297 True: Lo, False: Hi, Cond: ISD::SETLT);
2298 GetPairElements(Pair: Lo, Lo, Hi);
2299}
2300
2301
2302//===----------------------------------------------------------------------===//
2303// Float Operand Expansion
2304//===----------------------------------------------------------------------===//
2305
2306/// ExpandFloatOperand - This method is called when the specified operand of the
2307/// specified node is found to need expansion. At this point, all of the result
2308/// types of the node are known to be legal, but other operands of the node may
2309/// need promotion or expansion as well as the specified one.
2310bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2311 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2312 SDValue Res = SDValue();
2313
2314 // See if the target wants to custom expand this node.
2315 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2316 return false;
2317
2318 switch (N->getOpcode()) {
2319 default:
2320#ifndef NDEBUG
2321 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2322 N->dump(&DAG); dbgs() << "\n";
2323#endif
2324 report_fatal_error(reason: "Do not know how to expand this operator's operand!");
2325
2326 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2327 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2328 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2329
2330 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2331 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2332 case ISD::STRICT_FP_ROUND:
2333 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2334 case ISD::STRICT_FP_TO_SINT:
2335 case ISD::STRICT_FP_TO_UINT:
2336 case ISD::FP_TO_SINT:
2337 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2338 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2339 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2340 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2341 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2342 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2343 case ISD::STRICT_FSETCC:
2344 case ISD::STRICT_FSETCCS:
2345 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2346 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
2347 OpNo); break;
2348 }
2349
2350 // If the result is null, the sub-method took care of registering results etc.
2351 if (!Res.getNode()) return false;
2352
2353 // If the result is N, the sub-method updated N in place. Tell the legalizer
2354 // core about this.
2355 if (Res.getNode() == N)
2356 return true;
2357
2358 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2359 "Invalid operand expansion");
2360
2361 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2362 return false;
2363}
2364
2365/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2366/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2367void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2368 SDValue &NewRHS,
2369 ISD::CondCode &CCCode,
2370 const SDLoc &dl, SDValue &Chain,
2371 bool IsSignaling) {
2372 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2373 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
2374 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
2375
2376 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2377
2378 // FIXME: This generated code sucks. We want to generate
2379 // FCMPU crN, hi1, hi2
2380 // BNE crN, L:
2381 // FCMPU crN, lo1, lo2
2382 // The following can be improved, but not that much.
2383 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2384 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2385 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
2386 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2387 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
2388 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
2389 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2390 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2391 Tmp1 =
2392 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
2393 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
2394 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2395 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2396 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
2397 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2398 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2399 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
2400 NewRHS = SDValue(); // LHS is the result, not a compare.
2401 Chain = OutputChain;
2402}
2403
2404SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2405 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
2406 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
2407 SDValue Chain;
2408 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2409
2410 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2411 // against zero to select between true and false values.
2412 if (!NewRHS.getNode()) {
2413 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2414 CCCode = ISD::SETNE;
2415 }
2416
2417 // Update N to have the operands specified.
2418 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2419 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
2420 Op5: N->getOperand(Num: 4)), 0);
2421}
2422
2423SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2424 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2425 "Logic only correct for ppcf128!");
2426 SDValue Lo, Hi;
2427 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2428 // The ppcf128 value is providing only the sign; take it from the
2429 // higher-order double (which must have the larger magnitude).
2430 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2431 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2432}
2433
2434SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2435 bool IsStrict = N->isStrictFPOpcode();
2436 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2437 "Logic only correct for ppcf128!");
2438 SDValue Lo, Hi;
2439 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2440
2441 if (!IsStrict)
2442 // Round it the rest of the way (e.g. to f32) if needed.
2443 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2444 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2445
2446 // Eliminate the node if the input float type is the same as the output float
2447 // type.
2448 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2449 // Connect the output chain to the input chain, unlinking the node.
2450 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2451 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2452 return SDValue();
2453 }
2454
2455 SDValue Expansion = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
2456 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
2457 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
2458 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2459 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2460 return SDValue();
2461}
2462
2463SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2464 EVT RVT = N->getValueType(ResNo: 0);
2465 SDLoc dl(N);
2466
2467 bool IsStrict = N->isStrictFPOpcode();
2468 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2469 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2470 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2471 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2472
2473 EVT NVT;
2474 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2475 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2476 "Unsupported FP_TO_XINT!");
2477 TargetLowering::MakeLibCallOptions CallOptions;
2478 std::pair<SDValue, SDValue> Tmp =
2479 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2480 if (!IsStrict)
2481 return Tmp.first;
2482
2483 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2484 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2485 return SDValue();
2486}
2487
2488SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2489 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2490 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2491 SDValue Chain;
2492 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2493
2494 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2495 // against zero to select between true and false values.
2496 if (!NewRHS.getNode()) {
2497 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2498 CCCode = ISD::SETNE;
2499 }
2500
2501 // Update N to have the operands specified.
2502 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2503 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2504 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2505}
2506
2507SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2508 bool IsStrict = N->isStrictFPOpcode();
2509 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2510 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2511 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2512 ISD::CondCode CCCode =
2513 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2514 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2515 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2516
2517 // FloatExpandSetCCOperands always returned a scalar.
2518 assert(!NewRHS.getNode() && "Expect to return scalar");
2519 assert(NewLHS.getValueType() == N->getValueType(0) &&
2520 "Unexpected setcc expansion!");
2521 if (Chain) {
2522 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2523 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2524 return SDValue();
2525 }
2526 return NewLHS;
2527}
2528
2529SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2530 if (ISD::isNormalStore(N))
2531 return ExpandOp_NormalStore(N, OpNo);
2532
2533 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2534 assert(OpNo == 1 && "Can only expand the stored value so far");
2535 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2536
2537 SDValue Chain = ST->getChain();
2538 SDValue Ptr = ST->getBasePtr();
2539
2540 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2541 VT: ST->getValue().getValueType());
2542 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2543 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2544 (void)NVT;
2545
2546 SDValue Lo, Hi;
2547 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2548
2549 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2550 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2551}
2552
2553SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2554 EVT RVT = N->getValueType(ResNo: 0);
2555 EVT RetVT = N->getOperand(Num: 0).getValueType();
2556 TargetLowering::MakeLibCallOptions CallOptions;
2557 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2558 Call_F32: RTLIB::LROUND_F32,
2559 Call_F64: RTLIB::LROUND_F64,
2560 Call_F80: RTLIB::LROUND_F80,
2561 Call_F128: RTLIB::LROUND_F128,
2562 Call_PPCF128: RTLIB::LROUND_PPCF128),
2563 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2564}
2565
2566SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2567 EVT RVT = N->getValueType(ResNo: 0);
2568 EVT RetVT = N->getOperand(Num: 0).getValueType();
2569 TargetLowering::MakeLibCallOptions CallOptions;
2570 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2571 Call_F32: RTLIB::LLROUND_F32,
2572 Call_F64: RTLIB::LLROUND_F64,
2573 Call_F80: RTLIB::LLROUND_F80,
2574 Call_F128: RTLIB::LLROUND_F128,
2575 Call_PPCF128: RTLIB::LLROUND_PPCF128),
2576 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2577}
2578
2579SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2580 EVT RVT = N->getValueType(ResNo: 0);
2581 EVT RetVT = N->getOperand(Num: 0).getValueType();
2582 TargetLowering::MakeLibCallOptions CallOptions;
2583 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2584 Call_F32: RTLIB::LRINT_F32,
2585 Call_F64: RTLIB::LRINT_F64,
2586 Call_F80: RTLIB::LRINT_F80,
2587 Call_F128: RTLIB::LRINT_F128,
2588 Call_PPCF128: RTLIB::LRINT_PPCF128),
2589 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2590}
2591
2592SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2593 EVT RVT = N->getValueType(ResNo: 0);
2594 EVT RetVT = N->getOperand(Num: 0).getValueType();
2595 TargetLowering::MakeLibCallOptions CallOptions;
2596 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2597 Call_F32: RTLIB::LLRINT_F32,
2598 Call_F64: RTLIB::LLRINT_F64,
2599 Call_F80: RTLIB::LLRINT_F80,
2600 Call_F128: RTLIB::LLRINT_F128,
2601 Call_PPCF128: RTLIB::LLRINT_PPCF128),
2602 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2603}
2604
2605//===----------------------------------------------------------------------===//
2606// Float Operand Promotion
2607//===----------------------------------------------------------------------===//
2608//
2609
2610static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2611 if (OpVT == MVT::f16)
2612 return ISD::FP16_TO_FP;
2613 if (RetVT == MVT::f16)
2614 return ISD::FP_TO_FP16;
2615 if (OpVT == MVT::bf16)
2616 return ISD::BF16_TO_FP;
2617 if (RetVT == MVT::bf16)
2618 return ISD::FP_TO_BF16;
2619 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2620}
2621
2622static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2623 if (OpVT == MVT::f16)
2624 return ISD::STRICT_FP16_TO_FP;
2625 if (RetVT == MVT::f16)
2626 return ISD::STRICT_FP_TO_FP16;
2627 if (OpVT == MVT::bf16)
2628 return ISD::STRICT_BF16_TO_FP;
2629 if (RetVT == MVT::bf16)
2630 return ISD::STRICT_FP_TO_BF16;
2631 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2632}
2633
2634SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2635 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2636 SDLoc SL(N);
2637
2638 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2639 EVT CastVT = CastVal.getValueType();
2640
2641 SDValue NewAtomic
2642 = DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl: SL, MemVT: CastVT,
2643 VTList: DAG.getVTList(VT1: CastVT, VT2: MVT::Other),
2644 Ops: { AM->getChain(), AM->getBasePtr(), CastVal },
2645 MMO: AM->getMemOperand());
2646
2647 SDValue Result = NewAtomic;
2648
2649 // Legalize the chain result by replacing uses of the old value chain with the
2650 // new one
2651 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2652
2653 return Result;
2654}
2655
2656//===----------------------------------------------------------------------===//
2657// Half Result Soft Promotion
2658//===----------------------------------------------------------------------===//
2659
2660void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2661 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2662 N->dump(&DAG));
2663 SDValue R = SDValue();
2664
2665 // See if the target wants to custom expand this node.
2666 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2667 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2668 return;
2669 }
2670
2671 switch (N->getOpcode()) {
2672 default:
2673#ifndef NDEBUG
2674 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2675 N->dump(&DAG); dbgs() << "\n";
2676#endif
2677 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2678 "result!");
2679
2680 case ISD::ARITH_FENCE:
2681 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2682 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2683 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2684 case ISD::EXTRACT_VECTOR_ELT:
2685 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2686 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2687 case ISD::STRICT_FP_ROUND:
2688 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2689
2690 // Unary FP Operations
2691 case ISD::FACOS:
2692 case ISD::FASIN:
2693 case ISD::FATAN:
2694 case ISD::FCBRT:
2695 case ISD::FCEIL:
2696 case ISD::FCOS:
2697 case ISD::FCOSH:
2698 case ISD::FEXP:
2699 case ISD::FEXP2:
2700 case ISD::FEXP10:
2701 case ISD::FFLOOR:
2702 case ISD::FLOG:
2703 case ISD::FLOG2:
2704 case ISD::FLOG10:
2705 case ISD::FNEARBYINT:
2706 case ISD::FREEZE:
2707 case ISD::FRINT:
2708 case ISD::FROUND:
2709 case ISD::FROUNDEVEN:
2710 case ISD::FSIN:
2711 case ISD::FSINH:
2712 case ISD::FSQRT:
2713 case ISD::FTRUNC:
2714 case ISD::FTAN:
2715 case ISD::FTANH:
2716 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2717 case ISD::FABS:
2718 R = SoftPromoteHalfRes_FABS(N);
2719 break;
2720 case ISD::FNEG:
2721 R = SoftPromoteHalfRes_FNEG(N);
2722 break;
2723 case ISD::AssertNoFPClass:
2724 R = SoftPromoteHalfRes_AssertNoFPClass(N);
2725 break;
2726
2727 // Binary FP Operations
2728 case ISD::FADD:
2729 case ISD::FDIV:
2730 case ISD::FMAXIMUM:
2731 case ISD::FMINIMUM:
2732 case ISD::FMAXIMUMNUM:
2733 case ISD::FMINIMUMNUM:
2734 case ISD::FMAXNUM:
2735 case ISD::FMINNUM:
2736 case ISD::FMUL:
2737 case ISD::FPOW:
2738 case ISD::FATAN2:
2739 case ISD::FREM:
2740 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2741
2742 case ISD::FMA: // FMA is same as FMAD
2743 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2744
2745 case ISD::FPOWI:
2746 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2747
2748 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2749
2750 case ISD::FMODF:
2751 case ISD::FSINCOS:
2752 case ISD::FSINCOSPI:
2753 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
2754 break;
2755
2756 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2757 case ISD::ATOMIC_LOAD:
2758 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2759 break;
2760 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2761 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2762 case ISD::STRICT_SINT_TO_FP:
2763 case ISD::STRICT_UINT_TO_FP:
2764 case ISD::SINT_TO_FP:
2765 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2766 case ISD::POISON:
2767 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2768 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2769 case ISD::VECREDUCE_FADD:
2770 case ISD::VECREDUCE_FMUL:
2771 case ISD::VECREDUCE_FMIN:
2772 case ISD::VECREDUCE_FMAX:
2773 case ISD::VECREDUCE_FMAXIMUM:
2774 case ISD::VECREDUCE_FMINIMUM:
2775 R = SoftPromoteHalfRes_VECREDUCE(N);
2776 break;
2777 case ISD::VECREDUCE_SEQ_FADD:
2778 case ISD::VECREDUCE_SEQ_FMUL:
2779 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2780 break;
2781 }
2782
2783 if (R.getNode())
2784 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
2785}
2786
2787SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2788 return DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: MVT::i16,
2789 Operand: BitConvertToInteger(Op: N->getOperand(Num: 0)));
2790}
2791
2792SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2793 return BitConvertToInteger(Op: N->getOperand(Num: 0));
2794}
2795
2796SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2797 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
2798
2799 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2800 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
2801 VT: MVT::i16);
2802}
2803
2804SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2805 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2806 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
2807 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
2808 N2: N->getOperand(Num: 1));
2809}
2810
2811SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2812 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2813 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
2814 SDLoc dl(N);
2815
2816 EVT LVT = LHS.getValueType();
2817 EVT RVT = RHS.getValueType();
2818
2819 unsigned LSize = LVT.getSizeInBits();
2820 unsigned RSize = RVT.getSizeInBits();
2821
2822 // First get the sign bit of second operand.
2823 SDValue SignBit = DAG.getNode(
2824 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
2825 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
2826 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
2827 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
2828
2829 // Shift right or sign-extend it if the two operands have different types.
2830 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2831 if (SizeDiff > 0) {
2832 SignBit =
2833 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
2834 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
2835 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2836 DL: DAG.getDataLayout())));
2837 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
2838 } else if (SizeDiff < 0) {
2839 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
2840 SignBit =
2841 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
2842 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
2843 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2844 DL: DAG.getDataLayout())));
2845 }
2846
2847 // Clear the sign bit of the first operand.
2848 SDValue Mask = DAG.getNode(
2849 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
2850 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
2851 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
2852 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
2853 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
2854
2855 // Or the value with the sign bit.
2856 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
2857}
2858
2859SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2860 EVT OVT = N->getValueType(ResNo: 0);
2861 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2862 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2863 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2864 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2865 SDNodeFlags Flags = N->getFlags();
2866 SDLoc dl(N);
2867
2868 // Promote to the larger FP type.
2869 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2870 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2871 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2872 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
2873
2874 SDValue Res;
2875 if (OVT == MVT::f16) {
2876 // If f16 fma is not natively supported, the value must be promoted to an
2877 // f64 (and not to f32!) to prevent double rounding issues.
2878 SDValue A64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op0, Flags);
2879 SDValue B64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op1, Flags);
2880 SDValue C64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op2, Flags);
2881
2882 // Prefer a wide FMA node if available; otherwise expand to mul+add.
2883 SDValue WideRes;
2884 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), MVT::f64)) {
2885 WideRes = DAG.getNode(Opcode: ISD::FMA, DL: dl, VT: MVT::f64, N1: A64, N2: B64, N3: C64, Flags);
2886 } else {
2887 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f64, N1: A64, N2: B64, Flags);
2888 WideRes = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f64, N1: Mul, N2: C64, Flags);
2889 }
2890
2891 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: MVT::f64, RetVT: OVT), DL: dl, VT: MVT::i16,
2892 Operand: WideRes);
2893 }
2894
2895 Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2, Flags);
2896 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2897}
2898
2899SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2900 EVT OVT = N->getValueType(ResNo: 0);
2901 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2902 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2903 SDValue Op1 = N->getOperand(Num: 1);
2904 SDLoc dl(N);
2905
2906 // Promote to the larger FP type.
2907 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
2908
2909 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2910
2911 // Convert back to FP16 as an integer.
2912 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2913}
2914
2915SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2916 EVT OVT = N->getValueType(ResNo: 0);
2917 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2918 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2919 SDLoc dl(N);
2920
2921 // Promote to the larger FP type.
2922 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2923
2924 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
2925 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
2926
2927 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2928
2929 // Convert back to FP16 as an integer.
2930 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2931}
2932
2933SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
2934 EVT OVT = N->getValueType(ResNo: 0);
2935 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2936 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2937 SDLoc dl(N);
2938
2939 // Promote to the larger FP type.
2940 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2941 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Op);
2942
2943 // Convert back to FP16 as an integer.
2944 ISD::NodeType Truncate = GetPromotionOpcode(OpVT: NVT, RetVT: OVT);
2945 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
2946 ++ResNum) {
2947 SDValue Trunc = DAG.getNode(Opcode: Truncate, DL: dl, VT: MVT::i16, Operand: Res.getValue(R: ResNum));
2948 SetSoftPromotedHalf(Op: SDValue(N, ResNum), Result: Trunc);
2949 }
2950
2951 return SDValue();
2952}
2953
2954SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2955 EVT RVT = N->getValueType(ResNo: 0);
2956 bool IsStrict = N->isStrictFPOpcode();
2957 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2958 EVT SVT = Op.getValueType();
2959
2960 // If the input type needs to be softened, do that now so that call lowering
2961 // will see the f16 type.
2962 if (getTypeAction(VT: SVT) == TargetLowering::TypeSoftenFloat) {
2963 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: RVT);
2964 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
2965
2966 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2967 Op = GetSoftenedFloat(Op);
2968 TargetLowering::MakeLibCallOptions CallOptions;
2969 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
2970 std::pair<SDValue, SDValue> Tmp =
2971 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
2972 if (IsStrict)
2973 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2974 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: MVT::i16, Operand: Tmp.first);
2975 }
2976
2977 if (IsStrict) {
2978 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
2979 ResultTys: {MVT::i16, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
2980 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2981 return Res;
2982 }
2983
2984 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: MVT::i16,
2985 Operand: N->getOperand(Num: 0));
2986}
2987
2988SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2989 LoadSDNode *L = cast<LoadSDNode>(Val: N);
2990
2991 // Load the value as an integer value with the same number of bits.
2992 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2993 SDValue NewL =
2994 DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: MVT::i16,
2995 dl: SDLoc(N), Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
2996 PtrInfo: L->getPointerInfo(), MemVT: MVT::i16, Alignment: L->getBaseAlign(),
2997 MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo());
2998 // Legalize the chain result by replacing uses of the old value chain with the
2999 // new one
3000 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3001 return NewL;
3002}
3003
3004SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3005 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
3006
3007 // Load the value as an integer value with the same number of bits.
3008 SDValue NewL = DAG.getAtomic(
3009 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: MVT::i16, VTList: DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other),
3010 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
3011
3012 // Legalize the chain result by replacing uses of the old value chain with the
3013 // new one
3014 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3015 return NewL;
3016}
3017
3018SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3019 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3020 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3021 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1,
3022 RHS: Op2);
3023}
3024
3025SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3026 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3027 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
3028 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
3029 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
3030 N5: N->getOperand(Num: 4));
3031}
3032
3033SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3034 EVT OVT = N->getValueType(ResNo: 0);
3035 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3036 SDLoc dl(N);
3037
3038 if (N->isStrictFPOpcode()) {
3039 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
3040 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
3041 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: NVT, RetVT: OVT), DL: dl,
3042 ResultTys: {MVT::i16, MVT::Other}, Ops: {Op.getValue(R: 1), Op});
3043 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3044 return Op;
3045 }
3046
3047 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
3048
3049 // Round the value to the softened type.
3050 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3051}
3052
3053SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3054 return DAG.getUNDEF(VT: MVT::i16);
3055}
3056
3057SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3058 EVT OVT = N->getValueType(ResNo: 0);
3059 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3060 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3061 SDLoc dl(N);
3062
3063 // Promote to the larger FP type.
3064 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3065
3066 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
3067
3068 // Convert back to FP16 as an integer.
3069 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3070}
3071
3072SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FABS(SDNode *N) {
3073 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3074 SDLoc dl(N);
3075
3076 // Clear the sign bit.
3077 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i16, N1: Op,
3078 N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: MVT::i16));
3079}
3080
3081SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FNEG(SDNode *N) {
3082 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3083 SDLoc dl(N);
3084
3085 // Invert the sign bit.
3086 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i16, N1: Op,
3087 N2: DAG.getConstant(Val: 0x8000, DL: dl, VT: MVT::i16));
3088}
3089
3090SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
3091 return GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3092}
3093
3094SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3095 EVT OVT = N->getValueType(ResNo: 0);
3096 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3097 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3098 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3099 SDLoc dl(N);
3100
3101 // Promote to the larger FP type.
3102 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
3103 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3104 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3105
3106 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3107
3108 // Convert back to FP16 as an integer.
3109 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3110}
3111
3112SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3113 // Expand and soften recursively.
3114 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
3115 return SDValue();
3116}
3117
3118SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3119 // Expand and soften.
3120 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3121 return SDValue();
3122}
3123
3124//===----------------------------------------------------------------------===//
3125// Half Operand Soft Promotion
3126//===----------------------------------------------------------------------===//
3127
3128bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3129 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3130 N->dump(&DAG));
3131 SDValue Res = SDValue();
3132
3133 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
3134 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3135 return false;
3136 }
3137
3138 // Nodes that use a promotion-requiring floating point operand, but doesn't
3139 // produce a soft promotion-requiring floating point result, need to be
3140 // legalized to use the soft promoted float operand. Nodes that produce at
3141 // least one soft promotion-requiring floating point result have their
3142 // operands legalized as a part of PromoteFloatResult.
3143 switch (N->getOpcode()) {
3144 default:
3145 #ifndef NDEBUG
3146 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3147 N->dump(&DAG); dbgs() << "\n";
3148 #endif
3149 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3150 "operand!");
3151
3152 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3153 case ISD::FAKE_USE:
3154 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
3155 break;
3156 case ISD::FCOPYSIGN:
3157 Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo);
3158 break;
3159 case ISD::FP_TO_SINT:
3160 case ISD::FP_TO_UINT:
3161 case ISD::STRICT_FP_TO_SINT:
3162 case ISD::STRICT_FP_TO_UINT:
3163 case ISD::LLRINT:
3164 case ISD::LLROUND:
3165 case ISD::LRINT:
3166 case ISD::LROUND:
3167 case ISD::STRICT_LLRINT:
3168 case ISD::STRICT_LLROUND:
3169 case ISD::STRICT_LRINT:
3170 case ISD::STRICT_LROUND:
3171 Res = SoftPromoteHalfOp_Op0WithStrict(N);
3172 break;
3173 case ISD::FP_TO_SINT_SAT:
3174 case ISD::FP_TO_UINT_SAT:
3175 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3176 case ISD::STRICT_FP_EXTEND:
3177 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3178 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3179 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3180 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3181 case ISD::ATOMIC_STORE:
3182 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3183 break;
3184 case ISD::STACKMAP:
3185 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3186 break;
3187 case ISD::PATCHPOINT:
3188 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3189 break;
3190 }
3191
3192 if (!Res.getNode())
3193 return false;
3194
3195 assert(Res.getNode() != N && "Expected a new node!");
3196
3197 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3198 "Invalid operand expansion");
3199
3200 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3201 return false;
3202}
3203
3204SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3205 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3206
3207 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3208}
3209
3210SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3211 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3212 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3213 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::Other, N1: N->getOperand(Num: 0),
3214 N2: Op);
3215}
3216
3217SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3218 unsigned OpNo) {
3219 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3220 SDValue Op1 = N->getOperand(Num: 1);
3221 EVT RVT = Op1.getValueType();
3222 SDLoc dl(N);
3223
3224 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3225
3226 Op1 = GetSoftPromotedHalf(Op: Op1);
3227 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3228
3229 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3230 N2: Op1);
3231}
3232
3233SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3234 EVT RVT = N->getValueType(ResNo: 0);
3235 bool IsStrict = N->isStrictFPOpcode();
3236 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3237 EVT SVT = Op.getValueType();
3238 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3239
3240 if (IsStrict) {
3241 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
3242 ResultTys: {RVT, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
3243 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3244 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3245 return SDValue();
3246 }
3247
3248 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3249}
3250
3251SDValue DAGTypeLegalizer::SoftPromoteHalfOp_Op0WithStrict(SDNode *N) {
3252 EVT RVT = N->getValueType(ResNo: 0);
3253 bool IsStrict = N->isStrictFPOpcode();
3254 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3255 EVT SVT = Op.getValueType();
3256 SDLoc dl(N);
3257
3258 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3259 Op = GetSoftPromotedHalf(Op);
3260
3261 if (IsStrict) {
3262 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: dl, ResultTys: {NVT, MVT::Other},
3263 Ops: {N->getOperand(Num: 0), Op});
3264 Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {RVT, MVT::Other},
3265 Ops: {Op.getValue(R: 1), Op});
3266 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3267 ReplaceValueWith(From: SDValue(N, 0), To: Op);
3268 return SDValue();
3269 }
3270
3271 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3272 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: RVT, Operand: Res);
3273}
3274
3275SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3276 EVT RVT = N->getValueType(ResNo: 0);
3277 SDValue Op = N->getOperand(Num: 0);
3278 EVT SVT = Op.getValueType();
3279 SDLoc dl(N);
3280
3281 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3282
3283 Op = GetSoftPromotedHalf(Op);
3284
3285 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3286
3287 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3288 N2: N->getOperand(Num: 1));
3289}
3290
3291SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3292 unsigned OpNo) {
3293 assert(OpNo == 0 && "Can only soften the comparison values");
3294 SDValue Op0 = N->getOperand(Num: 0);
3295 SDValue Op1 = N->getOperand(Num: 1);
3296 SDLoc dl(N);
3297
3298 EVT SVT = Op0.getValueType();
3299 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3300
3301 Op0 = GetSoftPromotedHalf(Op: Op0);
3302 Op1 = GetSoftPromotedHalf(Op: Op1);
3303
3304 // Promote to the larger FP type.
3305 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3306 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3307 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3308
3309 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3310 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3311}
3312
3313SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3314 SDValue Op0 = N->getOperand(Num: 0);
3315 SDValue Op1 = N->getOperand(Num: 1);
3316 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3317 SDLoc dl(N);
3318
3319 EVT SVT = Op0.getValueType();
3320 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3321
3322 Op0 = GetSoftPromotedHalf(Op: Op0);
3323 Op1 = GetSoftPromotedHalf(Op: Op1);
3324
3325 // Promote to the larger FP type.
3326 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3327 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3328 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3329
3330 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3331}
3332
3333SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3334 assert(OpNo == 1 && "Can only soften the stored value!");
3335 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3336 SDValue Val = ST->getValue();
3337 SDLoc dl(N);
3338
3339 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3340 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3341 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3342 MMO: ST->getMemOperand());
3343}
3344
3345SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3346 unsigned OpNo) {
3347 assert(OpNo == 1 && "Can only soften the stored value!");
3348 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
3349 SDValue Val = ST->getVal();
3350 SDLoc dl(N);
3351
3352 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3353 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: Promoted.getValueType(),
3354 Chain: ST->getChain(), Ptr: Promoted, Val: ST->getBasePtr(),
3355 MMO: ST->getMemOperand());
3356}
3357
3358SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3359 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3360 SmallVector<SDValue> NewOps(N->ops());
3361 SDValue Op = N->getOperand(Num: OpNo);
3362 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3363 SDValue NewNode =
3364 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3365
3366 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3367 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3368
3369 return SDValue(); // Signal that we replaced the node ourselves.
3370}
3371
3372SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3373 unsigned OpNo) {
3374 assert(OpNo >= 7);
3375 SmallVector<SDValue> NewOps(N->ops());
3376 SDValue Op = N->getOperand(Num: OpNo);
3377 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3378 SDValue NewNode =
3379 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3380
3381 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3382 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3383
3384 return SDValue(); // Signal that we replaced the node ourselves.
3385}
3386