1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error(reason: "Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::FCANONICALIZE:
74 R = SoftenFloatRes_FCANONICALIZE(N); break;
75 case ISD::STRICT_FMINNUM:
76 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
77 case ISD::STRICT_FMAXNUM:
78 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
79 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
80 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
81 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
82 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
83 case ISD::STRICT_FADD:
84 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
85 case ISD::STRICT_FACOS:
86 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
87 case ISD::STRICT_FASIN:
88 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
89 case ISD::STRICT_FATAN:
90 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
91 case ISD::STRICT_FATAN2:
92 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
93 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
94 case ISD::STRICT_FCEIL:
95 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
96 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
97 case ISD::STRICT_FCOS:
98 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
99 case ISD::STRICT_FCOSH:
100 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
101 case ISD::STRICT_FDIV:
102 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
103 case ISD::STRICT_FEXP:
104 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
105 case ISD::STRICT_FEXP2:
106 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
107 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
108 case ISD::STRICT_FFLOOR:
109 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
110 case ISD::STRICT_FLOG:
111 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
112 case ISD::STRICT_FLOG2:
113 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
114 case ISD::STRICT_FLOG10:
115 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
116 case ISD::STRICT_FMA:
117 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
118 case ISD::STRICT_FMUL:
119 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
120 case ISD::STRICT_FNEARBYINT:
121 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
122 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
123 case ISD::STRICT_FP_EXTEND:
124 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
125 case ISD::STRICT_FP_ROUND:
126 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
127 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
128 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
129 case ISD::STRICT_FPOW:
130 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
131 case ISD::STRICT_FPOWI:
132 case ISD::FPOWI:
133 case ISD::FLDEXP:
134 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
135 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
136 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
137 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
138 case ISD::STRICT_FREM:
139 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
140 case ISD::STRICT_FRINT:
141 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
142 case ISD::STRICT_FROUND:
143 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
144 case ISD::STRICT_FROUNDEVEN:
145 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
146 case ISD::STRICT_FSIN:
147 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
148 case ISD::STRICT_FSINH:
149 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
150 case ISD::STRICT_FSQRT:
151 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
152 case ISD::STRICT_FSUB:
153 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
154 case ISD::STRICT_FTAN:
155 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
156 case ISD::STRICT_FTANH:
157 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
158 case ISD::STRICT_FTRUNC:
159 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
160 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
161 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
162 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
163 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
164 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
165 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
166 case ISD::STRICT_SINT_TO_FP:
167 case ISD::STRICT_UINT_TO_FP:
168 case ISD::SINT_TO_FP:
169 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
170 case ISD::POISON:
171 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
172 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
173 case ISD::AssertNoFPClass: R = GetSoftenedFloat(Op: N->getOperand(Num: 0)); break;
174 case ISD::VECREDUCE_FADD:
175 case ISD::VECREDUCE_FMUL:
176 case ISD::VECREDUCE_FMIN:
177 case ISD::VECREDUCE_FMAX:
178 case ISD::VECREDUCE_FMAXIMUM:
179 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
180 case ISD::VECREDUCE_SEQ_FADD:
181 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
182 // clang-format on
183 }
184
185 // If R is null, the sub-method took care of registering the result.
186 if (R.getNode()) {
187 assert(R.getNode() != N);
188 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
189 }
190}
191
192SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
193 bool IsStrict = N->isStrictFPOpcode();
194 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
195 unsigned Offset = IsStrict ? 1 : 0;
196 assert(N->getNumOperands() == (1 + Offset) &&
197 "Unexpected number of operands!");
198 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
199 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
200 TargetLowering::MakeLibCallOptions CallOptions;
201 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
202 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
203 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
204 CallOptions, dl: SDLoc(N),
205 Chain);
206 if (IsStrict)
207 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
208 return Tmp.first;
209}
210
211SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
212 bool IsStrict = N->isStrictFPOpcode();
213 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
214 unsigned Offset = IsStrict ? 1 : 0;
215 assert(N->getNumOperands() == (2 + Offset) &&
216 "Unexpected number of operands!");
217 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
218 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
219 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
220 TargetLowering::MakeLibCallOptions CallOptions;
221 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
222 N->getOperand(Num: 1 + Offset).getValueType() };
223 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
224 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
225 CallOptions, dl: SDLoc(N),
226 Chain);
227 if (IsStrict)
228 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
229 return Tmp.first;
230}
231
232SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
233 return BitConvertToInteger(Op: N->getOperand(Num: 0));
234}
235
236SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
237 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
238 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
239 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
240}
241
242SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
243 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
244 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
245 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
246 return NewFence;
247}
248
249SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
250 unsigned ResNo) {
251 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
252 return BitConvertToInteger(Op);
253}
254
255SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
256 // Convert the inputs to integers, and build a new pair out of them.
257 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
258 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
259 VT: N->getValueType(ResNo: 0)),
260 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
261 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
262}
263
264SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
265 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
266 // In ppcf128, the high 64 bits are always first in memory regardless
267 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
268 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
269 // way. However, APInt's are serialized in an Endian-sensitive fashion,
270 // so on big-Endian targets, the two doubles are output in the wrong
271 // order. Fix this by manually flipping the order of the high 64 bits
272 // and the low 64 bits here.
273 if (DAG.getDataLayout().isBigEndian() &&
274 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
275 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
276 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
277 APInt Val(128, words);
278 return DAG.getConstant(Val, DL: SDLoc(CN),
279 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
280 VT: CN->getValueType(ResNo: 0)));
281 } else {
282 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
283 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
284 VT: CN->getValueType(ResNo: 0)));
285 }
286}
287
288SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
289 SDValue Src = N->getOperand(Num: 0);
290 assert(Src.getValueType() == MVT::ppcf128 &&
291 "In floats only ppcf128 can be extracted by element!");
292 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(N),
293 VT: N->getValueType(ResNo: 0).changeTypeToInteger(),
294 N1: DAG.getBitcast(VT: MVT::i128, V: Src), N2: N->getOperand(Num: 1));
295}
296
297SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
298 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
299 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
300 VT: NewOp.getValueType().getVectorElementType(),
301 N1: NewOp, N2: N->getOperand(Num: 1));
302}
303
304SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
305 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
306 unsigned Size = NVT.getSizeInBits();
307
308 // Mask = ~(1 << (Size-1))
309 APInt API = APInt::getAllOnes(numBits: Size);
310 API.clearBit(BitPosition: Size - 1);
311 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
312 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
313 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
314}
315
316SDValue DAGTypeLegalizer::SoftenFloatRes_FCANONICALIZE(SDNode *N) {
317 SDLoc dl(N);
318
319 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
320 // suggested in
321 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
322 // It uses strict_fp operations even outside a strict_fp context in order
323 // to guarantee that the canonicalization is not optimized away by later
324 // passes. The result chain introduced by that is intentionally ignored
325 // since no ordering requirement is intended here.
326
327 // Create strict multiplication by 1.0.
328 SDValue Operand = N->getOperand(Num: 0);
329 EVT VT = Operand.getValueType();
330 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
331 SDValue Chain = DAG.getEntryNode();
332 // Propagate existing flags on canonicalize, and additionally set
333 // NoFPExcept.
334 SDNodeFlags CanonicalizeFlags = N->getFlags();
335 CanonicalizeFlags.setNoFPExcept(true);
336 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
337 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
338 return BitConvertToInteger(Op: Mul);
339}
340
341SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
342 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
343 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
344 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
345 Call_F32: RTLIB::FMIN_F32,
346 Call_F64: RTLIB::FMIN_F64,
347 Call_F80: RTLIB::FMIN_F80,
348 Call_F128: RTLIB::FMIN_F128,
349 Call_PPCF128: RTLIB::FMIN_PPCF128));
350}
351
352SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
353 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
354 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
355 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
356 Call_F32: RTLIB::FMAX_F32,
357 Call_F64: RTLIB::FMAX_F64,
358 Call_F80: RTLIB::FMAX_F80,
359 Call_F128: RTLIB::FMAX_F128,
360 Call_PPCF128: RTLIB::FMAX_PPCF128));
361}
362
363SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
364 return SoftenFloatRes_Binary(
365 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_NUM_F32,
366 Call_F64: RTLIB::FMINIMUM_NUM_F64, Call_F80: RTLIB::FMINIMUM_NUM_F80,
367 Call_F128: RTLIB::FMINIMUM_NUM_F128, Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128));
368}
369
370SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
371 return SoftenFloatRes_Binary(
372 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_NUM_F32,
373 Call_F64: RTLIB::FMAXIMUM_NUM_F64, Call_F80: RTLIB::FMAXIMUM_NUM_F80,
374 Call_F128: RTLIB::FMAXIMUM_NUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128));
375}
376
377SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
378 return SoftenFloatRes_Binary(
379 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_F32,
380 Call_F64: RTLIB::FMINIMUM_F64, Call_F80: RTLIB::FMINIMUM_F80,
381 Call_F128: RTLIB::FMINIMUM_F128, Call_PPCF128: RTLIB::FMINIMUM_PPCF128));
382}
383
384SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
385 return SoftenFloatRes_Binary(
386 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_F32,
387 Call_F64: RTLIB::FMAXIMUM_F64, Call_F80: RTLIB::FMAXIMUM_F80,
388 Call_F128: RTLIB::FMAXIMUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_PPCF128));
389}
390
391SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
392 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
393 Call_F32: RTLIB::ADD_F32,
394 Call_F64: RTLIB::ADD_F64,
395 Call_F80: RTLIB::ADD_F80,
396 Call_F128: RTLIB::ADD_F128,
397 Call_PPCF128: RTLIB::ADD_PPCF128));
398}
399
400SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
401 return SoftenFloatRes_Unary(
402 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64,
403 Call_F80: RTLIB::ACOS_F80, Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128));
404}
405
406SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
407 return SoftenFloatRes_Unary(
408 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64,
409 Call_F80: RTLIB::ASIN_F80, Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128));
410}
411
412SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
413 return SoftenFloatRes_Unary(
414 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64,
415 Call_F80: RTLIB::ATAN_F80, Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128));
416}
417
418SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
419 return SoftenFloatRes_Binary(
420 N,
421 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN2_F32, Call_F64: RTLIB::ATAN2_F64,
422 Call_F80: RTLIB::ATAN2_F80, Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128));
423}
424
425SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
426 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
427 Call_F32: RTLIB::CBRT_F32,
428 Call_F64: RTLIB::CBRT_F64,
429 Call_F80: RTLIB::CBRT_F80,
430 Call_F128: RTLIB::CBRT_F128,
431 Call_PPCF128: RTLIB::CBRT_PPCF128));
432}
433
434SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
435 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
436 Call_F32: RTLIB::CEIL_F32,
437 Call_F64: RTLIB::CEIL_F64,
438 Call_F80: RTLIB::CEIL_F80,
439 Call_F128: RTLIB::CEIL_F128,
440 Call_PPCF128: RTLIB::CEIL_PPCF128));
441}
442
443SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
444 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
445 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
446 SDLoc dl(N);
447
448 EVT LVT = LHS.getValueType();
449 EVT RVT = RHS.getValueType();
450
451 unsigned LSize = LVT.getSizeInBits();
452 unsigned RSize = RVT.getSizeInBits();
453
454 // First get the sign bit of second operand.
455 SDValue SignBit = DAG.getNode(
456 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
457 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
458 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
459 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
460
461 // Shift right or sign-extend it if the two operands have different types.
462 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
463 if (SizeDiff > 0) {
464 SignBit =
465 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
466 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
467 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
468 DL: DAG.getDataLayout())));
469 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
470 } else if (SizeDiff < 0) {
471 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
472 SignBit =
473 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
474 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
475 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
476 DL: DAG.getDataLayout())));
477 }
478
479 // Clear the sign bit of the first operand.
480 SDValue Mask = DAG.getNode(
481 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
482 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
483 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
484 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
485 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
486
487 // Or the value with the sign bit.
488 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
489}
490
491SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
492 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
493 Call_F32: RTLIB::COS_F32,
494 Call_F64: RTLIB::COS_F64,
495 Call_F80: RTLIB::COS_F80,
496 Call_F128: RTLIB::COS_F128,
497 Call_PPCF128: RTLIB::COS_PPCF128));
498}
499
500SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
501 return SoftenFloatRes_Unary(
502 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64,
503 Call_F80: RTLIB::COSH_F80, Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128));
504}
505
506SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
507 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
508 Call_F32: RTLIB::DIV_F32,
509 Call_F64: RTLIB::DIV_F64,
510 Call_F80: RTLIB::DIV_F80,
511 Call_F128: RTLIB::DIV_F128,
512 Call_PPCF128: RTLIB::DIV_PPCF128));
513}
514
515SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
516 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
517 Call_F32: RTLIB::EXP_F32,
518 Call_F64: RTLIB::EXP_F64,
519 Call_F80: RTLIB::EXP_F80,
520 Call_F128: RTLIB::EXP_F128,
521 Call_PPCF128: RTLIB::EXP_PPCF128));
522}
523
524SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
525 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
526 Call_F32: RTLIB::EXP2_F32,
527 Call_F64: RTLIB::EXP2_F64,
528 Call_F80: RTLIB::EXP2_F80,
529 Call_F128: RTLIB::EXP2_F128,
530 Call_PPCF128: RTLIB::EXP2_PPCF128));
531}
532
533SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
534 return SoftenFloatRes_Unary(
535 N,
536 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64,
537 Call_F80: RTLIB::EXP10_F80, Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128));
538}
539
540SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
541 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
542 Call_F32: RTLIB::FLOOR_F32,
543 Call_F64: RTLIB::FLOOR_F64,
544 Call_F80: RTLIB::FLOOR_F80,
545 Call_F128: RTLIB::FLOOR_F128,
546 Call_PPCF128: RTLIB::FLOOR_PPCF128));
547}
548
549SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
550 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
551 Call_F32: RTLIB::LOG_F32,
552 Call_F64: RTLIB::LOG_F64,
553 Call_F80: RTLIB::LOG_F80,
554 Call_F128: RTLIB::LOG_F128,
555 Call_PPCF128: RTLIB::LOG_PPCF128));
556}
557
558SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
559 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
560 Call_F32: RTLIB::LOG2_F32,
561 Call_F64: RTLIB::LOG2_F64,
562 Call_F80: RTLIB::LOG2_F80,
563 Call_F128: RTLIB::LOG2_F128,
564 Call_PPCF128: RTLIB::LOG2_PPCF128));
565}
566
567SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
568 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
569 Call_F32: RTLIB::LOG10_F32,
570 Call_F64: RTLIB::LOG10_F64,
571 Call_F80: RTLIB::LOG10_F80,
572 Call_F128: RTLIB::LOG10_F128,
573 Call_PPCF128: RTLIB::LOG10_PPCF128));
574}
575
576SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
577 bool IsStrict = N->isStrictFPOpcode();
578 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
579 unsigned Offset = IsStrict ? 1 : 0;
580 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
581 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
582 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
583 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
584 TargetLowering::MakeLibCallOptions CallOptions;
585 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
586 N->getOperand(Num: 1 + Offset).getValueType(),
587 N->getOperand(Num: 2 + Offset).getValueType() };
588 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
589 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
590 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
591 Call_F32: RTLIB::FMA_F32,
592 Call_F64: RTLIB::FMA_F64,
593 Call_F80: RTLIB::FMA_F80,
594 Call_F128: RTLIB::FMA_F128,
595 Call_PPCF128: RTLIB::FMA_PPCF128),
596 RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
597 if (IsStrict)
598 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
599 return Tmp.first;
600}
601
602SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
603 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
604 Call_F32: RTLIB::MUL_F32,
605 Call_F64: RTLIB::MUL_F64,
606 Call_F80: RTLIB::MUL_F80,
607 Call_F128: RTLIB::MUL_F128,
608 Call_PPCF128: RTLIB::MUL_PPCF128));
609}
610
611SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
612 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
613 Call_F32: RTLIB::NEARBYINT_F32,
614 Call_F64: RTLIB::NEARBYINT_F64,
615 Call_F80: RTLIB::NEARBYINT_F80,
616 Call_F128: RTLIB::NEARBYINT_F128,
617 Call_PPCF128: RTLIB::NEARBYINT_PPCF128));
618}
619
620SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
621 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
622 SDLoc dl(N);
623
624 // Expand Y = FNEG(X) -> Y = X ^ sign mask
625 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
626 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
627 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
628}
629
630SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
631 bool IsStrict = N->isStrictFPOpcode();
632 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
633 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
634
635 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
636
637 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
638 // -> f32, so proceed in two stages. Also, it's entirely possible for both
639 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
640 // than FP16_TO_FP.
641 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
642 N->getValueType(ResNo: 0) != MVT::f32) {
643 if (IsStrict) {
644 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
645 ResultTys: { MVT::f32, MVT::Other }, Ops: { Chain, Op });
646 Chain = Op.getValue(R: 1);
647 } else {
648 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT: MVT::f32, Operand: Op);
649 }
650 }
651
652 if (Op.getValueType() == MVT::bf16) {
653 // FIXME: Need ReplaceValueWith on chain in strict case
654 return SoftenFloatRes_BF16_TO_FP(N);
655 }
656
657 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
658 if (LC == RTLIB::UNKNOWN_LIBCALL) {
659 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fp_extend");
660 if (IsStrict)
661 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
662 return DAG.getPOISON(VT: NVT);
663 }
664 TargetLowering::MakeLibCallOptions CallOptions;
665 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
666 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
667 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
668 CallOptions, dl: SDLoc(N),
669 Chain);
670 if (IsStrict)
671 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
672 return Tmp.first;
673}
674
675// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
676// nodes?
677SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
678 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
679 SDValue Op = N->getOperand(Num: 0);
680 TargetLowering::MakeLibCallOptions CallOptions;
681 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
682 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
683 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
684 CallOptions, dl: SDLoc(N)).first;
685 if (N->getValueType(ResNo: 0) == MVT::f32)
686 return Res32;
687
688 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
689 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: MVT::f32, RetVT: N->getValueType(ResNo: 0));
690 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
691 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
692}
693
694// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
695// nodes?
696SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
697 assert(N->getValueType(0) == MVT::f32 &&
698 "Can only soften BF16_TO_FP with f32 result");
699 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
700 SDValue Op = N->getOperand(Num: 0);
701 SDLoc DL(N);
702 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT,
703 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op));
704 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
705 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
706 return Res;
707}
708
709SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
710 bool IsStrict = N->isStrictFPOpcode();
711 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
712 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
713 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
714 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
715 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
716 TargetLowering::MakeLibCallOptions CallOptions;
717 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
718 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
719 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
720 CallOptions, dl: SDLoc(N),
721 Chain);
722 if (IsStrict)
723 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
724 return Tmp.first;
725}
726
727SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
728 return SoftenFloatRes_Binary(N, LC: RTLIB::getPOW(RetVT: N->getValueType(ResNo: 0)));
729}
730
731SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
732 bool IsStrict = N->isStrictFPOpcode();
733 unsigned Offset = IsStrict ? 1 : 0;
734 bool IsPowI =
735 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
736 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
737
738 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0))
739 : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0));
740 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
741 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
742 // Some targets don't have a powi libcall; use pow instead.
743 // FIXME: Implement this if some target needs it.
744 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fpowi to fpow");
745 if (IsStrict)
746 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
747 return DAG.getPOISON(VT: NVT);
748 }
749
750 if (DAG.getLibInfo().getIntSize() !=
751 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
752 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
753 // would use the wrong type for the argument.
754 DAG.getContext()->emitError(ErrorStr: "powi exponent does not match sizeof(int)");
755 if (IsStrict)
756 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
757 return DAG.getPOISON(VT: NVT);
758 }
759
760 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
761 N->getOperand(Num: 1 + Offset) };
762 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
763 TargetLowering::MakeLibCallOptions CallOptions;
764 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
765 N->getOperand(Num: 1 + Offset).getValueType() };
766 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
767 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
768 CallOptions, dl: SDLoc(N),
769 Chain);
770 if (IsStrict)
771 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
772 return Tmp.first;
773}
774
775SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
776 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
777 EVT VT0 = N->getValueType(ResNo: 0);
778 EVT VT1 = N->getValueType(ResNo: 1);
779 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT0);
780 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
781 SDLoc DL(N);
782
783 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
784 // If the exponent does not match with sizeof(int) a libcall would use the
785 // wrong type for the argument.
786 // TODO: Should be able to handle mismatches.
787 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
788 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
789 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
790 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
791 }
792
793 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
794
795 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
796 TargetLowering::MakeLibCallOptions CallOptions;
797 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
798 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
799 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
800
801 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
802 // but we only really need to handle the 0th one for softening anyway.
803 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0)
804 .setOpsTypeOverrides(CallOpsTypeOverrides);
805
806 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, RetVT: NVT0, Ops, CallOptions, dl: DL,
807 /*Chain=*/SDValue());
808 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
809 auto PtrInfo =
810 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
811
812 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
813
814 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
815 return ReturnVal;
816}
817
818bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
819 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
820 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
821 EVT VT = N->getValueType(ResNo: 0);
822
823 assert(VT == N->getValueType(1) &&
824 "expected both return values to have the same type");
825
826 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
827 if (LCImpl == RTLIB::Unsupported)
828 return false;
829
830 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
831
832 SDLoc DL(N);
833
834 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(Op: N->getOperand(Num: 0))};
835 SmallVector<EVT, 3> OpsVT = {VT};
836
837 std::array<SDValue, 2> StackSlots;
838 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
839 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
840 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
841 if (ResNum == CallRetResNo)
842 continue;
843 SDValue StackSlot = DAG.CreateStackTemporary(VT: NVT);
844 Ops.push_back(Elt: StackSlot);
845 OpsVT.push_back(Elt: StackSlot.getValueType());
846 StackSlots[ResNum] = StackSlot;
847 CallOpsTypeOverrides.push_back(Elt: PointerTy);
848 }
849
850 TargetLowering::MakeLibCallOptions CallOptions;
851 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
852 // but since both returns have the same type it should be okay.
853 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT)
854 .setOpsTypeOverrides(CallOpsTypeOverrides);
855
856 auto [ReturnVal, Chain] =
857 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: DL,
858 /*Chain=*/SDValue());
859
860 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
861 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
862 auto PtrInfo =
863 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
864 return DAG.getLoad(VT: NVT, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
865 };
866
867 for (auto [ResNum, SlackSlot] : enumerate(First&: StackSlots)) {
868 if (CallRetResNo == ResNum) {
869 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: ReturnVal);
870 continue;
871 }
872 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: CreateStackLoad(SlackSlot));
873 }
874
875 return true;
876}
877
878SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
879 EVT VT = N->getValueType(ResNo: 0);
880 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(RetVT: VT)))
881 return SDValue();
882
883 // Fall back on softening the separate sin and cos calls if available.
884 RTLIB::Libcall SinLC = RTLIB::getSIN(RetVT: VT);
885 RTLIB::Libcall CosLC = RTLIB::getCOS(RetVT: VT);
886
887 SDValue SoftSin, SoftCos;
888 if (DAG.getLibcalls().getLibcallImpl(Call: SinLC) == RTLIB::Unsupported ||
889 DAG.getLibcalls().getLibcallImpl(Call: CosLC) == RTLIB::Unsupported) {
890 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fsincos");
891
892 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
893 SoftSin = SoftCos = DAG.getPOISON(VT: NVT);
894 } else {
895 SoftSin = SoftenFloatRes_Unary(N, LC: SinLC);
896 SoftCos = SoftenFloatRes_Unary(N, LC: CosLC);
897 }
898
899 SetSoftenedFloat(Op: SDValue(N, 0), Result: SoftSin);
900 SetSoftenedFloat(Op: SDValue(N, 1), Result: SoftCos);
901 return SDValue();
902}
903
904SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
905 EVT VT = N->getValueType(ResNo: 0);
906 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT),
907 /*CallRetResNo=*/0))
908 return SDValue();
909
910 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
911 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fmodf");
912 SDValue Poison = DAG.getPOISON(VT: NVT);
913 SetSoftenedFloat(Op: SDValue(N, 0), Result: Poison);
914 SetSoftenedFloat(Op: SDValue(N, 1), Result: Poison);
915 return SDValue();
916}
917
918SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
919 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
920 Call_F32: RTLIB::REM_F32,
921 Call_F64: RTLIB::REM_F64,
922 Call_F80: RTLIB::REM_F80,
923 Call_F128: RTLIB::REM_F128,
924 Call_PPCF128: RTLIB::REM_PPCF128));
925}
926
927SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
928 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
929 Call_F32: RTLIB::RINT_F32,
930 Call_F64: RTLIB::RINT_F64,
931 Call_F80: RTLIB::RINT_F80,
932 Call_F128: RTLIB::RINT_F128,
933 Call_PPCF128: RTLIB::RINT_PPCF128));
934}
935
936SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
937 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
938 Call_F32: RTLIB::ROUND_F32,
939 Call_F64: RTLIB::ROUND_F64,
940 Call_F80: RTLIB::ROUND_F80,
941 Call_F128: RTLIB::ROUND_F128,
942 Call_PPCF128: RTLIB::ROUND_PPCF128));
943}
944
945SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
946 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
947 Call_F32: RTLIB::ROUNDEVEN_F32,
948 Call_F64: RTLIB::ROUNDEVEN_F64,
949 Call_F80: RTLIB::ROUNDEVEN_F80,
950 Call_F128: RTLIB::ROUNDEVEN_F128,
951 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128));
952}
953
954SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
955 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
956 Call_F32: RTLIB::SIN_F32,
957 Call_F64: RTLIB::SIN_F64,
958 Call_F80: RTLIB::SIN_F80,
959 Call_F128: RTLIB::SIN_F128,
960 Call_PPCF128: RTLIB::SIN_PPCF128));
961}
962
963SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
964 return SoftenFloatRes_Unary(
965 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64,
966 Call_F80: RTLIB::SINH_F80, Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128));
967}
968
969SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
970 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
971 Call_F32: RTLIB::SQRT_F32,
972 Call_F64: RTLIB::SQRT_F64,
973 Call_F80: RTLIB::SQRT_F80,
974 Call_F128: RTLIB::SQRT_F128,
975 Call_PPCF128: RTLIB::SQRT_PPCF128));
976}
977
978SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
979 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
980 Call_F32: RTLIB::SUB_F32,
981 Call_F64: RTLIB::SUB_F64,
982 Call_F80: RTLIB::SUB_F80,
983 Call_F128: RTLIB::SUB_F128,
984 Call_PPCF128: RTLIB::SUB_PPCF128));
985}
986
987SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
988 return SoftenFloatRes_Unary(
989 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64,
990 Call_F80: RTLIB::TAN_F80, Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128));
991}
992
993SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
994 return SoftenFloatRes_Unary(
995 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64,
996 Call_F80: RTLIB::TANH_F80, Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128));
997}
998
999SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
1000 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1001 Call_F32: RTLIB::TRUNC_F32,
1002 Call_F64: RTLIB::TRUNC_F64,
1003 Call_F80: RTLIB::TRUNC_F80,
1004 Call_F128: RTLIB::TRUNC_F128,
1005 Call_PPCF128: RTLIB::TRUNC_PPCF128));
1006}
1007
1008SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
1009 LoadSDNode *L = cast<LoadSDNode>(Val: N);
1010 EVT VT = N->getValueType(ResNo: 0);
1011 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1012 SDLoc dl(N);
1013
1014 auto MMOFlags =
1015 L->getMemOperand()->getFlags() &
1016 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
1017 SDValue NewL;
1018 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
1019 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: NVT, dl,
1020 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
1021 PtrInfo: L->getPointerInfo(), MemVT: NVT, Alignment: L->getBaseAlign(), MMOFlags,
1022 AAInfo: L->getAAInfo());
1023 // Legalized the chain result - switch anything that used the old chain to
1024 // use the new one.
1025 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
1026 return NewL;
1027 }
1028
1029 // Do a non-extending load followed by FP_EXTEND.
1030 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
1031 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
1032 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(), Alignment: L->getBaseAlign(),
1033 MMOFlags, AAInfo: L->getAAInfo());
1034 // Legalized the chain result - switch anything that used the old chain to
1035 // use the new one.
1036 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
1037 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
1038 return BitConvertToInteger(Op: ExtendNode);
1039}
1040
1041SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
1042 AtomicSDNode *L = cast<AtomicSDNode>(Val: N);
1043 EVT VT = N->getValueType(ResNo: 0);
1044 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1045 SDLoc dl(N);
1046
1047 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
1048 SDValue NewL =
1049 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
1050 Ops: {L->getChain(), L->getBasePtr()}, MMO: L->getMemOperand());
1051
1052 // Legalized the chain result - switch anything that used the old chain to
1053 // use the new one.
1054 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
1055 return NewL;
1056 }
1057
1058 report_fatal_error(reason: "softening fp extending atomic load not handled");
1059}
1060
1061SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
1062 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
1063 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
1064 return DAG.getSelect(DL: SDLoc(N),
1065 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
1066}
1067
1068SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
1069 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
1070 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
1071 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
1072 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
1073 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
1074}
1075
1076SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
1077 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
1078 VT: N->getValueType(ResNo: 0)));
1079}
1080
1081SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
1082 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
1083 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
1084 EVT VT = N->getValueType(ResNo: 0);
1085 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1086 SDLoc dl(N);
1087
1088 SDValue NewVAARG;
1089 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
1090 Align: N->getConstantOperandVal(Num: 3));
1091
1092 // Legalized the chain result - switch anything that used the old chain to
1093 // use the new one.
1094 if (N != NewVAARG.getValue(R: 1).getNode())
1095 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
1096 return NewVAARG;
1097}
1098
1099SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1100 bool IsStrict = N->isStrictFPOpcode();
1101 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1102 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1103 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
1104 EVT RVT = N->getValueType(ResNo: 0);
1105 EVT NVT = EVT();
1106 SDLoc dl(N);
1107
1108 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1109 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1110 // match. Look for an appropriate libcall.
1111 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1112 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1113 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1114 NVT = (MVT::SimpleValueType)t;
1115 // The source needs to big enough to hold the operand.
1116 if (NVT.bitsGE(VT: SVT))
1117 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
1118 }
1119 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1120
1121 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1122 // Sign/zero extend the argument if the libcall takes a larger type.
1123 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
1124 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
1125 TargetLowering::MakeLibCallOptions CallOptions;
1126 CallOptions.setIsSigned(Signed);
1127 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1128 std::pair<SDValue, SDValue> Tmp =
1129 TLI.makeLibCall(DAG, LC, RetVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT),
1130 Ops: Op, CallOptions, dl, Chain);
1131
1132 if (IsStrict)
1133 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1134 return Tmp.first;
1135}
1136
1137SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1138 // Expand and soften recursively.
1139 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
1140 return SDValue();
1141}
1142
1143SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1144 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
1145 return SDValue();
1146}
1147
1148//===----------------------------------------------------------------------===//
1149// Convert Float Operand to Integer
1150//===----------------------------------------------------------------------===//
1151
1152bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1153 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1154 SDValue Res = SDValue();
1155
1156 switch (N->getOpcode()) {
1157 default:
1158#ifndef NDEBUG
1159 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1160 N->dump(&DAG); dbgs() << "\n";
1161#endif
1162 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
1163
1164 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1165 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1166 case ISD::STRICT_FP_TO_FP16:
1167 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1168 case ISD::FP_TO_BF16:
1169 case ISD::STRICT_FP_TO_BF16:
1170 case ISD::STRICT_FP_ROUND:
1171 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1172 case ISD::STRICT_FP_TO_SINT:
1173 case ISD::STRICT_FP_TO_UINT:
1174 case ISD::FP_TO_SINT:
1175 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1176 case ISD::FP_TO_SINT_SAT:
1177 case ISD::FP_TO_UINT_SAT:
1178 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1179 case ISD::STRICT_LROUND:
1180 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1181 case ISD::STRICT_LLROUND:
1182 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1183 case ISD::STRICT_LRINT:
1184 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1185 case ISD::STRICT_LLRINT:
1186 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1187 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1188 case ISD::STRICT_FSETCC:
1189 case ISD::STRICT_FSETCCS:
1190 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1191 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1192 case ISD::ATOMIC_STORE:
1193 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1194 break;
1195 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1196 case ISD::FAKE_USE:
1197 Res = SoftenFloatOp_FAKE_USE(N);
1198 break;
1199 case ISD::STACKMAP:
1200 Res = SoftenFloatOp_STACKMAP(N, OpNo);
1201 break;
1202 case ISD::PATCHPOINT:
1203 Res = SoftenFloatOp_PATCHPOINT(N, OpNo);
1204 break;
1205 }
1206
1207 // If the result is null, the sub-method took care of registering results etc.
1208 if (!Res.getNode()) return false;
1209
1210 // If the result is N, the sub-method updated N in place. Tell the legalizer
1211 // core about this to re-analyze.
1212 if (Res.getNode() == N)
1213 return true;
1214
1215 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1216 "Invalid operand softening");
1217
1218 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1219 return false;
1220}
1221
1222SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1223 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
1224
1225 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
1226}
1227
1228SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1229 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1230 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1231 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1232 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1233 N->getOpcode() == ISD::FP_TO_BF16 ||
1234 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1235 N->getOpcode() == ISD::STRICT_FP_ROUND);
1236
1237 bool IsStrict = N->isStrictFPOpcode();
1238 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1239 EVT SVT = Op.getValueType();
1240 EVT RVT = N->getValueType(ResNo: 0);
1241 EVT FloatRVT = RVT;
1242 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1243 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1244 FloatRVT = MVT::f16;
1245 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1246 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1247 FloatRVT = MVT::bf16;
1248
1249 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
1250 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1251
1252 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1253 Op = GetSoftenedFloat(Op);
1254 TargetLowering::MakeLibCallOptions CallOptions;
1255 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1256 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op,
1257 CallOptions, dl: SDLoc(N),
1258 Chain);
1259 if (IsStrict) {
1260 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1261 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1262 return SDValue();
1263 }
1264 return Tmp.first;
1265}
1266
1267SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1268 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1269 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1270
1271 EVT VT = NewLHS.getValueType();
1272 NewLHS = GetSoftenedFloat(Op: NewLHS);
1273 NewRHS = GetSoftenedFloat(Op: NewRHS);
1274 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1275 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1276
1277 // If softenSetCCOperands returned a scalar, we need to compare the result
1278 // against zero to select between true and false values.
1279 if (!NewRHS.getNode()) {
1280 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1281 CCCode = ISD::SETNE;
1282 }
1283
1284 // Update N to have the operands specified.
1285 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1286 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1287 Op5: N->getOperand(Num: 4)),
1288 0);
1289}
1290
1291// Even if the result type is legal, no libcall may exactly match. (e.g. We
1292// don't have FP-i8 conversions) This helper method looks for an appropriate
1293// promoted libcall.
1294static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1295 bool Signed) {
1296 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1297 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1298 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1299 ++IntVT) {
1300 Promoted = (MVT::SimpleValueType)IntVT;
1301 // The type needs to big enough to hold the result.
1302 if (Promoted.bitsGE(VT: RetVT))
1303 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1304 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1305 }
1306 return LC;
1307}
1308
1309SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1310 bool IsStrict = N->isStrictFPOpcode();
1311 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1312 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1313
1314 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1315 EVT SVT = Op.getValueType();
1316 EVT RVT = N->getValueType(ResNo: 0);
1317 EVT NVT = EVT();
1318 SDLoc dl(N);
1319
1320 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1321 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1322 // match, eg. we don't have fp -> i8 conversions.
1323 // Look for an appropriate libcall.
1324 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1325 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1326 "Unsupported FP_TO_XINT!");
1327
1328 Op = GetSoftenedFloat(Op);
1329 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1330 TargetLowering::MakeLibCallOptions CallOptions;
1331 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1332 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1333 CallOptions, dl, Chain);
1334
1335 // Truncate the result if the libcall returns a larger type.
1336 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1337
1338 if (!IsStrict)
1339 return Res;
1340
1341 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1342 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1343 return SDValue();
1344}
1345
1346SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1347 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1348 return Res;
1349}
1350
1351SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1352 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1353 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1354
1355 EVT VT = NewLHS.getValueType();
1356 NewLHS = GetSoftenedFloat(Op: NewLHS);
1357 NewRHS = GetSoftenedFloat(Op: NewRHS);
1358 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1359 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1360
1361 // If softenSetCCOperands returned a scalar, we need to compare the result
1362 // against zero to select between true and false values.
1363 if (!NewRHS.getNode()) {
1364 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1365 CCCode = ISD::SETNE;
1366 }
1367
1368 // Update N to have the operands specified.
1369 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1370 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1371 Op5: DAG.getCondCode(Cond: CCCode)),
1372 0);
1373}
1374
1375SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1376 bool IsStrict = N->isStrictFPOpcode();
1377 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1378 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1379 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1380 ISD::CondCode CCCode =
1381 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1382
1383 EVT VT = Op0.getValueType();
1384 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1385 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1386 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1387 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1388
1389 // Update N to have the operands specified.
1390 if (NewRHS.getNode()) {
1391 if (IsStrict)
1392 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1393 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1394 else
1395 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1396 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1397 }
1398
1399 // Otherwise, softenSetCCOperands returned a scalar, use it.
1400 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1401 "Unexpected setcc expansion!");
1402
1403 if (IsStrict) {
1404 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1405 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1406 return SDValue();
1407 }
1408 return NewLHS;
1409}
1410
1411SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1412 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1413 assert(OpNo == 1 && "Can only soften the stored value!");
1414 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1415 SDValue Val = ST->getValue();
1416 SDLoc dl(N);
1417
1418 if (ST->isTruncatingStore())
1419 // Do an FP_ROUND followed by a non-truncating store.
1420 Val = BitConvertToInteger(
1421 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1422 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1423 else
1424 Val = GetSoftenedFloat(Op: Val);
1425
1426 return DAG.getStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(),
1427 MMO: ST->getMemOperand());
1428}
1429
1430SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1431 assert(OpNo == 1 && "Can only soften the stored value!");
1432 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
1433 SDValue Val = ST->getVal();
1434 EVT VT = Val.getValueType();
1435 SDLoc dl(N);
1436
1437 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1438
1439 SDValue NewVal = GetSoftenedFloat(Op: Val);
1440 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: VT, Chain: ST->getChain(), Ptr: NewVal,
1441 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
1442}
1443
1444SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1445 SDValue LHS = N->getOperand(Num: 0);
1446 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1447 SDLoc dl(N);
1448
1449 EVT LVT = LHS.getValueType();
1450 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1451 EVT RVT = RHS.getValueType();
1452
1453 unsigned LSize = LVT.getSizeInBits();
1454 unsigned RSize = RVT.getSizeInBits();
1455
1456 // Shift right or sign-extend it if the two operands have different types.
1457 int SizeDiff = RSize - LSize;
1458 if (SizeDiff > 0) {
1459 RHS =
1460 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1461 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1462 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1463 DL: DAG.getDataLayout())));
1464 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1465 } else if (SizeDiff < 0) {
1466 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1467 RHS =
1468 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1469 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1470 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1471 DL: DAG.getDataLayout())));
1472 }
1473
1474 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1475 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1476}
1477
1478SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1479 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1480 bool IsStrict = N->isStrictFPOpcode();
1481 unsigned Offset = IsStrict ? 1 : 0;
1482 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1483 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1484 TargetLowering::MakeLibCallOptions CallOptions;
1485 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1486 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
1487 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1488 CallOptions, dl: SDLoc(N),
1489 Chain);
1490 if (IsStrict) {
1491 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1492 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1493 return SDValue();
1494 }
1495
1496 return Tmp.first;
1497}
1498
1499SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1500 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1501 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1502 Call_F32: RTLIB::LROUND_F32,
1503 Call_F64: RTLIB::LROUND_F64,
1504 Call_F80: RTLIB::LROUND_F80,
1505 Call_F128: RTLIB::LROUND_F128,
1506 Call_PPCF128: RTLIB::LROUND_PPCF128));
1507}
1508
1509SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1510 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1511 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1512 Call_F32: RTLIB::LLROUND_F32,
1513 Call_F64: RTLIB::LLROUND_F64,
1514 Call_F80: RTLIB::LLROUND_F80,
1515 Call_F128: RTLIB::LLROUND_F128,
1516 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1517}
1518
1519SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1520 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1521 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1522 Call_F32: RTLIB::LRINT_F32,
1523 Call_F64: RTLIB::LRINT_F64,
1524 Call_F80: RTLIB::LRINT_F80,
1525 Call_F128: RTLIB::LRINT_F128,
1526 Call_PPCF128: RTLIB::LRINT_PPCF128));
1527}
1528
1529SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1530 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1531 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1532 Call_F32: RTLIB::LLRINT_F32,
1533 Call_F64: RTLIB::LLRINT_F64,
1534 Call_F80: RTLIB::LLRINT_F80,
1535 Call_F128: RTLIB::LLRINT_F128,
1536 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1537}
1538
1539SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1540 SDValue Op1 = BitConvertToInteger(Op: N->getOperand(Num: 1));
1541 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1542 N1: N->getOperand(Num: 0), N2: Op1);
1543}
1544
1545SDValue DAGTypeLegalizer::SoftenFloatOp_STACKMAP(SDNode *N, unsigned OpNo) {
1546 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
1547 SmallVector<SDValue> NewOps(N->ops());
1548 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1549 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1550}
1551
1552SDValue DAGTypeLegalizer::SoftenFloatOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
1553 assert(OpNo >= 7);
1554 SmallVector<SDValue> NewOps(N->ops());
1555 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1556 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1557}
1558
1559//===----------------------------------------------------------------------===//
1560// Float Result Expansion
1561//===----------------------------------------------------------------------===//
1562
1563/// ExpandFloatResult - This method is called when the specified result of the
1564/// specified node is found to need expansion. At this point, the node may also
1565/// have invalid operands or may have other results that need promotion, we just
1566/// know that (at least) one result needs expansion.
1567void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1568 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1569 SDValue Lo, Hi;
1570 Lo = Hi = SDValue();
1571
1572 // See if the target wants to custom expand this node.
1573 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1574 return;
1575
1576 switch (N->getOpcode()) {
1577 default:
1578#ifndef NDEBUG
1579 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1580 N->dump(&DAG); dbgs() << "\n";
1581#endif
1582 report_fatal_error(reason: "Do not know how to expand the result of this "
1583 "operator!");
1584 // clang-format off
1585 case ISD::POISON:
1586 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1587 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1588 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1589
1590 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1591 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1592 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1593 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1594 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1595 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1596
1597 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1598 case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
1599 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1600 case ISD::STRICT_FMINNUM:
1601 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1602 case ISD::STRICT_FMAXNUM:
1603 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1604 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1605 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1606 case ISD::STRICT_FADD:
1607 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1608 case ISD::STRICT_FACOS:
1609 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1610 case ISD::STRICT_FASIN:
1611 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1612 case ISD::STRICT_FATAN:
1613 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1614 case ISD::STRICT_FATAN2:
1615 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1616 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1617 case ISD::STRICT_FCEIL:
1618 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1619 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1620 case ISD::STRICT_FCOS:
1621 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1622 case ISD::STRICT_FCOSH:
1623 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1624 case ISD::STRICT_FDIV:
1625 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1626 case ISD::STRICT_FEXP:
1627 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1628 case ISD::STRICT_FEXP2:
1629 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1630 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1631 case ISD::STRICT_FFLOOR:
1632 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1633 case ISD::STRICT_FLOG:
1634 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1635 case ISD::STRICT_FLOG2:
1636 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1637 case ISD::STRICT_FLOG10:
1638 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1639 case ISD::STRICT_FMA:
1640 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1641 case ISD::STRICT_FMUL:
1642 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1643 case ISD::STRICT_FNEARBYINT:
1644 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1645 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1646 case ISD::STRICT_FP_EXTEND:
1647 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1648 case ISD::STRICT_FPOW:
1649 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1650 case ISD::STRICT_FPOWI:
1651 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1652 case ISD::FLDEXP:
1653 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1654 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1655 case ISD::STRICT_FRINT:
1656 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1657 case ISD::STRICT_FROUND:
1658 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1659 case ISD::STRICT_FROUNDEVEN:
1660 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1661 case ISD::STRICT_FSIN:
1662 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1663 case ISD::STRICT_FSINH:
1664 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1665 case ISD::STRICT_FSQRT:
1666 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1667 case ISD::STRICT_FSUB:
1668 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1669 case ISD::STRICT_FTAN:
1670 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1671 case ISD::STRICT_FTANH:
1672 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1673 case ISD::STRICT_FTRUNC:
1674 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1675 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1676 case ISD::STRICT_SINT_TO_FP:
1677 case ISD::STRICT_UINT_TO_FP:
1678 case ISD::SINT_TO_FP:
1679 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1680 case ISD::STRICT_FREM:
1681 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1682 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1683 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1684 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1685 // clang-format on
1686 }
1687
1688 // If Lo/Hi is null, the sub-method took care of registering results etc.
1689 if (Lo.getNode())
1690 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1691}
1692
1693void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1694 SDValue &Hi) {
1695 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1696 assert(NVT.getSizeInBits() == 64 &&
1697 "Do not know how to expand this float constant!");
1698 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1699 SDLoc dl(N);
1700 const fltSemantics &Sem = NVT.getFltSemantics();
1701 Lo = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 64)), DL: dl, VT: NVT);
1702 Hi = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 0)), DL: dl, VT: NVT);
1703}
1704
1705void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1706 SDValue &Lo, SDValue &Hi) {
1707 bool IsStrict = N->isStrictFPOpcode();
1708 unsigned Offset = IsStrict ? 1 : 0;
1709 SDValue Op = N->getOperand(Num: 0 + Offset);
1710 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1711 TargetLowering::MakeLibCallOptions CallOptions;
1712 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1713 Ops: Op, CallOptions, dl: SDLoc(N),
1714 Chain);
1715 if (IsStrict)
1716 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1717 GetPairElements(Pair: Tmp.first, Lo, Hi);
1718}
1719
1720void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1721 SDValue &Lo, SDValue &Hi) {
1722 bool IsStrict = N->isStrictFPOpcode();
1723 unsigned Offset = IsStrict ? 1 : 0;
1724 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1725 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1726 TargetLowering::MakeLibCallOptions CallOptions;
1727 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1728 Ops, CallOptions, dl: SDLoc(N),
1729 Chain);
1730 if (IsStrict)
1731 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1732 GetPairElements(Pair: Tmp.first, Lo, Hi);
1733}
1734
1735void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1736 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT: N->getValueType(ResNo: 0)),
1737 /*CallRetResNo=*/0);
1738}
1739
1740void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1741 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(RetVT: N->getValueType(ResNo: 0)));
1742}
1743
1744void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1745 ExpandFloatRes_UnaryWithTwoFPResults(N,
1746 LC: RTLIB::getSINCOSPI(RetVT: N->getValueType(ResNo: 0)));
1747}
1748
1749void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1750 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1751 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1752 SmallVector<SDValue> Results;
1753 TLI.expandMultipleResultFPLibCall(DAG, LC, Node: N, Results, CallRetResNo);
1754 for (auto [ResNo, Res] : enumerate(First&: Results)) {
1755 SDValue Lo, Hi;
1756 GetPairElements(Pair: Res, Lo, Hi);
1757 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1758 }
1759}
1760
1761void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1762 SDValue &Hi) {
1763 assert(N->getValueType(0) == MVT::ppcf128 &&
1764 "Logic only correct for ppcf128!");
1765 SDLoc dl(N);
1766 SDValue Tmp;
1767 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1768 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1769 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1770 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1771 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1772 Cond: ISD::SETEQ);
1773}
1774
1775void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1776 SDValue &Hi) {
1777 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1778 Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
1779 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
1780 Call_PPCF128: RTLIB::FMIN_PPCF128), Lo, Hi);
1781}
1782
1783void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1784 SDValue &Hi) {
1785 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1786 Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
1787 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
1788 Call_PPCF128: RTLIB::FMAX_PPCF128), Lo, Hi);
1789}
1790
1791void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1792 SDValue &Hi) {
1793 ExpandFloatRes_Binary(
1794 N,
1795 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_NUM_F32,
1796 Call_F64: RTLIB::FMINIMUM_NUM_F64, Call_F80: RTLIB::FMINIMUM_NUM_F80,
1797 Call_F128: RTLIB::FMINIMUM_NUM_F128, Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128),
1798 Lo, Hi);
1799}
1800
1801void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1802 SDValue &Hi) {
1803 ExpandFloatRes_Binary(
1804 N,
1805 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_NUM_F32,
1806 Call_F64: RTLIB::FMAXIMUM_NUM_F64, Call_F80: RTLIB::FMAXIMUM_NUM_F80,
1807 Call_F128: RTLIB::FMAXIMUM_NUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128),
1808 Lo, Hi);
1809}
1810
1811void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1812 SDValue &Hi) {
1813 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1814 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1815 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1816 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1817}
1818
1819void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1820 SDValue &Hi) {
1821 ExpandFloatRes_Unary(N,
1822 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ACOS_F32,
1823 Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
1824 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128),
1825 Lo, Hi);
1826}
1827
1828void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1829 SDValue &Hi) {
1830 ExpandFloatRes_Unary(N,
1831 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ASIN_F32,
1832 Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
1833 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128),
1834 Lo, Hi);
1835}
1836
1837void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1838 SDValue &Hi) {
1839 ExpandFloatRes_Unary(N,
1840 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN_F32,
1841 Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
1842 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128),
1843 Lo, Hi);
1844}
1845
1846void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1847 SDValue &Hi) {
1848 ExpandFloatRes_Binary(N,
1849 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN2_F32,
1850 Call_F64: RTLIB::ATAN2_F64, Call_F80: RTLIB::ATAN2_F80,
1851 Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128),
1852 Lo, Hi);
1853}
1854
1855void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1856 SDValue &Hi) {
1857 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::CBRT_F32,
1858 Call_F64: RTLIB::CBRT_F64, Call_F80: RTLIB::CBRT_F80,
1859 Call_F128: RTLIB::CBRT_F128,
1860 Call_PPCF128: RTLIB::CBRT_PPCF128), Lo, Hi);
1861}
1862
1863void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1864 SDValue &Lo, SDValue &Hi) {
1865 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1866 Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
1867 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
1868 Call_PPCF128: RTLIB::CEIL_PPCF128), Lo, Hi);
1869}
1870
1871void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1872 SDValue &Lo, SDValue &Hi) {
1873 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1874 Call_F32: RTLIB::COPYSIGN_F32,
1875 Call_F64: RTLIB::COPYSIGN_F64,
1876 Call_F80: RTLIB::COPYSIGN_F80,
1877 Call_F128: RTLIB::COPYSIGN_F128,
1878 Call_PPCF128: RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1879}
1880
1881void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1882 SDValue &Lo, SDValue &Hi) {
1883 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1884 Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
1885 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
1886 Call_PPCF128: RTLIB::COS_PPCF128), Lo, Hi);
1887}
1888
1889void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1890 SDValue &Hi) {
1891 ExpandFloatRes_Unary(N,
1892 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::COSH_F32,
1893 Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
1894 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128),
1895 Lo, Hi);
1896}
1897
1898void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1899 SDValue &Hi) {
1900 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1901 Call_F32: RTLIB::DIV_F32,
1902 Call_F64: RTLIB::DIV_F64,
1903 Call_F80: RTLIB::DIV_F80,
1904 Call_F128: RTLIB::DIV_F128,
1905 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1906}
1907
1908void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1909 SDValue &Lo, SDValue &Hi) {
1910 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1911 Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64,
1912 Call_F80: RTLIB::EXP_F80, Call_F128: RTLIB::EXP_F128,
1913 Call_PPCF128: RTLIB::EXP_PPCF128), Lo, Hi);
1914}
1915
1916void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1917 SDValue &Lo, SDValue &Hi) {
1918 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1919 Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64,
1920 Call_F80: RTLIB::EXP2_F80, Call_F128: RTLIB::EXP2_F128,
1921 Call_PPCF128: RTLIB::EXP2_PPCF128), Lo, Hi);
1922}
1923
1924void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1925 SDValue &Hi) {
1926 ExpandFloatRes_Unary(N,
1927 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32,
1928 Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
1929 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128),
1930 Lo, Hi);
1931}
1932
1933void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1934 SDValue &Lo, SDValue &Hi) {
1935 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1936 Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
1937 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
1938 Call_PPCF128: RTLIB::FLOOR_PPCF128), Lo, Hi);
1939}
1940
1941void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1942 SDValue &Lo, SDValue &Hi) {
1943 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1944 Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64,
1945 Call_F80: RTLIB::LOG_F80, Call_F128: RTLIB::LOG_F128,
1946 Call_PPCF128: RTLIB::LOG_PPCF128), Lo, Hi);
1947}
1948
1949void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1950 SDValue &Lo, SDValue &Hi) {
1951 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1952 Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64,
1953 Call_F80: RTLIB::LOG2_F80, Call_F128: RTLIB::LOG2_F128,
1954 Call_PPCF128: RTLIB::LOG2_PPCF128), Lo, Hi);
1955}
1956
1957void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1958 SDValue &Lo, SDValue &Hi) {
1959 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1960 Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64,
1961 Call_F80: RTLIB::LOG10_F80, Call_F128: RTLIB::LOG10_F128,
1962 Call_PPCF128: RTLIB::LOG10_PPCF128), Lo, Hi);
1963}
1964
1965void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1966 SDValue &Hi) {
1967 bool IsStrict = N->isStrictFPOpcode();
1968 unsigned Offset = IsStrict ? 1 : 0;
1969 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1970 N->getOperand(Num: 2 + Offset) };
1971 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1972 TargetLowering::MakeLibCallOptions CallOptions;
1973 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1974 Call_F32: RTLIB::FMA_F32,
1975 Call_F64: RTLIB::FMA_F64,
1976 Call_F80: RTLIB::FMA_F80,
1977 Call_F128: RTLIB::FMA_F128,
1978 Call_PPCF128: RTLIB::FMA_PPCF128),
1979 RetVT: N->getValueType(ResNo: 0), Ops, CallOptions,
1980 dl: SDLoc(N), Chain);
1981 if (IsStrict)
1982 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1983 GetPairElements(Pair: Tmp.first, Lo, Hi);
1984}
1985
1986void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1987 SDValue &Hi) {
1988 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1989 Call_F32: RTLIB::MUL_F32,
1990 Call_F64: RTLIB::MUL_F64,
1991 Call_F80: RTLIB::MUL_F80,
1992 Call_F128: RTLIB::MUL_F128,
1993 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1994}
1995
1996void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1997 SDValue &Lo, SDValue &Hi) {
1998 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1999 Call_F32: RTLIB::NEARBYINT_F32,
2000 Call_F64: RTLIB::NEARBYINT_F64,
2001 Call_F80: RTLIB::NEARBYINT_F80,
2002 Call_F128: RTLIB::NEARBYINT_F128,
2003 Call_PPCF128: RTLIB::NEARBYINT_PPCF128), Lo, Hi);
2004}
2005
2006void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
2007 SDValue &Hi) {
2008 SDLoc dl(N);
2009 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
2010 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
2011 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
2012}
2013
2014void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
2015 SDValue &Hi) {
2016 // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
2017 SDLoc dl(N);
2018 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
2019}
2020
2021void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
2022 SDValue &Hi) {
2023 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
2024 SDLoc dl(N);
2025 bool IsStrict = N->isStrictFPOpcode();
2026
2027 SDValue Chain;
2028 if (IsStrict) {
2029 // If the expanded type is the same as the input type, just bypass the node.
2030 if (NVT == N->getOperand(Num: 1).getValueType()) {
2031 Hi = N->getOperand(Num: 1);
2032 Chain = N->getOperand(Num: 0);
2033 } else {
2034 // Other we need to extend.
2035 Hi = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: { NVT, MVT::Other },
2036 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
2037 Chain = Hi.getValue(R: 1);
2038 }
2039 } else {
2040 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
2041 }
2042
2043 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2044
2045 if (IsStrict)
2046 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2047}
2048
2049void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
2050 SDValue &Lo, SDValue &Hi) {
2051 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2052 Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64,
2053 Call_F80: RTLIB::POW_F80, Call_F128: RTLIB::POW_F128,
2054 Call_PPCF128: RTLIB::POW_PPCF128), Lo, Hi);
2055}
2056
2057void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
2058 SDValue &Lo, SDValue &Hi) {
2059 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
2060}
2061
2062void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
2063 SDValue &Hi) {
2064 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
2065}
2066
2067void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
2068 SDValue &Lo, SDValue &Hi) {
2069 assert(N->getValueType(0) == MVT::ppcf128 &&
2070 "Logic only correct for ppcf128!");
2071
2072 SDLoc dl(N);
2073 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
2074 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
2075 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
2076}
2077
2078void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
2079 SDValue &Lo, SDValue &Hi) {
2080 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2081 Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
2082 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
2083 Call_PPCF128: RTLIB::REM_PPCF128), Lo, Hi);
2084}
2085
2086void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
2087 SDValue &Lo, SDValue &Hi) {
2088 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2089 Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
2090 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
2091 Call_PPCF128: RTLIB::RINT_PPCF128), Lo, Hi);
2092}
2093
2094void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
2095 SDValue &Lo, SDValue &Hi) {
2096 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2097 Call_F32: RTLIB::ROUND_F32,
2098 Call_F64: RTLIB::ROUND_F64,
2099 Call_F80: RTLIB::ROUND_F80,
2100 Call_F128: RTLIB::ROUND_F128,
2101 Call_PPCF128: RTLIB::ROUND_PPCF128), Lo, Hi);
2102}
2103
2104void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
2105 SDValue &Lo, SDValue &Hi) {
2106 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2107 Call_F32: RTLIB::ROUNDEVEN_F32,
2108 Call_F64: RTLIB::ROUNDEVEN_F64,
2109 Call_F80: RTLIB::ROUNDEVEN_F80,
2110 Call_F128: RTLIB::ROUNDEVEN_F128,
2111 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
2112}
2113
2114void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
2115 SDValue &Lo, SDValue &Hi) {
2116 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2117 Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
2118 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
2119 Call_PPCF128: RTLIB::SIN_PPCF128), Lo, Hi);
2120}
2121
2122void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
2123 SDValue &Hi) {
2124 ExpandFloatRes_Unary(N,
2125 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::SINH_F32,
2126 Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
2127 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128),
2128 Lo, Hi);
2129}
2130
2131void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
2132 SDValue &Lo, SDValue &Hi) {
2133 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2134 Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
2135 Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
2136 Call_PPCF128: RTLIB::SQRT_PPCF128), Lo, Hi);
2137}
2138
2139void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
2140 SDValue &Hi) {
2141 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2142 Call_F32: RTLIB::SUB_F32,
2143 Call_F64: RTLIB::SUB_F64,
2144 Call_F80: RTLIB::SUB_F80,
2145 Call_F128: RTLIB::SUB_F128,
2146 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
2147}
2148
2149void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
2150 SDValue &Hi) {
2151 ExpandFloatRes_Unary(N,
2152 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TAN_F32,
2153 Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
2154 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128),
2155 Lo, Hi);
2156}
2157
2158void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
2159 SDValue &Hi) {
2160 ExpandFloatRes_Unary(N,
2161 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TANH_F32,
2162 Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
2163 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128),
2164 Lo, Hi);
2165}
2166
2167void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
2168 SDValue &Lo, SDValue &Hi) {
2169 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2170 Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
2171 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
2172 Call_PPCF128: RTLIB::TRUNC_PPCF128), Lo, Hi);
2173}
2174
2175void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
2176 SDValue &Hi) {
2177 if (ISD::isNormalLoad(N)) {
2178 ExpandRes_NormalLoad(N, Lo, Hi);
2179 return;
2180 }
2181
2182 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2183 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
2184 SDValue Chain = LD->getChain();
2185 SDValue Ptr = LD->getBasePtr();
2186 SDLoc dl(N);
2187
2188 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
2189 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2190 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2191
2192 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
2193 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
2194
2195 // Remember the chain.
2196 Chain = Hi.getValue(R: 1);
2197
2198 // The low part is zero.
2199 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2200
2201 // Modified the chain - switch anything that used the old chain to use the
2202 // new one.
2203 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
2204}
2205
2206void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2207 SDValue &Hi) {
2208 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2209 EVT VT = N->getValueType(ResNo: 0);
2210 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2211 bool Strict = N->isStrictFPOpcode();
2212 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
2213 EVT SrcVT = Src.getValueType();
2214 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2215 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2216 SDLoc dl(N);
2217 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
2218
2219 // TODO: Any other flags to propagate?
2220 SDNodeFlags Flags;
2221 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2222
2223 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2224 // When promoting partial word types to i32 we must honor the signedness,
2225 // though.
2226 if (SrcVT.bitsLE(VT: MVT::i32)) {
2227 // The integer can be represented exactly in an f64.
2228 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2229 if (Strict) {
2230 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
2231 Ops: {Chain, Src}, Flags);
2232 Chain = Hi.getValue(R: 1);
2233 } else
2234 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
2235 } else {
2236 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2237 if (SrcVT.bitsLE(VT: MVT::i64)) {
2238 Src = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
2239 VT: MVT::i64, Operand: Src);
2240 LC = RTLIB::SINTTOFP_I64_PPCF128;
2241 } else if (SrcVT.bitsLE(VT: MVT::i128)) {
2242 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i128, Operand: Src);
2243 LC = RTLIB::SINTTOFP_I128_PPCF128;
2244 }
2245 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2246
2247 TargetLowering::MakeLibCallOptions CallOptions;
2248 CallOptions.setIsSigned(true);
2249 std::pair<SDValue, SDValue> Tmp =
2250 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
2251 if (Strict)
2252 Chain = Tmp.second;
2253 GetPairElements(Pair: Tmp.first, Lo, Hi);
2254 }
2255
2256 // No need to complement for unsigned 32-bit integers
2257 if (isSigned || SrcVT.bitsLE(VT: MVT::i32)) {
2258 if (Strict)
2259 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2260
2261 return;
2262 }
2263
2264 // Unsigned - fix up the SINT_TO_FP value just calculated.
2265 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2266 // keep semantics correctness if the integer is not exactly representable
2267 // here. See ExpandLegalINT_TO_FP.
2268 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
2269 SrcVT = Src.getValueType();
2270
2271 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2272 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2273 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2274 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2275 ArrayRef<uint64_t> Parts;
2276
2277 switch (SrcVT.getSimpleVT().SimpleTy) {
2278 default:
2279 llvm_unreachable("Unsupported UINT_TO_FP!");
2280 case MVT::i32:
2281 Parts = TwoE32;
2282 break;
2283 case MVT::i64:
2284 Parts = TwoE64;
2285 break;
2286 case MVT::i128:
2287 Parts = TwoE128;
2288 break;
2289 }
2290
2291 // TODO: Are there other fast-math-flags to propagate to this FADD?
2292 SDValue NewLo = DAG.getConstantFP(
2293 Val: APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), DL: dl, VT: MVT::ppcf128);
2294 if (Strict) {
2295 Lo = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other),
2296 Ops: {Chain, Hi, NewLo}, Flags);
2297 Chain = Lo.getValue(R: 1);
2298 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2299 } else
2300 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
2301 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
2302 True: Lo, False: Hi, Cond: ISD::SETLT);
2303 GetPairElements(Pair: Lo, Lo, Hi);
2304}
2305
2306
2307//===----------------------------------------------------------------------===//
2308// Float Operand Expansion
2309//===----------------------------------------------------------------------===//
2310
2311/// ExpandFloatOperand - This method is called when the specified operand of the
2312/// specified node is found to need expansion. At this point, all of the result
2313/// types of the node are known to be legal, but other operands of the node may
2314/// need promotion or expansion as well as the specified one.
2315bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2316 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2317 SDValue Res = SDValue();
2318
2319 // See if the target wants to custom expand this node.
2320 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2321 return false;
2322
2323 switch (N->getOpcode()) {
2324 default:
2325#ifndef NDEBUG
2326 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2327 N->dump(&DAG); dbgs() << "\n";
2328#endif
2329 report_fatal_error(reason: "Do not know how to expand this operator's operand!");
2330
2331 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2332 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2333 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2334
2335 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2336 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2337 case ISD::STRICT_FP_ROUND:
2338 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2339 case ISD::STRICT_FP_TO_SINT:
2340 case ISD::STRICT_FP_TO_UINT:
2341 case ISD::FP_TO_SINT:
2342 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2343 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2344 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2345 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2346 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2347 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2348 case ISD::STRICT_FSETCC:
2349 case ISD::STRICT_FSETCCS:
2350 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2351 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
2352 OpNo); break;
2353 }
2354
2355 // If the result is null, the sub-method took care of registering results etc.
2356 if (!Res.getNode()) return false;
2357
2358 // If the result is N, the sub-method updated N in place. Tell the legalizer
2359 // core about this.
2360 if (Res.getNode() == N)
2361 return true;
2362
2363 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2364 "Invalid operand expansion");
2365
2366 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2367 return false;
2368}
2369
2370/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2371/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2372void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2373 SDValue &NewRHS,
2374 ISD::CondCode &CCCode,
2375 const SDLoc &dl, SDValue &Chain,
2376 bool IsSignaling) {
2377 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2378 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
2379 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
2380
2381 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2382
2383 // FIXME: This generated code sucks. We want to generate
2384 // FCMPU crN, hi1, hi2
2385 // BNE crN, L:
2386 // FCMPU crN, lo1, lo2
2387 // The following can be improved, but not that much.
2388 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2389 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2390 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
2391 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2392 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
2393 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
2394 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2395 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2396 Tmp1 =
2397 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
2398 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
2399 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2400 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2401 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
2402 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2403 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2404 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
2405 NewRHS = SDValue(); // LHS is the result, not a compare.
2406 Chain = OutputChain;
2407}
2408
2409SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2410 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
2411 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
2412 SDValue Chain;
2413 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2414
2415 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2416 // against zero to select between true and false values.
2417 if (!NewRHS.getNode()) {
2418 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2419 CCCode = ISD::SETNE;
2420 }
2421
2422 // Update N to have the operands specified.
2423 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2424 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
2425 Op5: N->getOperand(Num: 4)), 0);
2426}
2427
2428SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2429 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2430 "Logic only correct for ppcf128!");
2431 SDValue Lo, Hi;
2432 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2433 // The ppcf128 value is providing only the sign; take it from the
2434 // higher-order double (which must have the larger magnitude).
2435 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2436 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2437}
2438
2439SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2440 bool IsStrict = N->isStrictFPOpcode();
2441 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2442 "Logic only correct for ppcf128!");
2443 SDValue Lo, Hi;
2444 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2445
2446 if (!IsStrict)
2447 // Round it the rest of the way (e.g. to f32) if needed.
2448 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2449 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2450
2451 // Eliminate the node if the input float type is the same as the output float
2452 // type.
2453 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2454 // Connect the output chain to the input chain, unlinking the node.
2455 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2456 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2457 return SDValue();
2458 }
2459
2460 SDValue Expansion = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
2461 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
2462 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
2463 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2464 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2465 return SDValue();
2466}
2467
2468SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2469 EVT RVT = N->getValueType(ResNo: 0);
2470 SDLoc dl(N);
2471
2472 bool IsStrict = N->isStrictFPOpcode();
2473 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2474 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2475 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2476 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2477
2478 EVT NVT;
2479 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2480 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2481 "Unsupported FP_TO_XINT!");
2482 TargetLowering::MakeLibCallOptions CallOptions;
2483 std::pair<SDValue, SDValue> Tmp =
2484 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2485 if (!IsStrict)
2486 return Tmp.first;
2487
2488 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2489 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2490 return SDValue();
2491}
2492
2493SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2494 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2495 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2496 SDValue Chain;
2497 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2498
2499 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2500 // against zero to select between true and false values.
2501 if (!NewRHS.getNode()) {
2502 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2503 CCCode = ISD::SETNE;
2504 }
2505
2506 // Update N to have the operands specified.
2507 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2508 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2509 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2510}
2511
2512SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2513 bool IsStrict = N->isStrictFPOpcode();
2514 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2515 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2516 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2517 ISD::CondCode CCCode =
2518 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2519 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2520 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2521
2522 // FloatExpandSetCCOperands always returned a scalar.
2523 assert(!NewRHS.getNode() && "Expect to return scalar");
2524 assert(NewLHS.getValueType() == N->getValueType(0) &&
2525 "Unexpected setcc expansion!");
2526 if (Chain) {
2527 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2528 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2529 return SDValue();
2530 }
2531 return NewLHS;
2532}
2533
2534SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2535 if (ISD::isNormalStore(N))
2536 return ExpandOp_NormalStore(N, OpNo);
2537
2538 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2539 assert(OpNo == 1 && "Can only expand the stored value so far");
2540 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2541
2542 SDValue Chain = ST->getChain();
2543 SDValue Ptr = ST->getBasePtr();
2544
2545 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2546 VT: ST->getValue().getValueType());
2547 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2548 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2549 (void)NVT;
2550
2551 SDValue Lo, Hi;
2552 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2553
2554 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2555 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2556}
2557
2558SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2559 EVT RVT = N->getValueType(ResNo: 0);
2560 EVT RetVT = N->getOperand(Num: 0).getValueType();
2561 TargetLowering::MakeLibCallOptions CallOptions;
2562 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2563 Call_F32: RTLIB::LROUND_F32,
2564 Call_F64: RTLIB::LROUND_F64,
2565 Call_F80: RTLIB::LROUND_F80,
2566 Call_F128: RTLIB::LROUND_F128,
2567 Call_PPCF128: RTLIB::LROUND_PPCF128),
2568 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2569}
2570
2571SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2572 EVT RVT = N->getValueType(ResNo: 0);
2573 EVT RetVT = N->getOperand(Num: 0).getValueType();
2574 TargetLowering::MakeLibCallOptions CallOptions;
2575 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2576 Call_F32: RTLIB::LLROUND_F32,
2577 Call_F64: RTLIB::LLROUND_F64,
2578 Call_F80: RTLIB::LLROUND_F80,
2579 Call_F128: RTLIB::LLROUND_F128,
2580 Call_PPCF128: RTLIB::LLROUND_PPCF128),
2581 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2582}
2583
2584SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2585 EVT RVT = N->getValueType(ResNo: 0);
2586 EVT RetVT = N->getOperand(Num: 0).getValueType();
2587 TargetLowering::MakeLibCallOptions CallOptions;
2588 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2589 Call_F32: RTLIB::LRINT_F32,
2590 Call_F64: RTLIB::LRINT_F64,
2591 Call_F80: RTLIB::LRINT_F80,
2592 Call_F128: RTLIB::LRINT_F128,
2593 Call_PPCF128: RTLIB::LRINT_PPCF128),
2594 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2595}
2596
2597SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2598 EVT RVT = N->getValueType(ResNo: 0);
2599 EVT RetVT = N->getOperand(Num: 0).getValueType();
2600 TargetLowering::MakeLibCallOptions CallOptions;
2601 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2602 Call_F32: RTLIB::LLRINT_F32,
2603 Call_F64: RTLIB::LLRINT_F64,
2604 Call_F80: RTLIB::LLRINT_F80,
2605 Call_F128: RTLIB::LLRINT_F128,
2606 Call_PPCF128: RTLIB::LLRINT_PPCF128),
2607 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2608}
2609
2610//===----------------------------------------------------------------------===//
2611// Float Operand Promotion
2612//===----------------------------------------------------------------------===//
2613//
2614
2615static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2616 if (OpVT == MVT::f16)
2617 return ISD::FP16_TO_FP;
2618 if (RetVT == MVT::f16)
2619 return ISD::FP_TO_FP16;
2620 if (OpVT == MVT::bf16)
2621 return ISD::BF16_TO_FP;
2622 if (RetVT == MVT::bf16)
2623 return ISD::FP_TO_BF16;
2624 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2625}
2626
2627static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2628 if (OpVT == MVT::f16)
2629 return ISD::STRICT_FP16_TO_FP;
2630 if (RetVT == MVT::f16)
2631 return ISD::STRICT_FP_TO_FP16;
2632 if (OpVT == MVT::bf16)
2633 return ISD::STRICT_BF16_TO_FP;
2634 if (RetVT == MVT::bf16)
2635 return ISD::STRICT_FP_TO_BF16;
2636 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2637}
2638
2639SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2640 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2641 SDLoc SL(N);
2642
2643 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2644 EVT CastVT = CastVal.getValueType();
2645
2646 SDValue NewAtomic
2647 = DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl: SL, MemVT: CastVT,
2648 VTList: DAG.getVTList(VT1: CastVT, VT2: MVT::Other),
2649 Ops: { AM->getChain(), AM->getBasePtr(), CastVal },
2650 MMO: AM->getMemOperand());
2651
2652 SDValue Result = NewAtomic;
2653
2654 // Legalize the chain result by replacing uses of the old value chain with the
2655 // new one
2656 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2657
2658 return Result;
2659}
2660
2661//===----------------------------------------------------------------------===//
2662// Half Result Soft Promotion
2663//===----------------------------------------------------------------------===//
2664
2665void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2666 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2667 N->dump(&DAG));
2668 SDValue R = SDValue();
2669
2670 // See if the target wants to custom expand this node.
2671 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2672 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2673 return;
2674 }
2675
2676 switch (N->getOpcode()) {
2677 default:
2678#ifndef NDEBUG
2679 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2680 N->dump(&DAG); dbgs() << "\n";
2681#endif
2682 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2683 "result!");
2684
2685 case ISD::ARITH_FENCE:
2686 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2687 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2688 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2689 case ISD::EXTRACT_VECTOR_ELT:
2690 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2691 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2692 case ISD::STRICT_FP_ROUND:
2693 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2694
2695 // Unary FP Operations
2696 case ISD::FACOS:
2697 case ISD::FASIN:
2698 case ISD::FATAN:
2699 case ISD::FCBRT:
2700 case ISD::FCEIL:
2701 case ISD::FCOS:
2702 case ISD::FCOSH:
2703 case ISD::FEXP:
2704 case ISD::FEXP2:
2705 case ISD::FEXP10:
2706 case ISD::FFLOOR:
2707 case ISD::FLOG:
2708 case ISD::FLOG2:
2709 case ISD::FLOG10:
2710 case ISD::FNEARBYINT:
2711 case ISD::FREEZE:
2712 case ISD::FRINT:
2713 case ISD::FROUND:
2714 case ISD::FROUNDEVEN:
2715 case ISD::FSIN:
2716 case ISD::FSINH:
2717 case ISD::FSQRT:
2718 case ISD::FTRUNC:
2719 case ISD::FTAN:
2720 case ISD::FTANH:
2721 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2722 case ISD::FABS:
2723 R = SoftPromoteHalfRes_FABS(N);
2724 break;
2725 case ISD::FNEG:
2726 R = SoftPromoteHalfRes_FNEG(N);
2727 break;
2728 case ISD::AssertNoFPClass:
2729 R = SoftPromoteHalfRes_AssertNoFPClass(N);
2730 break;
2731
2732 // Binary FP Operations
2733 case ISD::FADD:
2734 case ISD::FDIV:
2735 case ISD::FMAXIMUM:
2736 case ISD::FMINIMUM:
2737 case ISD::FMAXIMUMNUM:
2738 case ISD::FMINIMUMNUM:
2739 case ISD::FMAXNUM:
2740 case ISD::FMINNUM:
2741 case ISD::FMUL:
2742 case ISD::FPOW:
2743 case ISD::FATAN2:
2744 case ISD::FREM:
2745 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2746
2747 case ISD::FMA: // FMA is same as FMAD
2748 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2749
2750 case ISD::FPOWI:
2751 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2752
2753 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2754
2755 case ISD::FMODF:
2756 case ISD::FSINCOS:
2757 case ISD::FSINCOSPI:
2758 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
2759 break;
2760
2761 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2762 case ISD::ATOMIC_LOAD:
2763 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2764 break;
2765 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2766 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2767 case ISD::STRICT_SINT_TO_FP:
2768 case ISD::STRICT_UINT_TO_FP:
2769 case ISD::SINT_TO_FP:
2770 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2771 case ISD::POISON:
2772 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2773 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2774 case ISD::VECREDUCE_FADD:
2775 case ISD::VECREDUCE_FMUL:
2776 case ISD::VECREDUCE_FMIN:
2777 case ISD::VECREDUCE_FMAX:
2778 case ISD::VECREDUCE_FMAXIMUM:
2779 case ISD::VECREDUCE_FMINIMUM:
2780 R = SoftPromoteHalfRes_VECREDUCE(N);
2781 break;
2782 case ISD::VECREDUCE_SEQ_FADD:
2783 case ISD::VECREDUCE_SEQ_FMUL:
2784 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2785 break;
2786 }
2787
2788 if (R.getNode())
2789 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
2790}
2791
2792SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2793 return DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: MVT::i16,
2794 Operand: BitConvertToInteger(Op: N->getOperand(Num: 0)));
2795}
2796
2797SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2798 return BitConvertToInteger(Op: N->getOperand(Num: 0));
2799}
2800
2801SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2802 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
2803
2804 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2805 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
2806 VT: MVT::i16);
2807}
2808
2809SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2810 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2811 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
2812 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
2813 N2: N->getOperand(Num: 1));
2814}
2815
2816SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2817 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2818 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
2819 SDLoc dl(N);
2820
2821 EVT LVT = LHS.getValueType();
2822 EVT RVT = RHS.getValueType();
2823
2824 unsigned LSize = LVT.getSizeInBits();
2825 unsigned RSize = RVT.getSizeInBits();
2826
2827 // First get the sign bit of second operand.
2828 SDValue SignBit = DAG.getNode(
2829 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
2830 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
2831 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
2832 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
2833
2834 // Shift right or sign-extend it if the two operands have different types.
2835 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2836 if (SizeDiff > 0) {
2837 SignBit =
2838 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
2839 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
2840 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2841 DL: DAG.getDataLayout())));
2842 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
2843 } else if (SizeDiff < 0) {
2844 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
2845 SignBit =
2846 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
2847 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
2848 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2849 DL: DAG.getDataLayout())));
2850 }
2851
2852 // Clear the sign bit of the first operand.
2853 SDValue Mask = DAG.getNode(
2854 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
2855 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
2856 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
2857 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
2858 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
2859
2860 // Or the value with the sign bit.
2861 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
2862}
2863
2864SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2865 EVT OVT = N->getValueType(ResNo: 0);
2866 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2867 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2868 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2869 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2870 SDNodeFlags Flags = N->getFlags();
2871 SDLoc dl(N);
2872
2873 // Promote to the larger FP type.
2874 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2875 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2876 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2877 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
2878
2879 SDValue Res;
2880 if (OVT == MVT::f16) {
2881 // If f16 fma is not natively supported, the value must be promoted to an
2882 // f64 (and not to f32!) to prevent double rounding issues.
2883 SDValue A64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op0, Flags);
2884 SDValue B64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op1, Flags);
2885 SDValue C64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op2, Flags);
2886
2887 // Prefer a wide FMA node if available; otherwise expand to mul+add.
2888 SDValue WideRes;
2889 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), MVT::f64)) {
2890 WideRes = DAG.getNode(Opcode: ISD::FMA, DL: dl, VT: MVT::f64, N1: A64, N2: B64, N3: C64, Flags);
2891 } else {
2892 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f64, N1: A64, N2: B64, Flags);
2893 WideRes = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f64, N1: Mul, N2: C64, Flags);
2894 }
2895
2896 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: MVT::f64, RetVT: OVT), DL: dl, VT: MVT::i16,
2897 Operand: WideRes);
2898 }
2899
2900 Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2, Flags);
2901 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2902}
2903
2904SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2905 EVT OVT = N->getValueType(ResNo: 0);
2906 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2907 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2908 SDValue Op1 = N->getOperand(Num: 1);
2909 SDLoc dl(N);
2910
2911 // Promote to the larger FP type.
2912 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
2913
2914 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2915
2916 // Convert back to FP16 as an integer.
2917 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2918}
2919
2920SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2921 EVT OVT = N->getValueType(ResNo: 0);
2922 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2923 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2924 SDLoc dl(N);
2925
2926 // Promote to the larger FP type.
2927 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2928
2929 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
2930 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
2931
2932 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2933
2934 // Convert back to FP16 as an integer.
2935 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2936}
2937
2938SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
2939 EVT OVT = N->getValueType(ResNo: 0);
2940 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2941 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2942 SDLoc dl(N);
2943
2944 // Promote to the larger FP type.
2945 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2946 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Op);
2947
2948 // Convert back to FP16 as an integer.
2949 ISD::NodeType Truncate = GetPromotionOpcode(OpVT: NVT, RetVT: OVT);
2950 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
2951 ++ResNum) {
2952 SDValue Trunc = DAG.getNode(Opcode: Truncate, DL: dl, VT: MVT::i16, Operand: Res.getValue(R: ResNum));
2953 SetSoftPromotedHalf(Op: SDValue(N, ResNum), Result: Trunc);
2954 }
2955
2956 return SDValue();
2957}
2958
2959SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2960 EVT RVT = N->getValueType(ResNo: 0);
2961 bool IsStrict = N->isStrictFPOpcode();
2962 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2963 EVT SVT = Op.getValueType();
2964
2965 // If the input type needs to be softened, do that now so that call lowering
2966 // will see the f16 type.
2967 if (getTypeAction(VT: SVT) == TargetLowering::TypeSoftenFloat) {
2968 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: RVT);
2969 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
2970
2971 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2972 Op = GetSoftenedFloat(Op);
2973 TargetLowering::MakeLibCallOptions CallOptions;
2974 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
2975 std::pair<SDValue, SDValue> Tmp =
2976 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
2977 if (IsStrict)
2978 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2979 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: MVT::i16, Operand: Tmp.first);
2980 }
2981
2982 if (IsStrict) {
2983 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
2984 ResultTys: {MVT::i16, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
2985 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2986 return Res;
2987 }
2988
2989 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: MVT::i16,
2990 Operand: N->getOperand(Num: 0));
2991}
2992
2993SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2994 LoadSDNode *L = cast<LoadSDNode>(Val: N);
2995
2996 // Load the value as an integer value with the same number of bits.
2997 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2998 SDValue NewL =
2999 DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: MVT::i16,
3000 dl: SDLoc(N), Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
3001 PtrInfo: L->getPointerInfo(), MemVT: MVT::i16, Alignment: L->getBaseAlign(),
3002 MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo());
3003 // Legalize the chain result by replacing uses of the old value chain with the
3004 // new one
3005 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3006 return NewL;
3007}
3008
3009SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3010 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
3011
3012 // Load the value as an integer value with the same number of bits.
3013 SDValue NewL = DAG.getAtomic(
3014 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: MVT::i16, VTList: DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other),
3015 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
3016
3017 // Legalize the chain result by replacing uses of the old value chain with the
3018 // new one
3019 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3020 return NewL;
3021}
3022
3023SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3024 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3025 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3026 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1,
3027 RHS: Op2);
3028}
3029
3030SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3031 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3032 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
3033 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
3034 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
3035 N5: N->getOperand(Num: 4));
3036}
3037
3038SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3039 EVT OVT = N->getValueType(ResNo: 0);
3040 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3041 SDLoc dl(N);
3042
3043 if (N->isStrictFPOpcode()) {
3044 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
3045 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
3046 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: NVT, RetVT: OVT), DL: dl,
3047 ResultTys: {MVT::i16, MVT::Other}, Ops: {Op.getValue(R: 1), Op});
3048 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3049 return Op;
3050 }
3051
3052 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
3053
3054 // Round the value to the softened type.
3055 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3056}
3057
3058SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3059 return DAG.getUNDEF(VT: MVT::i16);
3060}
3061
3062SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3063 EVT OVT = N->getValueType(ResNo: 0);
3064 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3065 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3066 SDLoc dl(N);
3067
3068 // Promote to the larger FP type.
3069 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3070
3071 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
3072
3073 // Convert back to FP16 as an integer.
3074 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3075}
3076
3077SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FABS(SDNode *N) {
3078 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3079 SDLoc dl(N);
3080
3081 // Clear the sign bit.
3082 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i16, N1: Op,
3083 N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: MVT::i16));
3084}
3085
3086SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FNEG(SDNode *N) {
3087 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3088 SDLoc dl(N);
3089
3090 // Invert the sign bit.
3091 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i16, N1: Op,
3092 N2: DAG.getConstant(Val: 0x8000, DL: dl, VT: MVT::i16));
3093}
3094
3095SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
3096 return GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3097}
3098
3099SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3100 EVT OVT = N->getValueType(ResNo: 0);
3101 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3102 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3103 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3104 SDLoc dl(N);
3105
3106 // Promote to the larger FP type.
3107 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
3108 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3109 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3110
3111 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3112
3113 // Convert back to FP16 as an integer.
3114 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3115}
3116
3117SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3118 // Expand and soften recursively.
3119 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
3120 return SDValue();
3121}
3122
3123SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3124 // Expand and soften.
3125 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3126 return SDValue();
3127}
3128
3129//===----------------------------------------------------------------------===//
3130// Half Operand Soft Promotion
3131//===----------------------------------------------------------------------===//
3132
3133bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3134 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3135 N->dump(&DAG));
3136 SDValue Res = SDValue();
3137
3138 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
3139 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3140 return false;
3141 }
3142
3143 // Nodes that use a promotion-requiring floating point operand, but doesn't
3144 // produce a soft promotion-requiring floating point result, need to be
3145 // legalized to use the soft promoted float operand. Nodes that produce at
3146 // least one soft promotion-requiring floating point result have their
3147 // operands legalized as a part of PromoteFloatResult.
3148 switch (N->getOpcode()) {
3149 default:
3150 #ifndef NDEBUG
3151 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3152 N->dump(&DAG); dbgs() << "\n";
3153 #endif
3154 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3155 "operand!");
3156
3157 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3158 case ISD::FAKE_USE:
3159 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
3160 break;
3161 case ISD::FCOPYSIGN:
3162 Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo);
3163 break;
3164 case ISD::FP_TO_SINT:
3165 case ISD::FP_TO_UINT:
3166 case ISD::STRICT_FP_TO_SINT:
3167 case ISD::STRICT_FP_TO_UINT:
3168 case ISD::LLRINT:
3169 case ISD::LLROUND:
3170 case ISD::LRINT:
3171 case ISD::LROUND:
3172 case ISD::STRICT_LLRINT:
3173 case ISD::STRICT_LLROUND:
3174 case ISD::STRICT_LRINT:
3175 case ISD::STRICT_LROUND:
3176 Res = SoftPromoteHalfOp_Op0WithStrict(N);
3177 break;
3178 case ISD::FP_TO_SINT_SAT:
3179 case ISD::FP_TO_UINT_SAT:
3180 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3181 case ISD::STRICT_FP_EXTEND:
3182 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3183 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3184 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3185 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3186 case ISD::ATOMIC_STORE:
3187 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3188 break;
3189 case ISD::STACKMAP:
3190 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3191 break;
3192 case ISD::PATCHPOINT:
3193 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3194 break;
3195 }
3196
3197 if (!Res.getNode())
3198 return false;
3199
3200 assert(Res.getNode() != N && "Expected a new node!");
3201
3202 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3203 "Invalid operand expansion");
3204
3205 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3206 return false;
3207}
3208
3209SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3210 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3211
3212 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3213}
3214
3215SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3216 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3217 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3218 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::Other, N1: N->getOperand(Num: 0),
3219 N2: Op);
3220}
3221
3222SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3223 unsigned OpNo) {
3224 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3225 SDValue Op1 = N->getOperand(Num: 1);
3226 EVT RVT = Op1.getValueType();
3227 SDLoc dl(N);
3228
3229 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3230
3231 Op1 = GetSoftPromotedHalf(Op: Op1);
3232 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3233
3234 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3235 N2: Op1);
3236}
3237
3238SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3239 EVT RVT = N->getValueType(ResNo: 0);
3240 bool IsStrict = N->isStrictFPOpcode();
3241 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3242 EVT SVT = Op.getValueType();
3243 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3244
3245 if (IsStrict) {
3246 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
3247 ResultTys: {RVT, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
3248 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3249 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3250 return SDValue();
3251 }
3252
3253 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3254}
3255
3256SDValue DAGTypeLegalizer::SoftPromoteHalfOp_Op0WithStrict(SDNode *N) {
3257 EVT RVT = N->getValueType(ResNo: 0);
3258 bool IsStrict = N->isStrictFPOpcode();
3259 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3260 EVT SVT = Op.getValueType();
3261 SDLoc dl(N);
3262
3263 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3264 Op = GetSoftPromotedHalf(Op);
3265
3266 if (IsStrict) {
3267 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: dl, ResultTys: {NVT, MVT::Other},
3268 Ops: {N->getOperand(Num: 0), Op});
3269 Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {RVT, MVT::Other},
3270 Ops: {Op.getValue(R: 1), Op});
3271 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3272 ReplaceValueWith(From: SDValue(N, 0), To: Op);
3273 return SDValue();
3274 }
3275
3276 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3277 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: RVT, Operand: Res);
3278}
3279
3280SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3281 EVT RVT = N->getValueType(ResNo: 0);
3282 SDValue Op = N->getOperand(Num: 0);
3283 EVT SVT = Op.getValueType();
3284 SDLoc dl(N);
3285
3286 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3287
3288 Op = GetSoftPromotedHalf(Op);
3289
3290 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3291
3292 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3293 N2: N->getOperand(Num: 1));
3294}
3295
3296SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3297 unsigned OpNo) {
3298 assert(OpNo == 0 && "Can only soften the comparison values");
3299 SDValue Op0 = N->getOperand(Num: 0);
3300 SDValue Op1 = N->getOperand(Num: 1);
3301 SDLoc dl(N);
3302
3303 EVT SVT = Op0.getValueType();
3304 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3305
3306 Op0 = GetSoftPromotedHalf(Op: Op0);
3307 Op1 = GetSoftPromotedHalf(Op: Op1);
3308
3309 // Promote to the larger FP type.
3310 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3311 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3312 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3313
3314 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3315 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3316}
3317
3318SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3319 SDValue Op0 = N->getOperand(Num: 0);
3320 SDValue Op1 = N->getOperand(Num: 1);
3321 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3322 SDLoc dl(N);
3323
3324 EVT SVT = Op0.getValueType();
3325 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3326
3327 Op0 = GetSoftPromotedHalf(Op: Op0);
3328 Op1 = GetSoftPromotedHalf(Op: Op1);
3329
3330 // Promote to the larger FP type.
3331 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3332 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3333 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3334
3335 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3336}
3337
3338SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3339 assert(OpNo == 1 && "Can only soften the stored value!");
3340 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3341 SDValue Val = ST->getValue();
3342 SDLoc dl(N);
3343
3344 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3345 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3346 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3347 MMO: ST->getMemOperand());
3348}
3349
3350SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3351 unsigned OpNo) {
3352 assert(OpNo == 1 && "Can only soften the stored value!");
3353 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
3354 SDValue Val = ST->getVal();
3355 SDLoc dl(N);
3356
3357 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3358 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: Promoted.getValueType(),
3359 Chain: ST->getChain(), Ptr: Promoted, Val: ST->getBasePtr(),
3360 MMO: ST->getMemOperand());
3361}
3362
3363SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3364 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3365 SmallVector<SDValue> NewOps(N->ops());
3366 SDValue Op = N->getOperand(Num: OpNo);
3367 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3368 SDValue NewNode =
3369 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3370
3371 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3372 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3373
3374 return SDValue(); // Signal that we replaced the node ourselves.
3375}
3376
3377SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3378 unsigned OpNo) {
3379 assert(OpNo >= 7);
3380 SmallVector<SDValue> NewOps(N->ops());
3381 SDValue Op = N->getOperand(Num: OpNo);
3382 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3383 SDValue NewNode =
3384 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3385
3386 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3387 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3388
3389 return SDValue(); // Signal that we replaced the node ourselves.
3390}
3391