1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error(reason: "Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::STRICT_FADD:
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::STRICT_FACOS:
80 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
81 case ISD::STRICT_FASIN:
82 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
83 case ISD::STRICT_FATAN:
84 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
85 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
86 case ISD::STRICT_FCEIL:
87 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
88 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
89 case ISD::STRICT_FCOS:
90 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
91 case ISD::STRICT_FCOSH:
92 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
93 case ISD::STRICT_FDIV:
94 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
95 case ISD::STRICT_FEXP:
96 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
97 case ISD::STRICT_FEXP2:
98 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
99 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
100 case ISD::STRICT_FFLOOR:
101 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
102 case ISD::STRICT_FLOG:
103 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
104 case ISD::STRICT_FLOG2:
105 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
106 case ISD::STRICT_FLOG10:
107 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
108 case ISD::STRICT_FMA:
109 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
110 case ISD::STRICT_FMUL:
111 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
112 case ISD::STRICT_FNEARBYINT:
113 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
114 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
115 case ISD::STRICT_FP_EXTEND:
116 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
117 case ISD::STRICT_FP_ROUND:
118 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
119 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
120 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
121 case ISD::STRICT_FPOW:
122 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
123 case ISD::STRICT_FPOWI:
124 case ISD::FPOWI:
125 case ISD::FLDEXP:
126 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
127 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
128 case ISD::STRICT_FREM:
129 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
130 case ISD::STRICT_FRINT:
131 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
132 case ISD::STRICT_FROUND:
133 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
134 case ISD::STRICT_FROUNDEVEN:
135 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
136 case ISD::STRICT_FSIN:
137 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
138 case ISD::STRICT_FSINH:
139 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
140 case ISD::STRICT_FSQRT:
141 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
142 case ISD::STRICT_FSUB:
143 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
144 case ISD::STRICT_FTAN:
145 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
146 case ISD::STRICT_FTANH:
147 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
148 case ISD::STRICT_FTRUNC:
149 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
150 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
151 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
152 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
153 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
154 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
155 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
156 case ISD::STRICT_SINT_TO_FP:
157 case ISD::STRICT_UINT_TO_FP:
158 case ISD::SINT_TO_FP:
159 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
160 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
161 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
162 case ISD::VECREDUCE_FADD:
163 case ISD::VECREDUCE_FMUL:
164 case ISD::VECREDUCE_FMIN:
165 case ISD::VECREDUCE_FMAX:
166 case ISD::VECREDUCE_FMAXIMUM:
167 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
168 case ISD::VECREDUCE_SEQ_FADD:
169 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
170 // clang-format on
171 }
172
173 // If R is null, the sub-method took care of registering the result.
174 if (R.getNode()) {
175 assert(R.getNode() != N);
176 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
177 }
178}
179
180SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
181 bool IsStrict = N->isStrictFPOpcode();
182 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
183 unsigned Offset = IsStrict ? 1 : 0;
184 assert(N->getNumOperands() == (1 + Offset) &&
185 "Unexpected number of operands!");
186 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
187 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
188 TargetLowering::MakeLibCallOptions CallOptions;
189 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
190 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
191 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
192 CallOptions, dl: SDLoc(N),
193 Chain);
194 if (IsStrict)
195 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
196 return Tmp.first;
197}
198
199SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
200 bool IsStrict = N->isStrictFPOpcode();
201 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
202 unsigned Offset = IsStrict ? 1 : 0;
203 assert(N->getNumOperands() == (2 + Offset) &&
204 "Unexpected number of operands!");
205 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
206 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
207 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
208 TargetLowering::MakeLibCallOptions CallOptions;
209 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
210 N->getOperand(Num: 1 + Offset).getValueType() };
211 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
212 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
213 CallOptions, dl: SDLoc(N),
214 Chain);
215 if (IsStrict)
216 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
217 return Tmp.first;
218}
219
220SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
221 return BitConvertToInteger(Op: N->getOperand(Num: 0));
222}
223
224SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
225 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
226 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
227 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
228}
229
230SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
231 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
232 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
233 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
234 return NewFence;
235}
236
237SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
238 unsigned ResNo) {
239 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
240 return BitConvertToInteger(Op);
241}
242
243SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
244 // Convert the inputs to integers, and build a new pair out of them.
245 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
246 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
247 VT: N->getValueType(ResNo: 0)),
248 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
249 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
250}
251
252SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
253 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
254 // In ppcf128, the high 64 bits are always first in memory regardless
255 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
256 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
257 // way. However, APInt's are serialized in an Endian-sensitive fashion,
258 // so on big-Endian targets, the two doubles are output in the wrong
259 // order. Fix this by manually flipping the order of the high 64 bits
260 // and the low 64 bits here.
261 if (DAG.getDataLayout().isBigEndian() &&
262 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
263 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
264 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
265 APInt Val(128, words);
266 return DAG.getConstant(Val, DL: SDLoc(CN),
267 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
268 VT: CN->getValueType(ResNo: 0)));
269 } else {
270 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
271 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
272 VT: CN->getValueType(ResNo: 0)));
273 }
274}
275
276SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
277 SDValue Src = N->getOperand(Num: 0);
278 assert(Src.getValueType() == MVT::ppcf128 &&
279 "In floats only ppcf128 can be extracted by element!");
280 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(N),
281 VT: N->getValueType(ResNo: 0).changeTypeToInteger(),
282 N1: DAG.getBitcast(VT: MVT::i128, V: Src), N2: N->getOperand(Num: 1));
283}
284
285SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
286 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
287 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
288 VT: NewOp.getValueType().getVectorElementType(),
289 N1: NewOp, N2: N->getOperand(Num: 1));
290}
291
292SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
293 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
294 unsigned Size = NVT.getSizeInBits();
295
296 // Mask = ~(1 << (Size-1))
297 APInt API = APInt::getAllOnes(numBits: Size);
298 API.clearBit(BitPosition: Size - 1);
299 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
300 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
301 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
302}
303
304SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
305 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
306 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
307 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
308 Call_F32: RTLIB::FMIN_F32,
309 Call_F64: RTLIB::FMIN_F64,
310 Call_F80: RTLIB::FMIN_F80,
311 Call_F128: RTLIB::FMIN_F128,
312 Call_PPCF128: RTLIB::FMIN_PPCF128));
313}
314
315SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
316 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
317 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
318 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
319 Call_F32: RTLIB::FMAX_F32,
320 Call_F64: RTLIB::FMAX_F64,
321 Call_F80: RTLIB::FMAX_F80,
322 Call_F128: RTLIB::FMAX_F128,
323 Call_PPCF128: RTLIB::FMAX_PPCF128));
324}
325
326SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
327 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
328 Call_F32: RTLIB::ADD_F32,
329 Call_F64: RTLIB::ADD_F64,
330 Call_F80: RTLIB::ADD_F80,
331 Call_F128: RTLIB::ADD_F128,
332 Call_PPCF128: RTLIB::ADD_PPCF128));
333}
334
335SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
336 return SoftenFloatRes_Unary(
337 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64,
338 Call_F80: RTLIB::ACOS_F80, Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128));
339}
340
341SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
342 return SoftenFloatRes_Unary(
343 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64,
344 Call_F80: RTLIB::ASIN_F80, Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128));
345}
346
347SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
348 return SoftenFloatRes_Unary(
349 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64,
350 Call_F80: RTLIB::ATAN_F80, Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128));
351}
352
353SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
354 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
355 Call_F32: RTLIB::CBRT_F32,
356 Call_F64: RTLIB::CBRT_F64,
357 Call_F80: RTLIB::CBRT_F80,
358 Call_F128: RTLIB::CBRT_F128,
359 Call_PPCF128: RTLIB::CBRT_PPCF128));
360}
361
362SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
363 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
364 Call_F32: RTLIB::CEIL_F32,
365 Call_F64: RTLIB::CEIL_F64,
366 Call_F80: RTLIB::CEIL_F80,
367 Call_F128: RTLIB::CEIL_F128,
368 Call_PPCF128: RTLIB::CEIL_PPCF128));
369}
370
371SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
372 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
373 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
374 SDLoc dl(N);
375
376 EVT LVT = LHS.getValueType();
377 EVT RVT = RHS.getValueType();
378
379 unsigned LSize = LVT.getSizeInBits();
380 unsigned RSize = RVT.getSizeInBits();
381
382 // First get the sign bit of second operand.
383 SDValue SignBit = DAG.getNode(
384 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
385 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
386 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
387 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
388
389 // Shift right or sign-extend it if the two operands have different types.
390 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
391 if (SizeDiff > 0) {
392 SignBit =
393 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
394 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
395 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
396 DL: DAG.getDataLayout())));
397 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
398 } else if (SizeDiff < 0) {
399 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
400 SignBit =
401 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
402 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
403 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
404 DL: DAG.getDataLayout())));
405 }
406
407 // Clear the sign bit of the first operand.
408 SDValue Mask = DAG.getNode(
409 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
410 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
411 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
412 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
413 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
414
415 // Or the value with the sign bit.
416 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
417}
418
419SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
420 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
421 Call_F32: RTLIB::COS_F32,
422 Call_F64: RTLIB::COS_F64,
423 Call_F80: RTLIB::COS_F80,
424 Call_F128: RTLIB::COS_F128,
425 Call_PPCF128: RTLIB::COS_PPCF128));
426}
427
428SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
429 return SoftenFloatRes_Unary(
430 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64,
431 Call_F80: RTLIB::COSH_F80, Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128));
432}
433
434SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
435 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
436 Call_F32: RTLIB::DIV_F32,
437 Call_F64: RTLIB::DIV_F64,
438 Call_F80: RTLIB::DIV_F80,
439 Call_F128: RTLIB::DIV_F128,
440 Call_PPCF128: RTLIB::DIV_PPCF128));
441}
442
443SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
444 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
445 Call_F32: RTLIB::EXP_F32,
446 Call_F64: RTLIB::EXP_F64,
447 Call_F80: RTLIB::EXP_F80,
448 Call_F128: RTLIB::EXP_F128,
449 Call_PPCF128: RTLIB::EXP_PPCF128));
450}
451
452SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
453 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
454 Call_F32: RTLIB::EXP2_F32,
455 Call_F64: RTLIB::EXP2_F64,
456 Call_F80: RTLIB::EXP2_F80,
457 Call_F128: RTLIB::EXP2_F128,
458 Call_PPCF128: RTLIB::EXP2_PPCF128));
459}
460
461SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
462 return SoftenFloatRes_Unary(
463 N,
464 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64,
465 Call_F80: RTLIB::EXP10_F80, Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128));
466}
467
468SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
469 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
470 Call_F32: RTLIB::FLOOR_F32,
471 Call_F64: RTLIB::FLOOR_F64,
472 Call_F80: RTLIB::FLOOR_F80,
473 Call_F128: RTLIB::FLOOR_F128,
474 Call_PPCF128: RTLIB::FLOOR_PPCF128));
475}
476
477SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
478 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
479 Call_F32: RTLIB::LOG_F32,
480 Call_F64: RTLIB::LOG_F64,
481 Call_F80: RTLIB::LOG_F80,
482 Call_F128: RTLIB::LOG_F128,
483 Call_PPCF128: RTLIB::LOG_PPCF128));
484}
485
486SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
487 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
488 Call_F32: RTLIB::LOG2_F32,
489 Call_F64: RTLIB::LOG2_F64,
490 Call_F80: RTLIB::LOG2_F80,
491 Call_F128: RTLIB::LOG2_F128,
492 Call_PPCF128: RTLIB::LOG2_PPCF128));
493}
494
495SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
496 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
497 Call_F32: RTLIB::LOG10_F32,
498 Call_F64: RTLIB::LOG10_F64,
499 Call_F80: RTLIB::LOG10_F80,
500 Call_F128: RTLIB::LOG10_F128,
501 Call_PPCF128: RTLIB::LOG10_PPCF128));
502}
503
504SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
505 bool IsStrict = N->isStrictFPOpcode();
506 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
507 unsigned Offset = IsStrict ? 1 : 0;
508 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
509 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
510 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
511 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
512 TargetLowering::MakeLibCallOptions CallOptions;
513 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
514 N->getOperand(Num: 1 + Offset).getValueType(),
515 N->getOperand(Num: 2 + Offset).getValueType() };
516 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
517 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
518 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
519 Call_F32: RTLIB::FMA_F32,
520 Call_F64: RTLIB::FMA_F64,
521 Call_F80: RTLIB::FMA_F80,
522 Call_F128: RTLIB::FMA_F128,
523 Call_PPCF128: RTLIB::FMA_PPCF128),
524 RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
525 if (IsStrict)
526 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
527 return Tmp.first;
528}
529
530SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
531 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
532 Call_F32: RTLIB::MUL_F32,
533 Call_F64: RTLIB::MUL_F64,
534 Call_F80: RTLIB::MUL_F80,
535 Call_F128: RTLIB::MUL_F128,
536 Call_PPCF128: RTLIB::MUL_PPCF128));
537}
538
539SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
540 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
541 Call_F32: RTLIB::NEARBYINT_F32,
542 Call_F64: RTLIB::NEARBYINT_F64,
543 Call_F80: RTLIB::NEARBYINT_F80,
544 Call_F128: RTLIB::NEARBYINT_F128,
545 Call_PPCF128: RTLIB::NEARBYINT_PPCF128));
546}
547
548SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
549 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
550 SDLoc dl(N);
551
552 // Expand Y = FNEG(X) -> Y = X ^ sign mask
553 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
554 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
555 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
556}
557
558SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
559 bool IsStrict = N->isStrictFPOpcode();
560 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
561 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
562
563 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
564
565 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteFloat) {
566 Op = GetPromotedFloat(Op);
567 // If the promotion did the FP_EXTEND to the destination type for us,
568 // there's nothing left to do here.
569 if (Op.getValueType() == N->getValueType(ResNo: 0)) {
570 if (IsStrict)
571 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
572 return BitConvertToInteger(Op);
573 }
574 }
575
576 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
577 // -> f32, so proceed in two stages. Also, it's entirely possible for both
578 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
579 // than FP16_TO_FP.
580 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
581 N->getValueType(ResNo: 0) != MVT::f32) {
582 if (IsStrict) {
583 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
584 ResultTys: { MVT::f32, MVT::Other }, Ops: { Chain, Op });
585 Chain = Op.getValue(R: 1);
586 } else {
587 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT: MVT::f32, Operand: Op);
588 }
589 }
590
591 if (Op.getValueType() == MVT::bf16) {
592 // FIXME: Need ReplaceValueWith on chain in strict case
593 return SoftenFloatRes_BF16_TO_FP(N);
594 }
595
596 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
597 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
598 TargetLowering::MakeLibCallOptions CallOptions;
599 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
600 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
601 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
602 CallOptions, dl: SDLoc(N),
603 Chain);
604 if (IsStrict)
605 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
606 return Tmp.first;
607}
608
609// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
610// nodes?
611SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
612 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
613 SDValue Op = N->getOperand(Num: 0);
614 TargetLowering::MakeLibCallOptions CallOptions;
615 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
616 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
617 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
618 CallOptions, dl: SDLoc(N)).first;
619 if (N->getValueType(ResNo: 0) == MVT::f32)
620 return Res32;
621
622 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
623 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: MVT::f32, RetVT: N->getValueType(ResNo: 0));
624 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
625 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
626}
627
628// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
629// nodes?
630SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
631 assert(N->getValueType(0) == MVT::f32 &&
632 "Can only soften BF16_TO_FP with f32 result");
633 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
634 SDValue Op = N->getOperand(Num: 0);
635 SDLoc DL(N);
636 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT,
637 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op));
638 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
639 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
640 return Res;
641}
642
643SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
644 bool IsStrict = N->isStrictFPOpcode();
645 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
646 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
647 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
648 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
649 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
650 TargetLowering::MakeLibCallOptions CallOptions;
651 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
652 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
653 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
654 CallOptions, dl: SDLoc(N),
655 Chain);
656 if (IsStrict)
657 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
658 return Tmp.first;
659}
660
661SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
662 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
663 Call_F32: RTLIB::POW_F32,
664 Call_F64: RTLIB::POW_F64,
665 Call_F80: RTLIB::POW_F80,
666 Call_F128: RTLIB::POW_F128,
667 Call_PPCF128: RTLIB::POW_PPCF128));
668}
669
670SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
671 bool IsStrict = N->isStrictFPOpcode();
672 unsigned Offset = IsStrict ? 1 : 0;
673 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
674 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
675 "Unsupported power type!");
676 bool IsPowI =
677 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
678
679 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0))
680 : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0));
681 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
682 if (!TLI.getLibcallName(Call: LC)) {
683 // Some targets don't have a powi libcall; use pow instead.
684 // FIXME: Implement this if some target needs it.
685 DAG.getContext()->emitError(ErrorStr: "Don't know how to soften fpowi to fpow");
686 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
687 }
688
689 if (DAG.getLibInfo().getIntSize() !=
690 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
691 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
692 // would use the wrong type for the argument.
693 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
694 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
695 }
696
697 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
698 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
699 N->getOperand(Num: 1 + Offset) };
700 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
701 TargetLowering::MakeLibCallOptions CallOptions;
702 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
703 N->getOperand(Num: 1 + Offset).getValueType() };
704 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
705 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
706 CallOptions, dl: SDLoc(N),
707 Chain);
708 if (IsStrict)
709 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
710 return Tmp.first;
711}
712
713SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
714 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
715 EVT VT0 = N->getValueType(ResNo: 0);
716 EVT VT1 = N->getValueType(ResNo: 1);
717 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT0);
718
719 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
720 // If the exponent does not match with sizeof(int) a libcall would use the
721 // wrong type for the argument.
722 // TODO: Should be able to handle mismatches.
723 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
724 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
725 }
726
727 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
728 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
729
730 SDLoc DL(N);
731
732 TargetLowering::MakeLibCallOptions CallOptions;
733 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
734 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
735
736 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
737 // but we only really need to handle the 0th one for softening anyway.
738 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0, Value: true);
739
740 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, RetVT: NVT0, Ops, CallOptions, dl: DL,
741 /*Chain=*/SDValue());
742 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
743 auto PtrInfo =
744 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
745
746 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
747
748 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
749 return ReturnVal;
750}
751
752SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
753 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
754 Call_F32: RTLIB::REM_F32,
755 Call_F64: RTLIB::REM_F64,
756 Call_F80: RTLIB::REM_F80,
757 Call_F128: RTLIB::REM_F128,
758 Call_PPCF128: RTLIB::REM_PPCF128));
759}
760
761SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
762 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
763 Call_F32: RTLIB::RINT_F32,
764 Call_F64: RTLIB::RINT_F64,
765 Call_F80: RTLIB::RINT_F80,
766 Call_F128: RTLIB::RINT_F128,
767 Call_PPCF128: RTLIB::RINT_PPCF128));
768}
769
770SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
771 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
772 Call_F32: RTLIB::ROUND_F32,
773 Call_F64: RTLIB::ROUND_F64,
774 Call_F80: RTLIB::ROUND_F80,
775 Call_F128: RTLIB::ROUND_F128,
776 Call_PPCF128: RTLIB::ROUND_PPCF128));
777}
778
779SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
780 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
781 Call_F32: RTLIB::ROUNDEVEN_F32,
782 Call_F64: RTLIB::ROUNDEVEN_F64,
783 Call_F80: RTLIB::ROUNDEVEN_F80,
784 Call_F128: RTLIB::ROUNDEVEN_F128,
785 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128));
786}
787
788SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
789 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
790 Call_F32: RTLIB::SIN_F32,
791 Call_F64: RTLIB::SIN_F64,
792 Call_F80: RTLIB::SIN_F80,
793 Call_F128: RTLIB::SIN_F128,
794 Call_PPCF128: RTLIB::SIN_PPCF128));
795}
796
797SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
798 return SoftenFloatRes_Unary(
799 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64,
800 Call_F80: RTLIB::SINH_F80, Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128));
801}
802
803SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
804 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
805 Call_F32: RTLIB::SQRT_F32,
806 Call_F64: RTLIB::SQRT_F64,
807 Call_F80: RTLIB::SQRT_F80,
808 Call_F128: RTLIB::SQRT_F128,
809 Call_PPCF128: RTLIB::SQRT_PPCF128));
810}
811
812SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
813 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
814 Call_F32: RTLIB::SUB_F32,
815 Call_F64: RTLIB::SUB_F64,
816 Call_F80: RTLIB::SUB_F80,
817 Call_F128: RTLIB::SUB_F128,
818 Call_PPCF128: RTLIB::SUB_PPCF128));
819}
820
821SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
822 return SoftenFloatRes_Unary(
823 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64,
824 Call_F80: RTLIB::TAN_F80, Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128));
825}
826
827SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
828 return SoftenFloatRes_Unary(
829 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64,
830 Call_F80: RTLIB::TANH_F80, Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128));
831}
832
833SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
834 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
835 Call_F32: RTLIB::TRUNC_F32,
836 Call_F64: RTLIB::TRUNC_F64,
837 Call_F80: RTLIB::TRUNC_F80,
838 Call_F128: RTLIB::TRUNC_F128,
839 Call_PPCF128: RTLIB::TRUNC_PPCF128));
840}
841
842SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
843 LoadSDNode *L = cast<LoadSDNode>(Val: N);
844 EVT VT = N->getValueType(ResNo: 0);
845 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
846 SDLoc dl(N);
847
848 auto MMOFlags =
849 L->getMemOperand()->getFlags() &
850 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
851 SDValue NewL;
852 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
853 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: NVT, dl,
854 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
855 PtrInfo: L->getPointerInfo(), MemVT: NVT, Alignment: L->getOriginalAlign(),
856 MMOFlags, AAInfo: L->getAAInfo());
857 // Legalized the chain result - switch anything that used the old chain to
858 // use the new one.
859 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
860 return NewL;
861 }
862
863 // Do a non-extending load followed by FP_EXTEND.
864 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
865 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
866 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(),
867 Alignment: L->getOriginalAlign(), MMOFlags, AAInfo: L->getAAInfo());
868 // Legalized the chain result - switch anything that used the old chain to
869 // use the new one.
870 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
871 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
872 return BitConvertToInteger(Op: ExtendNode);
873}
874
875SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
876 AtomicSDNode *L = cast<AtomicSDNode>(Val: N);
877 EVT VT = N->getValueType(ResNo: 0);
878 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
879 SDLoc dl(N);
880
881 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
882 SDValue NewL =
883 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
884 Ops: {L->getChain(), L->getBasePtr()}, MMO: L->getMemOperand());
885
886 // Legalized the chain result - switch anything that used the old chain to
887 // use the new one.
888 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
889 return NewL;
890 }
891
892 report_fatal_error(reason: "softening fp extending atomic load not handled");
893}
894
895SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
896 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
897 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
898 return DAG.getSelect(DL: SDLoc(N),
899 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
900}
901
902SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
903 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
904 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
905 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
906 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
907 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
908}
909
910SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
911 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
912 VT: N->getValueType(ResNo: 0)));
913}
914
915SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
916 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
917 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
918 EVT VT = N->getValueType(ResNo: 0);
919 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
920 SDLoc dl(N);
921
922 SDValue NewVAARG;
923 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
924 Align: N->getConstantOperandVal(Num: 3));
925
926 // Legalized the chain result - switch anything that used the old chain to
927 // use the new one.
928 if (N != NewVAARG.getValue(R: 1).getNode())
929 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
930 return NewVAARG;
931}
932
933SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
934 bool IsStrict = N->isStrictFPOpcode();
935 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
936 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
937 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
938 EVT RVT = N->getValueType(ResNo: 0);
939 EVT NVT = EVT();
940 SDLoc dl(N);
941
942 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
943 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
944 // match. Look for an appropriate libcall.
945 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
946 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
947 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
948 NVT = (MVT::SimpleValueType)t;
949 // The source needs to big enough to hold the operand.
950 if (NVT.bitsGE(VT: SVT))
951 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
952 }
953 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
954
955 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
956 // Sign/zero extend the argument if the libcall takes a larger type.
957 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
958 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
959 TargetLowering::MakeLibCallOptions CallOptions;
960 CallOptions.setSExt(Signed);
961 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
962 std::pair<SDValue, SDValue> Tmp =
963 TLI.makeLibCall(DAG, LC, RetVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT),
964 Ops: Op, CallOptions, dl, Chain);
965
966 if (IsStrict)
967 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
968 return Tmp.first;
969}
970
971SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
972 // Expand and soften recursively.
973 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
974 return SDValue();
975}
976
977SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
978 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
979 return SDValue();
980}
981
982//===----------------------------------------------------------------------===//
983// Convert Float Operand to Integer
984//===----------------------------------------------------------------------===//
985
986bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
987 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
988 SDValue Res = SDValue();
989
990 switch (N->getOpcode()) {
991 default:
992#ifndef NDEBUG
993 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
994 N->dump(&DAG); dbgs() << "\n";
995#endif
996 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
997
998 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
999 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1000 case ISD::STRICT_FP_TO_FP16:
1001 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1002 case ISD::FP_TO_BF16:
1003 case ISD::STRICT_FP_TO_BF16:
1004 case ISD::STRICT_FP_ROUND:
1005 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1006 case ISD::STRICT_FP_TO_SINT:
1007 case ISD::STRICT_FP_TO_UINT:
1008 case ISD::FP_TO_SINT:
1009 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1010 case ISD::FP_TO_SINT_SAT:
1011 case ISD::FP_TO_UINT_SAT:
1012 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1013 case ISD::STRICT_LROUND:
1014 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1015 case ISD::STRICT_LLROUND:
1016 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1017 case ISD::STRICT_LRINT:
1018 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1019 case ISD::STRICT_LLRINT:
1020 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1021 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1022 case ISD::STRICT_FSETCC:
1023 case ISD::STRICT_FSETCCS:
1024 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1025 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1026 case ISD::ATOMIC_STORE:
1027 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1028 break;
1029 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1030 }
1031
1032 // If the result is null, the sub-method took care of registering results etc.
1033 if (!Res.getNode()) return false;
1034
1035 // If the result is N, the sub-method updated N in place. Tell the legalizer
1036 // core about this to re-analyze.
1037 if (Res.getNode() == N)
1038 return true;
1039
1040 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1041 "Invalid operand softening");
1042
1043 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1044 return false;
1045}
1046
1047SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1048 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
1049
1050 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
1051}
1052
1053SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1054 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1055 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1056 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1057 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1058 N->getOpcode() == ISD::FP_TO_BF16 ||
1059 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1060 N->getOpcode() == ISD::STRICT_FP_ROUND);
1061
1062 bool IsStrict = N->isStrictFPOpcode();
1063 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1064 EVT SVT = Op.getValueType();
1065 EVT RVT = N->getValueType(ResNo: 0);
1066 EVT FloatRVT = RVT;
1067 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1068 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1069 FloatRVT = MVT::f16;
1070 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1071 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1072 FloatRVT = MVT::bf16;
1073
1074 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
1075 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1076
1077 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1078 Op = GetSoftenedFloat(Op);
1079 TargetLowering::MakeLibCallOptions CallOptions;
1080 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
1081 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op,
1082 CallOptions, dl: SDLoc(N),
1083 Chain);
1084 if (IsStrict) {
1085 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1086 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1087 return SDValue();
1088 }
1089 return Tmp.first;
1090}
1091
1092SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1093 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1094 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1095
1096 EVT VT = NewLHS.getValueType();
1097 NewLHS = GetSoftenedFloat(Op: NewLHS);
1098 NewRHS = GetSoftenedFloat(Op: NewRHS);
1099 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1100 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1101
1102 // If softenSetCCOperands returned a scalar, we need to compare the result
1103 // against zero to select between true and false values.
1104 if (!NewRHS.getNode()) {
1105 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1106 CCCode = ISD::SETNE;
1107 }
1108
1109 // Update N to have the operands specified.
1110 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1111 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1112 Op5: N->getOperand(Num: 4)),
1113 0);
1114}
1115
1116// Even if the result type is legal, no libcall may exactly match. (e.g. We
1117// don't have FP-i8 conversions) This helper method looks for an appropriate
1118// promoted libcall.
1119static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1120 bool Signed) {
1121 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1122 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1123 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1124 ++IntVT) {
1125 Promoted = (MVT::SimpleValueType)IntVT;
1126 // The type needs to big enough to hold the result.
1127 if (Promoted.bitsGE(VT: RetVT))
1128 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1129 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1130 }
1131 return LC;
1132}
1133
1134SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1135 bool IsStrict = N->isStrictFPOpcode();
1136 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1137 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1138
1139 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1140 EVT SVT = Op.getValueType();
1141 EVT RVT = N->getValueType(ResNo: 0);
1142 EVT NVT = EVT();
1143 SDLoc dl(N);
1144
1145 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1146 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1147 // match, eg. we don't have fp -> i8 conversions.
1148 // Look for an appropriate libcall.
1149 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1150 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1151 "Unsupported FP_TO_XINT!");
1152
1153 Op = GetSoftenedFloat(Op);
1154 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1155 TargetLowering::MakeLibCallOptions CallOptions;
1156 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
1157 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1158 CallOptions, dl, Chain);
1159
1160 // Truncate the result if the libcall returns a larger type.
1161 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1162
1163 if (!IsStrict)
1164 return Res;
1165
1166 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1167 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1168 return SDValue();
1169}
1170
1171SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1172 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1173 return Res;
1174}
1175
1176SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1177 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1178 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1179
1180 EVT VT = NewLHS.getValueType();
1181 NewLHS = GetSoftenedFloat(Op: NewLHS);
1182 NewRHS = GetSoftenedFloat(Op: NewRHS);
1183 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1184 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1185
1186 // If softenSetCCOperands returned a scalar, we need to compare the result
1187 // against zero to select between true and false values.
1188 if (!NewRHS.getNode()) {
1189 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1190 CCCode = ISD::SETNE;
1191 }
1192
1193 // Update N to have the operands specified.
1194 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1195 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1196 Op5: DAG.getCondCode(Cond: CCCode)),
1197 0);
1198}
1199
1200SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1201 bool IsStrict = N->isStrictFPOpcode();
1202 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1203 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1204 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1205 ISD::CondCode CCCode =
1206 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1207
1208 EVT VT = Op0.getValueType();
1209 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1210 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1211 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1212 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1213
1214 // Update N to have the operands specified.
1215 if (NewRHS.getNode()) {
1216 if (IsStrict)
1217 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1218 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1219 else
1220 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1221 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1222 }
1223
1224 // Otherwise, softenSetCCOperands returned a scalar, use it.
1225 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1226 "Unexpected setcc expansion!");
1227
1228 if (IsStrict) {
1229 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1230 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1231 return SDValue();
1232 }
1233 return NewLHS;
1234}
1235
1236SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1237 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1238 assert(OpNo == 1 && "Can only soften the stored value!");
1239 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1240 SDValue Val = ST->getValue();
1241 SDLoc dl(N);
1242
1243 if (ST->isTruncatingStore())
1244 // Do an FP_ROUND followed by a non-truncating store.
1245 Val = BitConvertToInteger(
1246 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1247 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1248 else
1249 Val = GetSoftenedFloat(Op: Val);
1250
1251 return DAG.getStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(),
1252 MMO: ST->getMemOperand());
1253}
1254
1255SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1256 assert(OpNo == 1 && "Can only soften the stored value!");
1257 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
1258 SDValue Val = ST->getVal();
1259 EVT VT = Val.getValueType();
1260 SDLoc dl(N);
1261
1262 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1263
1264 SDValue NewVal = GetSoftenedFloat(Op: Val);
1265 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: VT, Chain: ST->getChain(), Ptr: NewVal,
1266 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
1267}
1268
1269SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1270 SDValue LHS = N->getOperand(Num: 0);
1271 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1272 SDLoc dl(N);
1273
1274 EVT LVT = LHS.getValueType();
1275 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1276 EVT RVT = RHS.getValueType();
1277
1278 unsigned LSize = LVT.getSizeInBits();
1279 unsigned RSize = RVT.getSizeInBits();
1280
1281 // Shift right or sign-extend it if the two operands have different types.
1282 int SizeDiff = RSize - LSize;
1283 if (SizeDiff > 0) {
1284 RHS =
1285 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1286 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1287 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1288 DL: DAG.getDataLayout())));
1289 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1290 } else if (SizeDiff < 0) {
1291 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1292 RHS =
1293 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1294 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1295 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1296 DL: DAG.getDataLayout())));
1297 }
1298
1299 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1300 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1301}
1302
1303SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1304 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1305 bool IsStrict = N->isStrictFPOpcode();
1306 unsigned Offset = IsStrict ? 1 : 0;
1307 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1308 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1309 TargetLowering::MakeLibCallOptions CallOptions;
1310 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1311 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
1312 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1313 CallOptions, dl: SDLoc(N),
1314 Chain);
1315 if (IsStrict) {
1316 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1317 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1318 return SDValue();
1319 }
1320
1321 return Tmp.first;
1322}
1323
1324SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1325 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1326 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1327 Call_F32: RTLIB::LROUND_F32,
1328 Call_F64: RTLIB::LROUND_F64,
1329 Call_F80: RTLIB::LROUND_F80,
1330 Call_F128: RTLIB::LROUND_F128,
1331 Call_PPCF128: RTLIB::LROUND_PPCF128));
1332}
1333
1334SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1335 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1336 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1337 Call_F32: RTLIB::LLROUND_F32,
1338 Call_F64: RTLIB::LLROUND_F64,
1339 Call_F80: RTLIB::LLROUND_F80,
1340 Call_F128: RTLIB::LLROUND_F128,
1341 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1342}
1343
1344SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1345 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1346 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1347 Call_F32: RTLIB::LRINT_F32,
1348 Call_F64: RTLIB::LRINT_F64,
1349 Call_F80: RTLIB::LRINT_F80,
1350 Call_F128: RTLIB::LRINT_F128,
1351 Call_PPCF128: RTLIB::LRINT_PPCF128));
1352}
1353
1354SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1355 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1356 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1357 Call_F32: RTLIB::LLRINT_F32,
1358 Call_F64: RTLIB::LLRINT_F64,
1359 Call_F80: RTLIB::LLRINT_F80,
1360 Call_F128: RTLIB::LLRINT_F128,
1361 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1362}
1363
1364//===----------------------------------------------------------------------===//
1365// Float Result Expansion
1366//===----------------------------------------------------------------------===//
1367
1368/// ExpandFloatResult - This method is called when the specified result of the
1369/// specified node is found to need expansion. At this point, the node may also
1370/// have invalid operands or may have other results that need promotion, we just
1371/// know that (at least) one result needs expansion.
1372void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1373 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1374 SDValue Lo, Hi;
1375 Lo = Hi = SDValue();
1376
1377 // See if the target wants to custom expand this node.
1378 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1379 return;
1380
1381 switch (N->getOpcode()) {
1382 default:
1383#ifndef NDEBUG
1384 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1385 N->dump(&DAG); dbgs() << "\n";
1386#endif
1387 report_fatal_error(reason: "Do not know how to expand the result of this "
1388 "operator!");
1389 // clang-format off
1390 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1391 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1392 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1393
1394 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1395 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1396 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1397 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1398 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1399 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1400
1401 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1402 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1403 case ISD::STRICT_FMINNUM:
1404 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1405 case ISD::STRICT_FMAXNUM:
1406 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1407 case ISD::STRICT_FADD:
1408 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1409 case ISD::STRICT_FACOS:
1410 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1411 case ISD::STRICT_FASIN:
1412 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1413 case ISD::STRICT_FATAN:
1414 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1415 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1416 case ISD::STRICT_FCEIL:
1417 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1418 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1419 case ISD::STRICT_FCOS:
1420 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1421 case ISD::STRICT_FCOSH:
1422 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1423 case ISD::STRICT_FDIV:
1424 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1425 case ISD::STRICT_FEXP:
1426 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1427 case ISD::STRICT_FEXP2:
1428 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1429 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1430 case ISD::STRICT_FFLOOR:
1431 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1432 case ISD::STRICT_FLOG:
1433 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1434 case ISD::STRICT_FLOG2:
1435 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1436 case ISD::STRICT_FLOG10:
1437 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1438 case ISD::STRICT_FMA:
1439 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1440 case ISD::STRICT_FMUL:
1441 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1442 case ISD::STRICT_FNEARBYINT:
1443 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1444 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1445 case ISD::STRICT_FP_EXTEND:
1446 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1447 case ISD::STRICT_FPOW:
1448 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1449 case ISD::STRICT_FPOWI:
1450 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1451 case ISD::FLDEXP:
1452 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1453 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1454 case ISD::STRICT_FRINT:
1455 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1456 case ISD::STRICT_FROUND:
1457 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1458 case ISD::STRICT_FROUNDEVEN:
1459 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1460 case ISD::STRICT_FSIN:
1461 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1462 case ISD::STRICT_FSINH:
1463 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1464 case ISD::STRICT_FSQRT:
1465 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1466 case ISD::STRICT_FSUB:
1467 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1468 case ISD::STRICT_FTAN:
1469 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1470 case ISD::STRICT_FTANH:
1471 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1472 case ISD::STRICT_FTRUNC:
1473 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1474 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1475 case ISD::STRICT_SINT_TO_FP:
1476 case ISD::STRICT_UINT_TO_FP:
1477 case ISD::SINT_TO_FP:
1478 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1479 case ISD::STRICT_FREM:
1480 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1481 // clang-format on
1482 }
1483
1484 // If Lo/Hi is null, the sub-method took care of registering results etc.
1485 if (Lo.getNode())
1486 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1487}
1488
1489void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1490 SDValue &Hi) {
1491 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1492 assert(NVT.getSizeInBits() == 64 &&
1493 "Do not know how to expand this float constant!");
1494 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1495 SDLoc dl(N);
1496 Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1497 APInt(64, C.getRawData()[1])),
1498 DL: dl, VT: NVT);
1499 Hi = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1500 APInt(64, C.getRawData()[0])),
1501 DL: dl, VT: NVT);
1502}
1503
1504void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1505 SDValue &Lo, SDValue &Hi) {
1506 bool IsStrict = N->isStrictFPOpcode();
1507 unsigned Offset = IsStrict ? 1 : 0;
1508 SDValue Op = N->getOperand(Num: 0 + Offset);
1509 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1510 TargetLowering::MakeLibCallOptions CallOptions;
1511 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1512 Ops: Op, CallOptions, dl: SDLoc(N),
1513 Chain);
1514 if (IsStrict)
1515 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1516 GetPairElements(Pair: Tmp.first, Lo, Hi);
1517}
1518
1519void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1520 SDValue &Lo, SDValue &Hi) {
1521 bool IsStrict = N->isStrictFPOpcode();
1522 unsigned Offset = IsStrict ? 1 : 0;
1523 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1524 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1525 TargetLowering::MakeLibCallOptions CallOptions;
1526 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1527 Ops, CallOptions, dl: SDLoc(N),
1528 Chain);
1529 if (IsStrict)
1530 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1531 GetPairElements(Pair: Tmp.first, Lo, Hi);
1532}
1533
1534void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1535 SDValue &Hi) {
1536 assert(N->getValueType(0) == MVT::ppcf128 &&
1537 "Logic only correct for ppcf128!");
1538 SDLoc dl(N);
1539 SDValue Tmp;
1540 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1541 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1542 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1543 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1544 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1545 Cond: ISD::SETEQ);
1546}
1547
1548void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1549 SDValue &Hi) {
1550 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1551 Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
1552 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
1553 Call_PPCF128: RTLIB::FMIN_PPCF128), Lo, Hi);
1554}
1555
1556void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1557 SDValue &Hi) {
1558 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1559 Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
1560 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
1561 Call_PPCF128: RTLIB::FMAX_PPCF128), Lo, Hi);
1562}
1563
1564void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1565 SDValue &Hi) {
1566 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1567 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1568 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1569 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1570}
1571
1572void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1573 SDValue &Hi) {
1574 ExpandFloatRes_Unary(N,
1575 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ACOS_F32,
1576 Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
1577 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128),
1578 Lo, Hi);
1579}
1580
1581void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1582 SDValue &Hi) {
1583 ExpandFloatRes_Unary(N,
1584 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ASIN_F32,
1585 Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
1586 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128),
1587 Lo, Hi);
1588}
1589
1590void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1591 SDValue &Hi) {
1592 ExpandFloatRes_Unary(N,
1593 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN_F32,
1594 Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
1595 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128),
1596 Lo, Hi);
1597}
1598
1599void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1600 SDValue &Hi) {
1601 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::CBRT_F32,
1602 Call_F64: RTLIB::CBRT_F64, Call_F80: RTLIB::CBRT_F80,
1603 Call_F128: RTLIB::CBRT_F128,
1604 Call_PPCF128: RTLIB::CBRT_PPCF128), Lo, Hi);
1605}
1606
1607void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1608 SDValue &Lo, SDValue &Hi) {
1609 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1610 Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
1611 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
1612 Call_PPCF128: RTLIB::CEIL_PPCF128), Lo, Hi);
1613}
1614
1615void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1616 SDValue &Lo, SDValue &Hi) {
1617 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1618 Call_F32: RTLIB::COPYSIGN_F32,
1619 Call_F64: RTLIB::COPYSIGN_F64,
1620 Call_F80: RTLIB::COPYSIGN_F80,
1621 Call_F128: RTLIB::COPYSIGN_F128,
1622 Call_PPCF128: RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1623}
1624
1625void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1626 SDValue &Lo, SDValue &Hi) {
1627 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1628 Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
1629 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
1630 Call_PPCF128: RTLIB::COS_PPCF128), Lo, Hi);
1631}
1632
1633void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1634 SDValue &Hi) {
1635 ExpandFloatRes_Unary(N,
1636 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::COSH_F32,
1637 Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
1638 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128),
1639 Lo, Hi);
1640}
1641
1642void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1643 SDValue &Hi) {
1644 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1645 Call_F32: RTLIB::DIV_F32,
1646 Call_F64: RTLIB::DIV_F64,
1647 Call_F80: RTLIB::DIV_F80,
1648 Call_F128: RTLIB::DIV_F128,
1649 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1650}
1651
1652void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1653 SDValue &Lo, SDValue &Hi) {
1654 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1655 Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64,
1656 Call_F80: RTLIB::EXP_F80, Call_F128: RTLIB::EXP_F128,
1657 Call_PPCF128: RTLIB::EXP_PPCF128), Lo, Hi);
1658}
1659
1660void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1661 SDValue &Lo, SDValue &Hi) {
1662 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1663 Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64,
1664 Call_F80: RTLIB::EXP2_F80, Call_F128: RTLIB::EXP2_F128,
1665 Call_PPCF128: RTLIB::EXP2_PPCF128), Lo, Hi);
1666}
1667
1668void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1669 SDValue &Hi) {
1670 ExpandFloatRes_Unary(N,
1671 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32,
1672 Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
1673 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128),
1674 Lo, Hi);
1675}
1676
1677void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1678 SDValue &Lo, SDValue &Hi) {
1679 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1680 Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
1681 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
1682 Call_PPCF128: RTLIB::FLOOR_PPCF128), Lo, Hi);
1683}
1684
1685void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1686 SDValue &Lo, SDValue &Hi) {
1687 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1688 Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64,
1689 Call_F80: RTLIB::LOG_F80, Call_F128: RTLIB::LOG_F128,
1690 Call_PPCF128: RTLIB::LOG_PPCF128), Lo, Hi);
1691}
1692
1693void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1694 SDValue &Lo, SDValue &Hi) {
1695 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1696 Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64,
1697 Call_F80: RTLIB::LOG2_F80, Call_F128: RTLIB::LOG2_F128,
1698 Call_PPCF128: RTLIB::LOG2_PPCF128), Lo, Hi);
1699}
1700
1701void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1702 SDValue &Lo, SDValue &Hi) {
1703 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1704 Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64,
1705 Call_F80: RTLIB::LOG10_F80, Call_F128: RTLIB::LOG10_F128,
1706 Call_PPCF128: RTLIB::LOG10_PPCF128), Lo, Hi);
1707}
1708
1709void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1710 SDValue &Hi) {
1711 bool IsStrict = N->isStrictFPOpcode();
1712 unsigned Offset = IsStrict ? 1 : 0;
1713 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1714 N->getOperand(Num: 2 + Offset) };
1715 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1716 TargetLowering::MakeLibCallOptions CallOptions;
1717 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1718 Call_F32: RTLIB::FMA_F32,
1719 Call_F64: RTLIB::FMA_F64,
1720 Call_F80: RTLIB::FMA_F80,
1721 Call_F128: RTLIB::FMA_F128,
1722 Call_PPCF128: RTLIB::FMA_PPCF128),
1723 RetVT: N->getValueType(ResNo: 0), Ops, CallOptions,
1724 dl: SDLoc(N), Chain);
1725 if (IsStrict)
1726 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1727 GetPairElements(Pair: Tmp.first, Lo, Hi);
1728}
1729
1730void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1731 SDValue &Hi) {
1732 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1733 Call_F32: RTLIB::MUL_F32,
1734 Call_F64: RTLIB::MUL_F64,
1735 Call_F80: RTLIB::MUL_F80,
1736 Call_F128: RTLIB::MUL_F128,
1737 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1738}
1739
1740void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1741 SDValue &Lo, SDValue &Hi) {
1742 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1743 Call_F32: RTLIB::NEARBYINT_F32,
1744 Call_F64: RTLIB::NEARBYINT_F64,
1745 Call_F80: RTLIB::NEARBYINT_F80,
1746 Call_F128: RTLIB::NEARBYINT_F128,
1747 Call_PPCF128: RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1748}
1749
1750void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1751 SDValue &Hi) {
1752 SDLoc dl(N);
1753 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1754 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1755 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1756}
1757
1758void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1759 SDValue &Hi) {
1760 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1761 SDLoc dl(N);
1762 bool IsStrict = N->isStrictFPOpcode();
1763
1764 SDValue Chain;
1765 if (IsStrict) {
1766 // If the expanded type is the same as the input type, just bypass the node.
1767 if (NVT == N->getOperand(Num: 1).getValueType()) {
1768 Hi = N->getOperand(Num: 1);
1769 Chain = N->getOperand(Num: 0);
1770 } else {
1771 // Other we need to extend.
1772 Hi = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: { NVT, MVT::Other },
1773 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
1774 Chain = Hi.getValue(R: 1);
1775 }
1776 } else {
1777 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1778 }
1779
1780 Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1781 APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1782
1783 if (IsStrict)
1784 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1785}
1786
1787void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1788 SDValue &Lo, SDValue &Hi) {
1789 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1790 Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64,
1791 Call_F80: RTLIB::POW_F80, Call_F128: RTLIB::POW_F128,
1792 Call_PPCF128: RTLIB::POW_PPCF128), Lo, Hi);
1793}
1794
1795void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1796 SDValue &Lo, SDValue &Hi) {
1797 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
1798}
1799
1800void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1801 SDValue &Hi) {
1802 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
1803}
1804
1805void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1806 SDValue &Lo, SDValue &Hi) {
1807 assert(N->getValueType(0) == MVT::ppcf128 &&
1808 "Logic only correct for ppcf128!");
1809
1810 SDLoc dl(N);
1811 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1812 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1813 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1814}
1815
1816void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1817 SDValue &Lo, SDValue &Hi) {
1818 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1819 Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
1820 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
1821 Call_PPCF128: RTLIB::REM_PPCF128), Lo, Hi);
1822}
1823
1824void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1825 SDValue &Lo, SDValue &Hi) {
1826 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1827 Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
1828 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
1829 Call_PPCF128: RTLIB::RINT_PPCF128), Lo, Hi);
1830}
1831
1832void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1833 SDValue &Lo, SDValue &Hi) {
1834 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1835 Call_F32: RTLIB::ROUND_F32,
1836 Call_F64: RTLIB::ROUND_F64,
1837 Call_F80: RTLIB::ROUND_F80,
1838 Call_F128: RTLIB::ROUND_F128,
1839 Call_PPCF128: RTLIB::ROUND_PPCF128), Lo, Hi);
1840}
1841
1842void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1843 SDValue &Lo, SDValue &Hi) {
1844 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1845 Call_F32: RTLIB::ROUNDEVEN_F32,
1846 Call_F64: RTLIB::ROUNDEVEN_F64,
1847 Call_F80: RTLIB::ROUNDEVEN_F80,
1848 Call_F128: RTLIB::ROUNDEVEN_F128,
1849 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1850}
1851
1852void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1853 SDValue &Lo, SDValue &Hi) {
1854 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1855 Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
1856 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
1857 Call_PPCF128: RTLIB::SIN_PPCF128), Lo, Hi);
1858}
1859
1860void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
1861 SDValue &Hi) {
1862 ExpandFloatRes_Unary(N,
1863 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::SINH_F32,
1864 Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
1865 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128),
1866 Lo, Hi);
1867}
1868
1869void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1870 SDValue &Lo, SDValue &Hi) {
1871 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1872 Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
1873 Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
1874 Call_PPCF128: RTLIB::SQRT_PPCF128), Lo, Hi);
1875}
1876
1877void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1878 SDValue &Hi) {
1879 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1880 Call_F32: RTLIB::SUB_F32,
1881 Call_F64: RTLIB::SUB_F64,
1882 Call_F80: RTLIB::SUB_F80,
1883 Call_F128: RTLIB::SUB_F128,
1884 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
1885}
1886
1887void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
1888 SDValue &Hi) {
1889 ExpandFloatRes_Unary(N,
1890 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TAN_F32,
1891 Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
1892 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128),
1893 Lo, Hi);
1894}
1895
1896void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
1897 SDValue &Hi) {
1898 ExpandFloatRes_Unary(N,
1899 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TANH_F32,
1900 Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
1901 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128),
1902 Lo, Hi);
1903}
1904
1905void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1906 SDValue &Lo, SDValue &Hi) {
1907 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1908 Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
1909 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
1910 Call_PPCF128: RTLIB::TRUNC_PPCF128), Lo, Hi);
1911}
1912
1913void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1914 SDValue &Hi) {
1915 if (ISD::isNormalLoad(N)) {
1916 ExpandRes_NormalLoad(N, Lo, Hi);
1917 return;
1918 }
1919
1920 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1921 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1922 SDValue Chain = LD->getChain();
1923 SDValue Ptr = LD->getBasePtr();
1924 SDLoc dl(N);
1925
1926 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
1927 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1928 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1929
1930 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
1931 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
1932
1933 // Remember the chain.
1934 Chain = Hi.getValue(R: 1);
1935
1936 // The low part is zero.
1937 Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1938 APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1939
1940 // Modified the chain - switch anything that used the old chain to use the
1941 // new one.
1942 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
1943}
1944
1945void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1946 SDValue &Hi) {
1947 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1948 EVT VT = N->getValueType(ResNo: 0);
1949 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1950 bool Strict = N->isStrictFPOpcode();
1951 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
1952 EVT SrcVT = Src.getValueType();
1953 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1954 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1955 SDLoc dl(N);
1956 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
1957
1958 // TODO: Any other flags to propagate?
1959 SDNodeFlags Flags;
1960 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1961
1962 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1963 // When promoting partial word types to i32 we must honor the signedness,
1964 // though.
1965 if (SrcVT.bitsLE(VT: MVT::i32)) {
1966 // The integer can be represented exactly in an f64.
1967 Lo = DAG.getConstantFP(Val: APFloat(DAG.EVTToAPFloatSemantics(VT: NVT),
1968 APInt(NVT.getSizeInBits(), 0)), DL: dl, VT: NVT);
1969 if (Strict) {
1970 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
1971 Ops: {Chain, Src}, Flags);
1972 Chain = Hi.getValue(R: 1);
1973 } else
1974 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
1975 } else {
1976 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1977 if (SrcVT.bitsLE(VT: MVT::i64)) {
1978 Src = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
1979 VT: MVT::i64, Operand: Src);
1980 LC = RTLIB::SINTTOFP_I64_PPCF128;
1981 } else if (SrcVT.bitsLE(VT: MVT::i128)) {
1982 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i128, Operand: Src);
1983 LC = RTLIB::SINTTOFP_I128_PPCF128;
1984 }
1985 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1986
1987 TargetLowering::MakeLibCallOptions CallOptions;
1988 CallOptions.setSExt(true);
1989 std::pair<SDValue, SDValue> Tmp =
1990 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
1991 if (Strict)
1992 Chain = Tmp.second;
1993 GetPairElements(Pair: Tmp.first, Lo, Hi);
1994 }
1995
1996 // No need to complement for unsigned 32-bit integers
1997 if (isSigned || SrcVT.bitsLE(VT: MVT::i32)) {
1998 if (Strict)
1999 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2000
2001 return;
2002 }
2003
2004 // Unsigned - fix up the SINT_TO_FP value just calculated.
2005 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2006 // keep semantics correctness if the integer is not exactly representable
2007 // here. See ExpandLegalINT_TO_FP.
2008 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
2009 SrcVT = Src.getValueType();
2010
2011 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2012 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2013 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2014 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2015 ArrayRef<uint64_t> Parts;
2016
2017 switch (SrcVT.getSimpleVT().SimpleTy) {
2018 default:
2019 llvm_unreachable("Unsupported UINT_TO_FP!");
2020 case MVT::i32:
2021 Parts = TwoE32;
2022 break;
2023 case MVT::i64:
2024 Parts = TwoE64;
2025 break;
2026 case MVT::i128:
2027 Parts = TwoE128;
2028 break;
2029 }
2030
2031 // TODO: Are there other fast-math-flags to propagate to this FADD?
2032 SDValue NewLo = DAG.getConstantFP(
2033 Val: APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), DL: dl, VT: MVT::ppcf128);
2034 if (Strict) {
2035 Lo = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other),
2036 Ops: {Chain, Hi, NewLo}, Flags);
2037 Chain = Lo.getValue(R: 1);
2038 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2039 } else
2040 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
2041 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
2042 True: Lo, False: Hi, Cond: ISD::SETLT);
2043 GetPairElements(Pair: Lo, Lo, Hi);
2044}
2045
2046
2047//===----------------------------------------------------------------------===//
2048// Float Operand Expansion
2049//===----------------------------------------------------------------------===//
2050
2051/// ExpandFloatOperand - This method is called when the specified operand of the
2052/// specified node is found to need expansion. At this point, all of the result
2053/// types of the node are known to be legal, but other operands of the node may
2054/// need promotion or expansion as well as the specified one.
2055bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2056 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2057 SDValue Res = SDValue();
2058
2059 // See if the target wants to custom expand this node.
2060 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2061 return false;
2062
2063 switch (N->getOpcode()) {
2064 default:
2065#ifndef NDEBUG
2066 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2067 N->dump(&DAG); dbgs() << "\n";
2068#endif
2069 report_fatal_error(reason: "Do not know how to expand this operator's operand!");
2070
2071 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2072 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2073 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2074
2075 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2076 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2077 case ISD::STRICT_FP_ROUND:
2078 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2079 case ISD::STRICT_FP_TO_SINT:
2080 case ISD::STRICT_FP_TO_UINT:
2081 case ISD::FP_TO_SINT:
2082 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2083 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2084 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2085 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2086 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2087 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2088 case ISD::STRICT_FSETCC:
2089 case ISD::STRICT_FSETCCS:
2090 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2091 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
2092 OpNo); break;
2093 }
2094
2095 // If the result is null, the sub-method took care of registering results etc.
2096 if (!Res.getNode()) return false;
2097
2098 // If the result is N, the sub-method updated N in place. Tell the legalizer
2099 // core about this.
2100 if (Res.getNode() == N)
2101 return true;
2102
2103 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2104 "Invalid operand expansion");
2105
2106 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2107 return false;
2108}
2109
2110/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2111/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2112void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2113 SDValue &NewRHS,
2114 ISD::CondCode &CCCode,
2115 const SDLoc &dl, SDValue &Chain,
2116 bool IsSignaling) {
2117 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2118 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
2119 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
2120
2121 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2122
2123 // FIXME: This generated code sucks. We want to generate
2124 // FCMPU crN, hi1, hi2
2125 // BNE crN, L:
2126 // FCMPU crN, lo1, lo2
2127 // The following can be improved, but not that much.
2128 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2129 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2130 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
2131 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2132 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
2133 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
2134 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2135 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2136 Tmp1 =
2137 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
2138 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
2139 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2140 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2141 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
2142 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2143 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2144 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
2145 NewRHS = SDValue(); // LHS is the result, not a compare.
2146 Chain = OutputChain;
2147}
2148
2149SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2150 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
2151 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
2152 SDValue Chain;
2153 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2154
2155 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2156 // against zero to select between true and false values.
2157 if (!NewRHS.getNode()) {
2158 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2159 CCCode = ISD::SETNE;
2160 }
2161
2162 // Update N to have the operands specified.
2163 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2164 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
2165 Op5: N->getOperand(Num: 4)), 0);
2166}
2167
2168SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2169 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2170 "Logic only correct for ppcf128!");
2171 SDValue Lo, Hi;
2172 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2173 // The ppcf128 value is providing only the sign; take it from the
2174 // higher-order double (which must have the larger magnitude).
2175 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2176 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2177}
2178
2179SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2180 bool IsStrict = N->isStrictFPOpcode();
2181 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2182 "Logic only correct for ppcf128!");
2183 SDValue Lo, Hi;
2184 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2185
2186 if (!IsStrict)
2187 // Round it the rest of the way (e.g. to f32) if needed.
2188 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2189 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2190
2191 // Eliminate the node if the input float type is the same as the output float
2192 // type.
2193 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2194 // Connect the output chain to the input chain, unlinking the node.
2195 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2196 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2197 return SDValue();
2198 }
2199
2200 SDValue Expansion = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
2201 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
2202 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
2203 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2204 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2205 return SDValue();
2206}
2207
2208SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2209 EVT RVT = N->getValueType(ResNo: 0);
2210 SDLoc dl(N);
2211
2212 bool IsStrict = N->isStrictFPOpcode();
2213 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2214 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2215 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2216 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2217
2218 EVT NVT;
2219 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2220 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2221 "Unsupported FP_TO_XINT!");
2222 TargetLowering::MakeLibCallOptions CallOptions;
2223 std::pair<SDValue, SDValue> Tmp =
2224 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2225 if (!IsStrict)
2226 return Tmp.first;
2227
2228 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2229 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2230 return SDValue();
2231}
2232
2233SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2234 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2235 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2236 SDValue Chain;
2237 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2238
2239 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2240 // against zero to select between true and false values.
2241 if (!NewRHS.getNode()) {
2242 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2243 CCCode = ISD::SETNE;
2244 }
2245
2246 // Update N to have the operands specified.
2247 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2248 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2249 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2250}
2251
2252SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2253 bool IsStrict = N->isStrictFPOpcode();
2254 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2255 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2256 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2257 ISD::CondCode CCCode =
2258 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2259 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2260 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2261
2262 // FloatExpandSetCCOperands always returned a scalar.
2263 assert(!NewRHS.getNode() && "Expect to return scalar");
2264 assert(NewLHS.getValueType() == N->getValueType(0) &&
2265 "Unexpected setcc expansion!");
2266 if (Chain) {
2267 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2268 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2269 return SDValue();
2270 }
2271 return NewLHS;
2272}
2273
2274SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2275 if (ISD::isNormalStore(N))
2276 return ExpandOp_NormalStore(N, OpNo);
2277
2278 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2279 assert(OpNo == 1 && "Can only expand the stored value so far");
2280 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2281
2282 SDValue Chain = ST->getChain();
2283 SDValue Ptr = ST->getBasePtr();
2284
2285 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2286 VT: ST->getValue().getValueType());
2287 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2288 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2289 (void)NVT;
2290
2291 SDValue Lo, Hi;
2292 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2293
2294 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2295 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2296}
2297
2298SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2299 EVT RVT = N->getValueType(ResNo: 0);
2300 EVT RetVT = N->getOperand(Num: 0).getValueType();
2301 TargetLowering::MakeLibCallOptions CallOptions;
2302 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2303 Call_F32: RTLIB::LROUND_F32,
2304 Call_F64: RTLIB::LROUND_F64,
2305 Call_F80: RTLIB::LROUND_F80,
2306 Call_F128: RTLIB::LROUND_F128,
2307 Call_PPCF128: RTLIB::LROUND_PPCF128),
2308 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2309}
2310
2311SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2312 EVT RVT = N->getValueType(ResNo: 0);
2313 EVT RetVT = N->getOperand(Num: 0).getValueType();
2314 TargetLowering::MakeLibCallOptions CallOptions;
2315 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2316 Call_F32: RTLIB::LLROUND_F32,
2317 Call_F64: RTLIB::LLROUND_F64,
2318 Call_F80: RTLIB::LLROUND_F80,
2319 Call_F128: RTLIB::LLROUND_F128,
2320 Call_PPCF128: RTLIB::LLROUND_PPCF128),
2321 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2322}
2323
2324SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2325 EVT RVT = N->getValueType(ResNo: 0);
2326 EVT RetVT = N->getOperand(Num: 0).getValueType();
2327 TargetLowering::MakeLibCallOptions CallOptions;
2328 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2329 Call_F32: RTLIB::LRINT_F32,
2330 Call_F64: RTLIB::LRINT_F64,
2331 Call_F80: RTLIB::LRINT_F80,
2332 Call_F128: RTLIB::LRINT_F128,
2333 Call_PPCF128: RTLIB::LRINT_PPCF128),
2334 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2335}
2336
2337SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2338 EVT RVT = N->getValueType(ResNo: 0);
2339 EVT RetVT = N->getOperand(Num: 0).getValueType();
2340 TargetLowering::MakeLibCallOptions CallOptions;
2341 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2342 Call_F32: RTLIB::LLRINT_F32,
2343 Call_F64: RTLIB::LLRINT_F64,
2344 Call_F80: RTLIB::LLRINT_F80,
2345 Call_F128: RTLIB::LLRINT_F128,
2346 Call_PPCF128: RTLIB::LLRINT_PPCF128),
2347 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2348}
2349
2350//===----------------------------------------------------------------------===//
2351// Float Operand Promotion
2352//===----------------------------------------------------------------------===//
2353//
2354
2355static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2356 if (OpVT == MVT::f16) {
2357 return ISD::FP16_TO_FP;
2358 } else if (RetVT == MVT::f16) {
2359 return ISD::FP_TO_FP16;
2360 } else if (OpVT == MVT::bf16) {
2361 return ISD::BF16_TO_FP;
2362 } else if (RetVT == MVT::bf16) {
2363 return ISD::FP_TO_BF16;
2364 }
2365
2366 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2367}
2368
2369static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2370 if (OpVT == MVT::f16)
2371 return ISD::STRICT_FP16_TO_FP;
2372
2373 if (RetVT == MVT::f16)
2374 return ISD::STRICT_FP_TO_FP16;
2375
2376 if (OpVT == MVT::bf16)
2377 return ISD::STRICT_BF16_TO_FP;
2378
2379 if (RetVT == MVT::bf16)
2380 return ISD::STRICT_FP_TO_BF16;
2381
2382 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2383}
2384
2385bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2386 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2387 SDValue R = SDValue();
2388
2389 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
2390 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2391 return false;
2392 }
2393
2394 // Nodes that use a promotion-requiring floating point operand, but doesn't
2395 // produce a promotion-requiring floating point result, need to be legalized
2396 // to use the promoted float operand. Nodes that produce at least one
2397 // promotion-requiring floating point result have their operands legalized as
2398 // a part of PromoteFloatResult.
2399 // clang-format off
2400 switch (N->getOpcode()) {
2401 default:
2402 #ifndef NDEBUG
2403 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2404 N->dump(&DAG); dbgs() << "\n";
2405 #endif
2406 report_fatal_error(reason: "Do not know how to promote this operator's operand!");
2407
2408 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2409 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2410 case ISD::FP_TO_SINT:
2411 case ISD::FP_TO_UINT:
2412 case ISD::LRINT:
2413 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2414 case ISD::FP_TO_SINT_SAT:
2415 case ISD::FP_TO_UINT_SAT:
2416 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2417 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2418 case ISD::STRICT_FP_EXTEND:
2419 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2420 break;
2421 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2422 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2423 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2424 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2425 }
2426 // clang-format on
2427
2428 if (R.getNode())
2429 ReplaceValueWith(From: SDValue(N, 0), To: R);
2430 return false;
2431}
2432
2433SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2434 SDValue Op = N->getOperand(Num: 0);
2435 EVT OpVT = Op->getValueType(ResNo: 0);
2436
2437 SDValue Promoted = GetPromotedFloat(Op: N->getOperand(Num: 0));
2438 EVT PromotedVT = Promoted->getValueType(ResNo: 0);
2439
2440 // Convert the promoted float value to the desired IVT.
2441 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OpVT.getSizeInBits());
2442 SDValue Convert = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: PromotedVT, RetVT: OpVT), DL: SDLoc(N),
2443 VT: IVT, Operand: Promoted);
2444 // The final result type might not be an scalar so we need a bitcast. The
2445 // bitcast will be further legalized if needed.
2446 return DAG.getBitcast(VT: N->getValueType(ResNo: 0), V: Convert);
2447}
2448
2449// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2450// PromoteFloatRes_FCOPYSIGN.
2451SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2452 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2453 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2454
2455 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2456 N1: N->getOperand(Num: 0), N2: Op1);
2457}
2458
2459// Convert the promoted float value to the desired integer type
2460SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2461 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2462 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
2463}
2464
2465SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2466 unsigned OpNo) {
2467 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2468 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op,
2469 N2: N->getOperand(Num: 1));
2470}
2471
2472SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2473 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2474 EVT VT = N->getValueType(ResNo: 0);
2475
2476 // Desired VT is same as promoted type. Use promoted float directly.
2477 if (VT == Op->getValueType(ResNo: 0))
2478 return Op;
2479
2480 // Else, extend the promoted float value to the desired VT.
2481 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Op);
2482}
2483
2484SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2485 unsigned OpNo) {
2486 assert(OpNo == 1 && "Promoting unpromotable operand");
2487
2488 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 1));
2489 EVT VT = N->getValueType(ResNo: 0);
2490
2491 // Desired VT is same as promoted type. Use promoted float directly.
2492 if (VT == Op->getValueType(ResNo: 0)) {
2493 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2494 return Op;
2495 }
2496
2497 // Else, extend the promoted float value to the desired VT.
2498 SDValue Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N), VTList: N->getVTList(),
2499 N1: N->getOperand(Num: 0), N2: Op);
2500 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2501 return Res;
2502}
2503
2504// Promote the float operands used for comparison. The true- and false-
2505// operands have the same type as the result and are promoted, if needed, by
2506// PromoteFloatRes_SELECT_CC
2507SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2508 SDValue LHS = GetPromotedFloat(Op: N->getOperand(Num: 0));
2509 SDValue RHS = GetPromotedFloat(Op: N->getOperand(Num: 1));
2510
2511 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2512 N1: LHS, N2: RHS, N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3),
2513 N5: N->getOperand(Num: 4));
2514}
2515
2516// Construct a SETCC that compares the promoted values and sets the conditional
2517// code.
2518SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2519 EVT VT = N->getValueType(ResNo: 0);
2520 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2521 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2522 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
2523
2524 return DAG.getSetCC(DL: SDLoc(N), VT, LHS: Op0, RHS: Op1, Cond: CCCode);
2525
2526}
2527
2528// Lower the promoted Float down to the integer value of same size and construct
2529// a STORE of the integer value.
2530SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2531 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2532 SDValue Val = ST->getValue();
2533 SDLoc DL(N);
2534
2535 SDValue Promoted = GetPromotedFloat(Op: Val);
2536 EVT VT = ST->getOperand(Num: 1).getValueType();
2537 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2538
2539 SDValue NewVal;
2540 NewVal = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: Promoted.getValueType(), RetVT: VT), DL,
2541 VT: IVT, Operand: Promoted);
2542
2543 return DAG.getStore(Chain: ST->getChain(), dl: DL, Val: NewVal, Ptr: ST->getBasePtr(),
2544 MMO: ST->getMemOperand());
2545}
2546
2547SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2548 unsigned OpNo) {
2549 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
2550 SDValue Val = ST->getVal();
2551 SDLoc DL(N);
2552
2553 SDValue Promoted = GetPromotedFloat(Op: Val);
2554 EVT VT = ST->getOperand(Num: 1).getValueType();
2555 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2556
2557 SDValue NewVal = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: Promoted.getValueType(), RetVT: VT),
2558 DL, VT: IVT, Operand: Promoted);
2559
2560 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: IVT, Chain: ST->getChain(), Ptr: NewVal,
2561 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
2562}
2563
2564//===----------------------------------------------------------------------===//
2565// Float Result Promotion
2566//===----------------------------------------------------------------------===//
2567
2568void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2569 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2570 SDValue R = SDValue();
2571
2572 // See if the target wants to custom expand this node.
2573 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2574 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2575 return;
2576 }
2577
2578 switch (N->getOpcode()) {
2579 // These opcodes cannot appear if promotion of FP16 is done in the backend
2580 // instead of Clang
2581 case ISD::FP16_TO_FP:
2582 case ISD::FP_TO_FP16:
2583 default:
2584#ifndef NDEBUG
2585 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2586 N->dump(&DAG); dbgs() << "\n";
2587#endif
2588 report_fatal_error(reason: "Do not know how to promote this operator's result!");
2589
2590 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2591 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2592 case ISD::EXTRACT_VECTOR_ELT:
2593 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2594 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2595
2596 // Unary FP Operations
2597 case ISD::FABS:
2598 case ISD::FACOS:
2599 case ISD::FASIN:
2600 case ISD::FATAN:
2601 case ISD::FCBRT:
2602 case ISD::FCEIL:
2603 case ISD::FCOS:
2604 case ISD::FCOSH:
2605 case ISD::FEXP:
2606 case ISD::FEXP2:
2607 case ISD::FEXP10:
2608 case ISD::FFLOOR:
2609 case ISD::FLOG:
2610 case ISD::FLOG2:
2611 case ISD::FLOG10:
2612 case ISD::FNEARBYINT:
2613 case ISD::FNEG:
2614 case ISD::FRINT:
2615 case ISD::FROUND:
2616 case ISD::FROUNDEVEN:
2617 case ISD::FSIN:
2618 case ISD::FSINH:
2619 case ISD::FSQRT:
2620 case ISD::FTRUNC:
2621 case ISD::FTAN:
2622 case ISD::FTANH:
2623 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2624
2625 // Binary FP Operations
2626 case ISD::FADD:
2627 case ISD::FDIV:
2628 case ISD::FMAXIMUM:
2629 case ISD::FMINIMUM:
2630 case ISD::FMAXNUM:
2631 case ISD::FMINNUM:
2632 case ISD::FMAXNUM_IEEE:
2633 case ISD::FMINNUM_IEEE:
2634 case ISD::FMUL:
2635 case ISD::FPOW:
2636 case ISD::FREM:
2637 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2638
2639 case ISD::FMA: // FMA is same as FMAD
2640 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2641
2642 case ISD::FPOWI:
2643 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2644 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2645
2646 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2647 case ISD::STRICT_FP_ROUND:
2648 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2649 break;
2650 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2651 case ISD::ATOMIC_LOAD:
2652 R = PromoteFloatRes_ATOMIC_LOAD(N);
2653 break;
2654 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2655 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2656
2657 case ISD::SINT_TO_FP:
2658 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2659 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2660 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2661 case ISD::VECREDUCE_FADD:
2662 case ISD::VECREDUCE_FMUL:
2663 case ISD::VECREDUCE_FMIN:
2664 case ISD::VECREDUCE_FMAX:
2665 case ISD::VECREDUCE_FMAXIMUM:
2666 case ISD::VECREDUCE_FMINIMUM:
2667 R = PromoteFloatRes_VECREDUCE(N);
2668 break;
2669 case ISD::VECREDUCE_SEQ_FADD:
2670 case ISD::VECREDUCE_SEQ_FMUL:
2671 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2672 break;
2673 }
2674
2675 if (R.getNode())
2676 SetPromotedFloat(Op: SDValue(N, ResNo), Result: R);
2677}
2678
2679// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2680// At this point, it is not possible to determine if the bitcast value is
2681// eventually stored to memory or promoted to f32 or promoted to a floating
2682// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2683// STORE promotion handlers.
2684SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2685 EVT VT = N->getValueType(ResNo: 0);
2686 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2687 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2688 // bitcast will be legalized further if necessary.
2689 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
2690 BitWidth: N->getOperand(Num: 0).getValueType().getSizeInBits());
2691 SDValue Cast = DAG.getBitcast(VT: IVT, V: N->getOperand(Num: 0));
2692 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: Cast);
2693}
2694
2695SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2696 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(Val: N);
2697 EVT VT = N->getValueType(ResNo: 0);
2698 SDLoc DL(N);
2699
2700 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2701 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2702 SDValue C = DAG.getConstant(Val: CFPNode->getValueAPF().bitcastToAPInt(), DL,
2703 VT: IVT);
2704
2705 // Convert the Constant to the desired FP type
2706 // FIXME We might be able to do the conversion during compilation and get rid
2707 // of it from the object code
2708 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2709 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: C);
2710}
2711
2712// If the Index operand is a constant, try to redirect the extract operation to
2713// the correct legalized vector. If not, bit-convert the input vector to
2714// equivalent integer vector. Extract the element as an (bit-cast) integer
2715// value and convert it to the promoted type.
2716SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2717 SDLoc DL(N);
2718
2719 // If the index is constant, try to extract the value from the legalized
2720 // vector type.
2721 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
2722 SDValue Vec = N->getOperand(Num: 0);
2723 SDValue Idx = N->getOperand(Num: 1);
2724 EVT VecVT = Vec->getValueType(ResNo: 0);
2725 EVT EltVT = VecVT.getVectorElementType();
2726
2727 uint64_t IdxVal = Idx->getAsZExtVal();
2728
2729 switch (getTypeAction(VT: VecVT)) {
2730 default: break;
2731 case TargetLowering::TypeScalarizeVector: {
2732 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
2733 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2734 return SDValue();
2735 }
2736 case TargetLowering::TypeWidenVector: {
2737 Vec = GetWidenedVector(Op: Vec);
2738 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Vec, N2: Idx);
2739 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2740 return SDValue();
2741 }
2742 case TargetLowering::TypeSplitVector: {
2743 SDValue Lo, Hi;
2744 GetSplitVector(Op: Vec, Lo, Hi);
2745
2746 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2747 SDValue Res;
2748 if (IdxVal < LoElts)
2749 Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Lo, N2: Idx);
2750 else
2751 Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Hi,
2752 N2: DAG.getConstant(Val: IdxVal - LoElts, DL,
2753 VT: Idx.getValueType()));
2754 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2755 return SDValue();
2756 }
2757
2758 }
2759 }
2760
2761 // Bit-convert the input vector to the equivalent integer vector
2762 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2763 EVT IVT = NewOp.getValueType().getVectorElementType();
2764
2765 // Extract the element as an (bit-cast) integer value
2766 SDValue NewVal = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IVT,
2767 N1: NewOp, N2: N->getOperand(Num: 1));
2768
2769 // Convert the element to the desired FP type
2770 EVT VT = N->getValueType(ResNo: 0);
2771 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2772 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: NewVal);
2773}
2774
2775// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2776// needs promotion, so does the argument X. Note that Y, if needed, will be
2777// handled during operand promotion.
2778SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2779 EVT VT = N->getValueType(ResNo: 0);
2780 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2781 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2782
2783 SDValue Op1 = N->getOperand(Num: 1);
2784
2785 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1);
2786}
2787
2788// Unary operation where the result and the operand have PromoteFloat type
2789// action. Construct a new SDNode with the promoted float value of the old
2790// operand.
2791SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2792 EVT VT = N->getValueType(ResNo: 0);
2793 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2794 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2795
2796 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, Operand: Op);
2797}
2798
2799// Binary operations where the result and both operands have PromoteFloat type
2800// action. Construct a new SDNode with the promoted float values of the old
2801// operands.
2802SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2803 EVT VT = N->getValueType(ResNo: 0);
2804 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2805 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2806 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2807 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, Flags: N->getFlags());
2808}
2809
2810SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2811 EVT VT = N->getValueType(ResNo: 0);
2812 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2813 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2814 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2815 SDValue Op2 = GetPromotedFloat(Op: N->getOperand(Num: 2));
2816
2817 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, N3: Op2);
2818}
2819
2820// Promote the Float (first) operand and retain the Integer (second) operand
2821SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2822 EVT VT = N->getValueType(ResNo: 0);
2823 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2824 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2825 SDValue Op1 = N->getOperand(Num: 1);
2826
2827 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1);
2828}
2829
2830SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2831 EVT VT = N->getValueType(ResNo: 0);
2832 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2833 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2834 SDValue Res =
2835 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {NVT, N->getValueType(ResNo: 1)}, Ops: Op);
2836
2837 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2838 return Res;
2839}
2840
2841// Explicit operation to reduce precision. Reduce the value to half precision
2842// and promote it back to the legal type.
2843SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2844 SDLoc DL(N);
2845
2846 SDValue Op = N->getOperand(Num: 0);
2847 EVT VT = N->getValueType(ResNo: 0);
2848 EVT OpVT = Op->getValueType(ResNo: 0);
2849 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
2850 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2851
2852 // Round promoted float to desired precision
2853 SDValue Round = DAG.getNode(Opcode: GetPromotionOpcode(OpVT, RetVT: VT), DL, VT: IVT, Operand: Op);
2854 // Promote it back to the legal output type
2855 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: Round);
2856}
2857
2858// Explicit operation to reduce precision. Reduce the value to half precision
2859// and promote it back to the legal type.
2860SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2861 SDLoc DL(N);
2862
2863 SDValue Chain = N->getOperand(Num: 0);
2864 SDValue Op = N->getOperand(Num: 1);
2865 EVT VT = N->getValueType(ResNo: 0);
2866 EVT OpVT = Op->getValueType(ResNo: 0);
2867 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
2868 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2869
2870 // Round promoted float to desired precision
2871 SDValue Round = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT, RetVT: VT), DL,
2872 VTList: DAG.getVTList(VT1: IVT, VT2: MVT::Other), N1: Chain, N2: Op);
2873 // Promote it back to the legal output type
2874 SDValue Res =
2875 DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: VT, RetVT: NVT), DL,
2876 VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other), N1: Round.getValue(R: 1), N2: Round);
2877 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2878 return Res;
2879}
2880
2881SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2882 LoadSDNode *L = cast<LoadSDNode>(Val: N);
2883 EVT VT = N->getValueType(ResNo: 0);
2884
2885 // Load the value as an integer value with the same number of bits.
2886 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2887 SDValue newL = DAG.getLoad(
2888 AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: IVT, dl: SDLoc(N),
2889 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(), PtrInfo: L->getPointerInfo(), MemVT: IVT,
2890 Alignment: L->getOriginalAlign(), MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo());
2891 // Legalize the chain result by replacing uses of the old value chain with the
2892 // new one
2893 ReplaceValueWith(From: SDValue(N, 1), To: newL.getValue(R: 1));
2894
2895 // Convert the integer value to the desired FP type
2896 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2897 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: newL);
2898}
2899
2900SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
2901 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2902 EVT VT = AM->getValueType(ResNo: 0);
2903
2904 // Load the value as an integer value with the same number of bits.
2905 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2906 SDValue newL = DAG.getAtomic(
2907 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: IVT, VTList: DAG.getVTList(VT1: IVT, VT2: MVT::Other),
2908 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
2909
2910 // Legalize the chain result by replacing uses of the old value chain with the
2911 // new one
2912 ReplaceValueWith(From: SDValue(N, 1), To: newL.getValue(R: 1));
2913
2914 // Convert the integer value to the desired FP type
2915 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2916 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: IVT), DL: SDLoc(N), VT: NVT, Operand: newL);
2917}
2918
2919// Construct a new SELECT node with the promoted true- and false- values.
2920SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2921 SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 1));
2922 SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 2));
2923
2924 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: TrueVal->getValueType(ResNo: 0),
2925 N1: N->getOperand(Num: 0), N2: TrueVal, N3: FalseVal);
2926}
2927
2928// Construct a new SELECT_CC node with the promoted true- and false- values.
2929// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2930SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2931 SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 2));
2932 SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 3));
2933
2934 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
2935 VT: TrueVal.getNode()->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
2936 N2: N->getOperand(Num: 1), N3: TrueVal, N4: FalseVal, N5: N->getOperand(Num: 4));
2937}
2938
2939// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2940// float type.
2941SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2942 SDLoc DL(N);
2943 EVT VT = N->getValueType(ResNo: 0);
2944 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2945 SDValue NV = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Operand: N->getOperand(Num: 0));
2946 // Round the value to the desired precision (that of the source type).
2947 return DAG.getNode(
2948 Opcode: ISD::FP_EXTEND, DL, VT: NVT,
2949 Operand: DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NV,
2950 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)));
2951}
2952
2953SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2954 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2955 VT: N->getValueType(ResNo: 0)));
2956}
2957
2958SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2959 // Expand and promote recursively.
2960 // TODO: This is non-optimal, but dealing with the concurrently happening
2961 // vector-legalization is non-trivial. We could do something similar to
2962 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2963 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
2964 return SDValue();
2965}
2966
2967SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2968 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
2969 return SDValue();
2970}
2971
2972SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2973 EVT VT = N->getValueType(ResNo: 0);
2974
2975 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2976 SDLoc SL(N);
2977
2978 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2979 EVT CastVT = CastVal.getValueType();
2980
2981 SDValue NewAtomic
2982 = DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl: SL, MemVT: CastVT,
2983 VTList: DAG.getVTList(VT1: CastVT, VT2: MVT::Other),
2984 Ops: { AM->getChain(), AM->getBasePtr(), CastVal },
2985 MMO: AM->getMemOperand());
2986
2987 SDValue Result = NewAtomic;
2988
2989 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2990 EVT NFPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2991 Result = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NFPVT), DL: SL, VT: NFPVT,
2992 Operand: NewAtomic);
2993 }
2994
2995 // Legalize the chain result by replacing uses of the old value chain with the
2996 // new one
2997 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2998
2999 return Result;
3000
3001}
3002
3003//===----------------------------------------------------------------------===//
3004// Half Result Soft Promotion
3005//===----------------------------------------------------------------------===//
3006
3007void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
3008 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
3009 N->dump(&DAG));
3010 SDValue R = SDValue();
3011
3012 // See if the target wants to custom expand this node.
3013 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
3014 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
3015 return;
3016 }
3017
3018 switch (N->getOpcode()) {
3019 default:
3020#ifndef NDEBUG
3021 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
3022 N->dump(&DAG); dbgs() << "\n";
3023#endif
3024 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3025 "result!");
3026
3027 case ISD::ARITH_FENCE:
3028 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
3029 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
3030 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
3031 case ISD::EXTRACT_VECTOR_ELT:
3032 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
3033 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
3034 case ISD::STRICT_FP_ROUND:
3035 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
3036
3037 // Unary FP Operations
3038 case ISD::FABS:
3039 case ISD::FACOS:
3040 case ISD::FASIN:
3041 case ISD::FATAN:
3042 case ISD::FCBRT:
3043 case ISD::FCEIL:
3044 case ISD::FCOS:
3045 case ISD::FCOSH:
3046 case ISD::FEXP:
3047 case ISD::FEXP2:
3048 case ISD::FEXP10:
3049 case ISD::FFLOOR:
3050 case ISD::FLOG:
3051 case ISD::FLOG2:
3052 case ISD::FLOG10:
3053 case ISD::FNEARBYINT:
3054 case ISD::FNEG:
3055 case ISD::FREEZE:
3056 case ISD::FRINT:
3057 case ISD::FROUND:
3058 case ISD::FROUNDEVEN:
3059 case ISD::FSIN:
3060 case ISD::FSINH:
3061 case ISD::FSQRT:
3062 case ISD::FTRUNC:
3063 case ISD::FTAN:
3064 case ISD::FTANH:
3065 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
3066
3067 // Binary FP Operations
3068 case ISD::FADD:
3069 case ISD::FDIV:
3070 case ISD::FMAXIMUM:
3071 case ISD::FMINIMUM:
3072 case ISD::FMAXNUM:
3073 case ISD::FMINNUM:
3074 case ISD::FMUL:
3075 case ISD::FPOW:
3076 case ISD::FREM:
3077 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
3078
3079 case ISD::FMA: // FMA is same as FMAD
3080 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
3081
3082 case ISD::FPOWI:
3083 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
3084
3085 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
3086
3087 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
3088 case ISD::ATOMIC_LOAD:
3089 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
3090 break;
3091 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
3092 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
3093 case ISD::SINT_TO_FP:
3094 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
3095 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
3096 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
3097 case ISD::VECREDUCE_FADD:
3098 case ISD::VECREDUCE_FMUL:
3099 case ISD::VECREDUCE_FMIN:
3100 case ISD::VECREDUCE_FMAX:
3101 case ISD::VECREDUCE_FMAXIMUM:
3102 case ISD::VECREDUCE_FMINIMUM:
3103 R = SoftPromoteHalfRes_VECREDUCE(N);
3104 break;
3105 case ISD::VECREDUCE_SEQ_FADD:
3106 case ISD::VECREDUCE_SEQ_FMUL:
3107 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
3108 break;
3109 }
3110
3111 if (R.getNode())
3112 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
3113}
3114
3115SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
3116 return DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: MVT::i16,
3117 Operand: BitConvertToInteger(Op: N->getOperand(Num: 0)));
3118}
3119
3120SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
3121 return BitConvertToInteger(Op: N->getOperand(Num: 0));
3122}
3123
3124SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
3125 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
3126
3127 // Get the (bit-cast) APInt of the APFloat and build an integer constant
3128 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
3129 VT: MVT::i16);
3130}
3131
3132SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
3133 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
3134 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
3135 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
3136 N2: N->getOperand(Num: 1));
3137}
3138
3139SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
3140 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3141 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
3142 SDLoc dl(N);
3143
3144 EVT LVT = LHS.getValueType();
3145 EVT RVT = RHS.getValueType();
3146
3147 unsigned LSize = LVT.getSizeInBits();
3148 unsigned RSize = RVT.getSizeInBits();
3149
3150 // First get the sign bit of second operand.
3151 SDValue SignBit = DAG.getNode(
3152 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
3153 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
3154 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
3155 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
3156
3157 // Shift right or sign-extend it if the two operands have different types.
3158 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3159 if (SizeDiff > 0) {
3160 SignBit =
3161 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
3162 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
3163 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
3164 DL: DAG.getDataLayout())));
3165 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
3166 } else if (SizeDiff < 0) {
3167 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
3168 SignBit =
3169 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
3170 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
3171 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
3172 DL: DAG.getDataLayout())));
3173 }
3174
3175 // Clear the sign bit of the first operand.
3176 SDValue Mask = DAG.getNode(
3177 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
3178 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
3179 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
3180 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
3181 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
3182
3183 // Or the value with the sign bit.
3184 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
3185}
3186
3187SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3188 EVT OVT = N->getValueType(ResNo: 0);
3189 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3190 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3191 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3192 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3193 SDLoc dl(N);
3194
3195 // Promote to the larger FP type.
3196 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
3197 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3198 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3199 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
3200
3201 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2);
3202
3203 // Convert back to FP16 as an integer.
3204 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3205}
3206
3207SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3208 EVT OVT = N->getValueType(ResNo: 0);
3209 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3210 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3211 SDValue Op1 = N->getOperand(Num: 1);
3212 SDLoc dl(N);
3213
3214 // Promote to the larger FP type.
3215 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
3216
3217 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3218
3219 // Convert back to FP16 as an integer.
3220 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3221}
3222
3223SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3224 EVT OVT = N->getValueType(ResNo: 0);
3225 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3226 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3227 SDLoc dl(N);
3228
3229 // Promote to the larger FP type.
3230 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3231
3232 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
3233 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
3234
3235 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3236
3237 // Convert back to FP16 as an integer.
3238 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3239}
3240
3241SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3242 EVT RVT = N->getValueType(ResNo: 0);
3243 EVT SVT = N->getOperand(Num: 0).getValueType();
3244
3245 if (N->isStrictFPOpcode()) {
3246 // FIXME: assume we only have two f16 variants for now.
3247 unsigned Opcode;
3248 if (RVT == MVT::f16)
3249 Opcode = ISD::STRICT_FP_TO_FP16;
3250 else if (RVT == MVT::bf16)
3251 Opcode = ISD::STRICT_FP_TO_BF16;
3252 else
3253 llvm_unreachable("unknown half type");
3254 SDValue Res = DAG.getNode(Opcode, DL: SDLoc(N), ResultTys: {MVT::i16, MVT::Other},
3255 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
3256 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3257 return Res;
3258 }
3259
3260 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: MVT::i16,
3261 Operand: N->getOperand(Num: 0));
3262}
3263
3264SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3265 LoadSDNode *L = cast<LoadSDNode>(Val: N);
3266
3267 // Load the value as an integer value with the same number of bits.
3268 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3269 SDValue NewL =
3270 DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: MVT::i16,
3271 dl: SDLoc(N), Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
3272 PtrInfo: L->getPointerInfo(), MemVT: MVT::i16, Alignment: L->getOriginalAlign(),
3273 MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo());
3274 // Legalize the chain result by replacing uses of the old value chain with the
3275 // new one
3276 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3277 return NewL;
3278}
3279
3280SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3281 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
3282
3283 // Load the value as an integer value with the same number of bits.
3284 SDValue NewL = DAG.getAtomic(
3285 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: MVT::i16, VTList: DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other),
3286 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
3287
3288 // Legalize the chain result by replacing uses of the old value chain with the
3289 // new one
3290 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3291 return NewL;
3292}
3293
3294SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3295 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3296 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3297 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1,
3298 RHS: Op2);
3299}
3300
3301SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3302 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3303 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
3304 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
3305 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
3306 N5: N->getOperand(Num: 4));
3307}
3308
3309SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3310 EVT OVT = N->getValueType(ResNo: 0);
3311 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3312 SDLoc dl(N);
3313
3314 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
3315
3316 // Round the value to the softened type.
3317 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3318}
3319
3320SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3321 return DAG.getUNDEF(VT: MVT::i16);
3322}
3323
3324SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3325 EVT OVT = N->getValueType(ResNo: 0);
3326 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3327 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3328 SDLoc dl(N);
3329
3330 // Promote to the larger FP type.
3331 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3332
3333 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
3334
3335 // Convert back to FP16 as an integer.
3336 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3337}
3338
3339SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3340 EVT OVT = N->getValueType(ResNo: 0);
3341 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3342 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3343 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3344 SDLoc dl(N);
3345
3346 // Promote to the larger FP type.
3347 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
3348 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3349 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3350
3351 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3352
3353 // Convert back to FP16 as an integer.
3354 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3355}
3356
3357SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3358 // Expand and soften recursively.
3359 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
3360 return SDValue();
3361}
3362
3363SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3364 // Expand and soften.
3365 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3366 return SDValue();
3367}
3368
3369//===----------------------------------------------------------------------===//
3370// Half Operand Soft Promotion
3371//===----------------------------------------------------------------------===//
3372
3373bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3374 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3375 N->dump(&DAG));
3376 SDValue Res = SDValue();
3377
3378 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
3379 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3380 return false;
3381 }
3382
3383 // Nodes that use a promotion-requiring floating point operand, but doesn't
3384 // produce a soft promotion-requiring floating point result, need to be
3385 // legalized to use the soft promoted float operand. Nodes that produce at
3386 // least one soft promotion-requiring floating point result have their
3387 // operands legalized as a part of PromoteFloatResult.
3388 switch (N->getOpcode()) {
3389 default:
3390 #ifndef NDEBUG
3391 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3392 N->dump(&DAG); dbgs() << "\n";
3393 #endif
3394 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3395 "operand!");
3396
3397 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3398 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3399 case ISD::FP_TO_SINT:
3400 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3401 case ISD::FP_TO_SINT_SAT:
3402 case ISD::FP_TO_UINT_SAT:
3403 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3404 case ISD::STRICT_FP_EXTEND:
3405 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3406 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3407 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3408 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3409 case ISD::ATOMIC_STORE:
3410 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3411 break;
3412 case ISD::STACKMAP:
3413 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3414 break;
3415 case ISD::PATCHPOINT:
3416 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3417 break;
3418 }
3419
3420 if (!Res.getNode())
3421 return false;
3422
3423 assert(Res.getNode() != N && "Expected a new node!");
3424
3425 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3426 "Invalid operand expansion");
3427
3428 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3429 return false;
3430}
3431
3432SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3433 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3434
3435 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3436}
3437
3438SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3439 unsigned OpNo) {
3440 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3441 SDValue Op1 = N->getOperand(Num: 1);
3442 EVT RVT = Op1.getValueType();
3443 SDLoc dl(N);
3444
3445 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3446
3447 Op1 = GetSoftPromotedHalf(Op: Op1);
3448 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3449
3450 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3451 N2: Op1);
3452}
3453
3454SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3455 EVT RVT = N->getValueType(ResNo: 0);
3456 bool IsStrict = N->isStrictFPOpcode();
3457 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3458 EVT SVT = Op.getValueType();
3459 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3460
3461 if (IsStrict) {
3462 unsigned Opcode;
3463 if (SVT == MVT::f16)
3464 Opcode = ISD::STRICT_FP16_TO_FP;
3465 else if (SVT == MVT::bf16)
3466 Opcode = ISD::STRICT_BF16_TO_FP;
3467 else
3468 llvm_unreachable("unknown half type");
3469 SDValue Res =
3470 DAG.getNode(Opcode, DL: SDLoc(N), ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
3471 Ops: {N->getOperand(Num: 0), Op});
3472 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3473 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3474 return SDValue();
3475 }
3476
3477 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3478}
3479
3480SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3481 EVT RVT = N->getValueType(ResNo: 0);
3482 SDValue Op = N->getOperand(Num: 0);
3483 EVT SVT = Op.getValueType();
3484 SDLoc dl(N);
3485
3486 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3487
3488 Op = GetSoftPromotedHalf(Op);
3489
3490 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3491
3492 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), Operand: Res);
3493}
3494
3495SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3496 EVT RVT = N->getValueType(ResNo: 0);
3497 SDValue Op = N->getOperand(Num: 0);
3498 EVT SVT = Op.getValueType();
3499 SDLoc dl(N);
3500
3501 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3502
3503 Op = GetSoftPromotedHalf(Op);
3504
3505 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3506
3507 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3508 N2: N->getOperand(Num: 1));
3509}
3510
3511SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3512 unsigned OpNo) {
3513 assert(OpNo == 0 && "Can only soften the comparison values");
3514 SDValue Op0 = N->getOperand(Num: 0);
3515 SDValue Op1 = N->getOperand(Num: 1);
3516 SDLoc dl(N);
3517
3518 EVT SVT = Op0.getValueType();
3519 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3520
3521 Op0 = GetSoftPromotedHalf(Op: Op0);
3522 Op1 = GetSoftPromotedHalf(Op: Op1);
3523
3524 // Promote to the larger FP type.
3525 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3526 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3527 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3528
3529 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3530 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3531}
3532
3533SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3534 SDValue Op0 = N->getOperand(Num: 0);
3535 SDValue Op1 = N->getOperand(Num: 1);
3536 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3537 SDLoc dl(N);
3538
3539 EVT SVT = Op0.getValueType();
3540 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3541
3542 Op0 = GetSoftPromotedHalf(Op: Op0);
3543 Op1 = GetSoftPromotedHalf(Op: Op1);
3544
3545 // Promote to the larger FP type.
3546 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3547 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3548 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3549
3550 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3551}
3552
3553SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3554 assert(OpNo == 1 && "Can only soften the stored value!");
3555 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3556 SDValue Val = ST->getValue();
3557 SDLoc dl(N);
3558
3559 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3560 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3561 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3562 MMO: ST->getMemOperand());
3563}
3564
3565SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3566 unsigned OpNo) {
3567 assert(OpNo == 1 && "Can only soften the stored value!");
3568 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
3569 SDValue Val = ST->getVal();
3570 SDLoc dl(N);
3571
3572 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3573 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: Promoted.getValueType(),
3574 Chain: ST->getChain(), Ptr: Promoted, Val: ST->getBasePtr(),
3575 MMO: ST->getMemOperand());
3576}
3577
3578SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3579 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3580 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3581 SDValue Op = N->getOperand(Num: OpNo);
3582 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3583 SDValue NewNode =
3584 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3585
3586 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3587 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3588
3589 return SDValue(); // Signal that we replaced the node ourselves.
3590}
3591
3592SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3593 unsigned OpNo) {
3594 assert(OpNo >= 7);
3595 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3596 SDValue Op = N->getOperand(Num: OpNo);
3597 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3598 SDValue NewNode =
3599 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3600
3601 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3602 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3603
3604 return SDValue(); // Signal that we replaced the node ourselves.
3605}
3606