1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error(reason: "Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::FCANONICALIZE:
74 R = SoftenFloatRes_FCANONICALIZE(N); break;
75 case ISD::STRICT_FMINNUM:
76 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
77 case ISD::STRICT_FMAXNUM:
78 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
79 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
80 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
81 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
82 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
83 case ISD::STRICT_FADD:
84 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
85 case ISD::STRICT_FACOS:
86 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
87 case ISD::STRICT_FASIN:
88 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
89 case ISD::STRICT_FATAN:
90 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
91 case ISD::STRICT_FATAN2:
92 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
93 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
94 case ISD::STRICT_FCEIL:
95 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
96 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
97 case ISD::STRICT_FCOS:
98 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
99 case ISD::STRICT_FCOSH:
100 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
101 case ISD::STRICT_FDIV:
102 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
103 case ISD::STRICT_FEXP:
104 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
105 case ISD::STRICT_FEXP2:
106 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
107 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
108 case ISD::STRICT_FFLOOR:
109 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
110 case ISD::STRICT_FLOG:
111 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
112 case ISD::STRICT_FLOG2:
113 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
114 case ISD::STRICT_FLOG10:
115 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
116 case ISD::STRICT_FMA:
117 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
118 case ISD::STRICT_FMUL:
119 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
120 case ISD::STRICT_FNEARBYINT:
121 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
122 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
123 case ISD::STRICT_FP_EXTEND:
124 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
125 case ISD::STRICT_FP_ROUND:
126 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
127 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
128 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
129 case ISD::STRICT_FPOW:
130 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
131 case ISD::STRICT_FPOWI:
132 case ISD::FPOWI:
133 case ISD::FLDEXP:
134 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
135 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
136 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
137 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
138 case ISD::STRICT_FREM:
139 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
140 case ISD::STRICT_FRINT:
141 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
142 case ISD::STRICT_FROUND:
143 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
144 case ISD::STRICT_FROUNDEVEN:
145 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
146 case ISD::STRICT_FSIN:
147 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
148 case ISD::STRICT_FSINH:
149 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
150 case ISD::STRICT_FSQRT:
151 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
152 case ISD::STRICT_FSUB:
153 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
154 case ISD::STRICT_FTAN:
155 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
156 case ISD::STRICT_FTANH:
157 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
158 case ISD::STRICT_FTRUNC:
159 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
160 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
161 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
162 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
163 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
164 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
165 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
166 case ISD::STRICT_SINT_TO_FP:
167 case ISD::STRICT_UINT_TO_FP:
168 case ISD::SINT_TO_FP:
169 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
170 case ISD::POISON:
171 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
172 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
173 case ISD::AssertNoFPClass: R = GetSoftenedFloat(Op: N->getOperand(Num: 0)); break;
174 case ISD::VECREDUCE_FADD:
175 case ISD::VECREDUCE_FMUL:
176 case ISD::VECREDUCE_FMIN:
177 case ISD::VECREDUCE_FMAX:
178 case ISD::VECREDUCE_FMAXIMUM:
179 case ISD::VECREDUCE_FMINIMUM:
180 case ISD::VECREDUCE_FMAXIMUMNUM:
181 case ISD::VECREDUCE_FMINIMUMNUM: R = SoftenFloatRes_VECREDUCE(N); break;
182 case ISD::VECREDUCE_SEQ_FADD:
183 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
184 // clang-format on
185 }
186
187 // If R is null, the sub-method took care of registering the result.
188 if (R.getNode()) {
189 assert(R.getNode() != N);
190 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
191 }
192}
193
194// No libcall is available to soften this operation. Emit a diagnostic and
195// produce a poison result of the softened type \p NVT.
196SDValue DAGTypeLegalizer::SoftenFloatRes_NoLibcall(SDNode *N, EVT NVT) {
197 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
198 N->getOperationName(G: &DAG));
199 if (N->isStrictFPOpcode())
200 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
201 return DAG.getPOISON(VT: NVT);
202}
203
204SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
205 bool IsStrict = N->isStrictFPOpcode();
206 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
207 unsigned Offset = IsStrict ? 1 : 0;
208 assert(N->getNumOperands() == (1 + Offset) &&
209 "Unexpected number of operands!");
210 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
211 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
212 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
213 if (LCImpl == RTLIB::Unsupported)
214 return SoftenFloatRes_NoLibcall(N, NVT);
215 TargetLowering::MakeLibCallOptions CallOptions;
216 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
217 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
218 std::pair<SDValue, SDValue> Tmp =
219 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
220 if (IsStrict)
221 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
222 return Tmp.first;
223}
224
225SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
226 bool IsStrict = N->isStrictFPOpcode();
227 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
228 unsigned Offset = IsStrict ? 1 : 0;
229 assert(N->getNumOperands() == (2 + Offset) &&
230 "Unexpected number of operands!");
231 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
232 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
233 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
234 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
235 if (LCImpl == RTLIB::Unsupported)
236 return SoftenFloatRes_NoLibcall(N, NVT);
237 TargetLowering::MakeLibCallOptions CallOptions;
238 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
239 N->getOperand(Num: 1 + Offset).getValueType() };
240 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
241 std::pair<SDValue, SDValue> Tmp =
242 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
243 if (IsStrict)
244 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
245 return Tmp.first;
246}
247
248SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
249 return BitConvertToInteger(Op: N->getOperand(Num: 0));
250}
251
252SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
253 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
254 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
255 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
256}
257
258SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
259 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
260 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
261 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
262 return NewFence;
263}
264
265SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
266 unsigned ResNo) {
267 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
268 return BitConvertToInteger(Op);
269}
270
271SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
272 // Convert the inputs to integers, and build a new pair out of them.
273 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
274 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
275 VT: N->getValueType(ResNo: 0)),
276 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
277 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
278}
279
280SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
281 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
282 // In ppcf128, the high 64 bits are always first in memory regardless
283 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
284 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
285 // way. However, APInt's are serialized in an Endian-sensitive fashion,
286 // so on big-Endian targets, the two doubles are output in the wrong
287 // order. Fix this by manually flipping the order of the high 64 bits
288 // and the low 64 bits here.
289 if (DAG.getDataLayout().isBigEndian() &&
290 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
291 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
292 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
293 APInt Val(128, words);
294 return DAG.getConstant(Val, DL: SDLoc(CN),
295 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
296 VT: CN->getValueType(ResNo: 0)));
297 } else {
298 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
299 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
300 VT: CN->getValueType(ResNo: 0)));
301 }
302}
303
304SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
305 SDValue Src = N->getOperand(Num: 0);
306 assert(Src.getValueType() == MVT::ppcf128 &&
307 "In floats only ppcf128 can be extracted by element!");
308 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(N),
309 VT: N->getValueType(ResNo: 0).changeTypeToInteger(),
310 N1: DAG.getBitcast(VT: MVT::i128, V: Src), N2: N->getOperand(Num: 1));
311}
312
313SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
314 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
315 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
316 VT: NewOp.getValueType().getVectorElementType(),
317 N1: NewOp, N2: N->getOperand(Num: 1));
318}
319
320SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
321 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
322 unsigned Size = NVT.getSizeInBits();
323
324 // Mask = ~(1 << (Size-1))
325 APInt API = APInt::getAllOnes(numBits: Size);
326 API.clearBit(BitPosition: Size - 1);
327 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
328 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
329 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
330}
331
332SDValue DAGTypeLegalizer::SoftenFloatRes_FCANONICALIZE(SDNode *N) {
333 SDLoc dl(N);
334
335 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
336 // suggested in
337 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
338 // It uses strict_fp operations even outside a strict_fp context in order
339 // to guarantee that the canonicalization is not optimized away by later
340 // passes. The result chain introduced by that is intentionally ignored
341 // since no ordering requirement is intended here.
342
343 // Create strict multiplication by 1.0.
344 SDValue Operand = N->getOperand(Num: 0);
345 EVT VT = Operand.getValueType();
346 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
347 SDValue Chain = DAG.getEntryNode();
348 // Propagate existing flags on canonicalize, and additionally set
349 // NoFPExcept.
350 SDNodeFlags CanonicalizeFlags = N->getFlags();
351 CanonicalizeFlags.setNoFPExcept(true);
352 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
353 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
354 return BitConvertToInteger(Op: Mul);
355}
356
357SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
358 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
359 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
360 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMIN(VT: N->getValueType(ResNo: 0)));
361}
362
363SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
364 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
365 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
366 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAX(VT: N->getValueType(ResNo: 0)));
367}
368
369SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
370 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMINIMUM_NUM(VT: N->getValueType(ResNo: 0)));
371}
372
373SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
374 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM_NUM(VT: N->getValueType(ResNo: 0)));
375}
376
377SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
378 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMINIMUM(VT: N->getValueType(ResNo: 0)));
379}
380
381SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
382 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM(VT: N->getValueType(ResNo: 0)));
383}
384
385SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
386 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
387 Call_F32: RTLIB::ADD_F32,
388 Call_F64: RTLIB::ADD_F64,
389 Call_F80: RTLIB::ADD_F80,
390 Call_F128: RTLIB::ADD_F128,
391 Call_PPCF128: RTLIB::ADD_PPCF128));
392}
393
394SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
395 return SoftenFloatRes_Unary(N, LC: RTLIB::getACOS(VT: N->getValueType(ResNo: 0)));
396}
397
398SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
399 return SoftenFloatRes_Unary(N, LC: RTLIB::getASIN(VT: N->getValueType(ResNo: 0)));
400}
401
402SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
403 return SoftenFloatRes_Unary(N, LC: RTLIB::getATAN(VT: N->getValueType(ResNo: 0)));
404}
405
406SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
407 return SoftenFloatRes_Binary(N, LC: RTLIB::getATAN2(VT: N->getValueType(ResNo: 0)));
408}
409
410SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
411 return SoftenFloatRes_Unary(N, LC: RTLIB::getCBRT(VT: N->getValueType(ResNo: 0)));
412}
413
414SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
415 return SoftenFloatRes_Unary(N, LC: RTLIB::getCEIL(VT: N->getValueType(ResNo: 0)));
416}
417
418SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
419 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
420 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
421 SDLoc dl(N);
422
423 EVT LVT = LHS.getValueType();
424 EVT RVT = RHS.getValueType();
425
426 unsigned LSize = LVT.getSizeInBits();
427 unsigned RSize = RVT.getSizeInBits();
428
429 // First get the sign bit of second operand.
430 SDValue SignBit = DAG.getNode(
431 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
432 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
433 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
434 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
435
436 // Shift right or sign-extend it if the two operands have different types.
437 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
438 if (SizeDiff > 0) {
439 SignBit =
440 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
441 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
442 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
443 DL: DAG.getDataLayout())));
444 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
445 } else if (SizeDiff < 0) {
446 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
447 SignBit =
448 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
449 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
450 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
451 DL: DAG.getDataLayout())));
452 }
453
454 // Clear the sign bit of the first operand.
455 SDValue Mask = DAG.getNode(
456 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
457 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
458 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
459 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
460 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
461
462 // Or the value with the sign bit.
463 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
464}
465
466SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
467 return SoftenFloatRes_Unary(N, LC: RTLIB::getCOS(VT: N->getValueType(ResNo: 0)));
468}
469
470SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
471 return SoftenFloatRes_Unary(N, LC: RTLIB::getCOSH(VT: N->getValueType(ResNo: 0)));
472}
473
474SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
475 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
476 Call_F32: RTLIB::DIV_F32,
477 Call_F64: RTLIB::DIV_F64,
478 Call_F80: RTLIB::DIV_F80,
479 Call_F128: RTLIB::DIV_F128,
480 Call_PPCF128: RTLIB::DIV_PPCF128));
481}
482
483SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
484 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP(VT: N->getValueType(ResNo: 0)));
485}
486
487SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
488 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP2(VT: N->getValueType(ResNo: 0)));
489}
490
491SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
492 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP10(VT: N->getValueType(ResNo: 0)));
493}
494
495SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
496 return SoftenFloatRes_Unary(N, LC: RTLIB::getFLOOR(VT: N->getValueType(ResNo: 0)));
497}
498
499SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
500 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG(VT: N->getValueType(ResNo: 0)));
501}
502
503SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
504 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG2(VT: N->getValueType(ResNo: 0)));
505}
506
507SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
508 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG10(VT: N->getValueType(ResNo: 0)));
509}
510
511SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
512 bool IsStrict = N->isStrictFPOpcode();
513 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
514 unsigned Offset = IsStrict ? 1 : 0;
515 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
516 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
517 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
518 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
519 TargetLowering::MakeLibCallOptions CallOptions;
520 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
521 N->getOperand(Num: 1 + Offset).getValueType(),
522 N->getOperand(Num: 2 + Offset).getValueType() };
523 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
524 std::pair<SDValue, SDValue> Tmp =
525 TLI.makeLibCall(DAG, LC: RTLIB::getFMA(VT: N->getValueType(ResNo: 0)), RetVT: NVT, Ops,
526 CallOptions, dl: SDLoc(N), Chain);
527 if (IsStrict)
528 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
529 return Tmp.first;
530}
531
532SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
533 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
534 Call_F32: RTLIB::MUL_F32,
535 Call_F64: RTLIB::MUL_F64,
536 Call_F80: RTLIB::MUL_F80,
537 Call_F128: RTLIB::MUL_F128,
538 Call_PPCF128: RTLIB::MUL_PPCF128));
539}
540
541SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
542 return SoftenFloatRes_Unary(N, LC: RTLIB::getNEARBYINT(VT: N->getValueType(ResNo: 0)));
543}
544
545SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
546 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
547 SDLoc dl(N);
548
549 // Expand Y = FNEG(X) -> Y = X ^ sign mask
550 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
551 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
552 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
553}
554
555SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
556 bool IsStrict = N->isStrictFPOpcode();
557 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
558 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
559
560 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
561
562 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
563 // -> f32, so proceed in two stages. Also, it's entirely possible for both
564 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
565 // than FP16_TO_FP.
566 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
567 N->getValueType(ResNo: 0) != MVT::f32) {
568 if (IsStrict) {
569 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
570 ResultTys: { MVT::f32, MVT::Other }, Ops: { Chain, Op });
571 Chain = Op.getValue(R: 1);
572 } else {
573 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT: MVT::f32, Operand: Op);
574 }
575 }
576
577 if (Op.getValueType() == MVT::bf16) {
578 // FIXME: Need ReplaceValueWith on chain in strict case
579 return SoftenFloatRes_BF16_TO_FP(N);
580 }
581
582 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
583 if (LC == RTLIB::UNKNOWN_LIBCALL) {
584 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fp_extend");
585 if (IsStrict)
586 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
587 return DAG.getPOISON(VT: NVT);
588 }
589 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
590 if (LCImpl == RTLIB::Unsupported)
591 return SoftenFloatRes_NoLibcall(N, NVT);
592 TargetLowering::MakeLibCallOptions CallOptions;
593 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
594 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
595 std::pair<SDValue, SDValue> Tmp =
596 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
597 if (IsStrict)
598 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
599 return Tmp.first;
600}
601
602// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
603// nodes?
604SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
605 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
606 SDValue Op = N->getOperand(Num: 0);
607 TargetLowering::MakeLibCallOptions CallOptions;
608 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
609 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
610 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
611 CallOptions, dl: SDLoc(N)).first;
612 if (N->getValueType(ResNo: 0) == MVT::f32)
613 return Res32;
614
615 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
616 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: MVT::f32, RetVT: N->getValueType(ResNo: 0));
617 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
618 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
619}
620
621// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
622// nodes?
623SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
624 assert(N->getValueType(0) == MVT::f32 &&
625 "Can only soften BF16_TO_FP with f32 result");
626 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
627 SDValue Op = N->getOperand(Num: 0);
628 SDLoc DL(N);
629 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT,
630 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op));
631 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
632 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
633 return Res;
634}
635
636SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
637 bool IsStrict = N->isStrictFPOpcode();
638 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
639 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
640 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
641 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
642 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
643 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
644 if (LCImpl == RTLIB::Unsupported)
645 return SoftenFloatRes_NoLibcall(N, NVT);
646 TargetLowering::MakeLibCallOptions CallOptions;
647 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
648 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
649 std::pair<SDValue, SDValue> Tmp =
650 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
651 if (IsStrict)
652 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
653 return Tmp.first;
654}
655
656SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
657 return SoftenFloatRes_Binary(N, LC: RTLIB::getPOW(VT: N->getValueType(ResNo: 0)));
658}
659
660SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
661 bool IsStrict = N->isStrictFPOpcode();
662 unsigned Offset = IsStrict ? 1 : 0;
663 bool IsPowI =
664 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
665 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
666
667 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(VT: N->getValueType(ResNo: 0))
668 : RTLIB::getLDEXP(VT: N->getValueType(ResNo: 0));
669 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
670 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
671 if (LCImpl == RTLIB::Unsupported) {
672 // Some targets don't have a powi libcall; use pow instead.
673 // FIXME: Implement this if some target needs it.
674 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fpowi to fpow");
675 if (IsStrict)
676 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
677 return DAG.getPOISON(VT: NVT);
678 }
679
680 if (DAG.getLibInfo().getIntSize() !=
681 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
682 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
683 // would use the wrong type for the argument.
684 DAG.getContext()->emitError(ErrorStr: "powi exponent does not match sizeof(int)");
685 if (IsStrict)
686 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
687 return DAG.getPOISON(VT: NVT);
688 }
689
690 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
691 N->getOperand(Num: 1 + Offset) };
692 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
693 TargetLowering::MakeLibCallOptions CallOptions;
694 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
695 N->getOperand(Num: 1 + Offset).getValueType() };
696 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
697 std::pair<SDValue, SDValue> Tmp =
698 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
699 if (IsStrict)
700 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
701 return Tmp.first;
702}
703
704SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
705 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
706 EVT VT0 = N->getValueType(ResNo: 0);
707 EVT VT1 = N->getValueType(ResNo: 1);
708 RTLIB::Libcall LC = RTLIB::getFREXP(VT: VT0);
709 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
710 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
711 SDLoc DL(N);
712
713 if (LCImpl == RTLIB::Unsupported) {
714 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
715 N->getOperationName(G: &DAG));
716 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
717 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
718 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
719 }
720
721 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
722 // If the exponent does not match with sizeof(int) a libcall would use the
723 // wrong type for the argument.
724 // TODO: Should be able to handle mismatches.
725 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
726 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
727 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
728 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
729 }
730
731 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
732
733 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
734 TargetLowering::MakeLibCallOptions CallOptions;
735 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
736 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
737 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
738
739 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
740 // but we only really need to handle the 0th one for softening anyway.
741 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0)
742 .setOpsTypeOverrides(CallOpsTypeOverrides);
743
744 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT0, Ops, CallOptions,
745 dl: DL, /*Chain=*/SDValue());
746 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
747 auto PtrInfo =
748 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
749
750 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
751
752 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
753 return ReturnVal;
754}
755
756bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
757 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
758 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
759 EVT VT = N->getValueType(ResNo: 0);
760
761 assert(VT == N->getValueType(1) &&
762 "expected both return values to have the same type");
763
764 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
765 if (LCImpl == RTLIB::Unsupported)
766 return false;
767
768 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
769
770 SDLoc DL(N);
771
772 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(Op: N->getOperand(Num: 0))};
773 SmallVector<EVT, 3> OpsVT = {VT};
774
775 std::array<SDValue, 2> StackSlots;
776 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
777 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
778 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
779 if (ResNum == CallRetResNo)
780 continue;
781 SDValue StackSlot = DAG.CreateStackTemporary(VT: NVT);
782 Ops.push_back(Elt: StackSlot);
783 OpsVT.push_back(Elt: StackSlot.getValueType());
784 StackSlots[ResNum] = StackSlot;
785 CallOpsTypeOverrides.push_back(Elt: PointerTy);
786 }
787
788 TargetLowering::MakeLibCallOptions CallOptions;
789 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
790 // but since both returns have the same type it should be okay.
791 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT)
792 .setOpsTypeOverrides(CallOpsTypeOverrides);
793
794 auto [ReturnVal, Chain] =
795 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: DL,
796 /*Chain=*/SDValue());
797
798 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
799 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
800 auto PtrInfo =
801 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
802 return DAG.getLoad(VT: NVT, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
803 };
804
805 for (auto [ResNum, SlackSlot] : enumerate(First&: StackSlots)) {
806 if (CallRetResNo == ResNum) {
807 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: ReturnVal);
808 continue;
809 }
810 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: CreateStackLoad(SlackSlot));
811 }
812
813 return true;
814}
815
816SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
817 EVT VT = N->getValueType(ResNo: 0);
818 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(VT)))
819 return SDValue();
820
821 // Fall back on softening the separate sin and cos calls if available.
822 RTLIB::Libcall SinLC = RTLIB::getSIN(VT);
823 RTLIB::Libcall CosLC = RTLIB::getCOS(VT);
824
825 SDValue SoftSin, SoftCos;
826 if (DAG.getLibcalls().getLibcallImpl(Call: SinLC) == RTLIB::Unsupported ||
827 DAG.getLibcalls().getLibcallImpl(Call: CosLC) == RTLIB::Unsupported) {
828 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fsincos");
829
830 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
831 SoftSin = SoftCos = DAG.getPOISON(VT: NVT);
832 } else {
833 SoftSin = SoftenFloatRes_Unary(N, LC: SinLC);
834 SoftCos = SoftenFloatRes_Unary(N, LC: CosLC);
835 }
836
837 SetSoftenedFloat(Op: SDValue(N, 0), Result: SoftSin);
838 SetSoftenedFloat(Op: SDValue(N, 1), Result: SoftCos);
839 return SDValue();
840}
841
842SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
843 EVT VT = N->getValueType(ResNo: 0);
844 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT),
845 /*CallRetResNo=*/0))
846 return SDValue();
847
848 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
849 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fmodf");
850 SDValue Poison = DAG.getPOISON(VT: NVT);
851 SetSoftenedFloat(Op: SDValue(N, 0), Result: Poison);
852 SetSoftenedFloat(Op: SDValue(N, 1), Result: Poison);
853 return SDValue();
854}
855
856SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
857 return SoftenFloatRes_Binary(N, LC: RTLIB::getREM(VT: N->getValueType(ResNo: 0)));
858}
859
860SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
861 return SoftenFloatRes_Unary(N, LC: RTLIB::getRINT(VT: N->getValueType(ResNo: 0)));
862}
863
864SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
865 return SoftenFloatRes_Unary(N, LC: RTLIB::getROUND(VT: N->getValueType(ResNo: 0)));
866}
867
868SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
869 return SoftenFloatRes_Unary(N, LC: RTLIB::getROUNDEVEN(VT: N->getValueType(ResNo: 0)));
870}
871
872SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
873 return SoftenFloatRes_Unary(N, LC: RTLIB::getSIN(VT: N->getValueType(ResNo: 0)));
874}
875
876SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
877 return SoftenFloatRes_Unary(N, LC: RTLIB::getSINH(VT: N->getValueType(ResNo: 0)));
878}
879
880SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
881 return SoftenFloatRes_Unary(N, LC: RTLIB::getSQRT(VT: N->getValueType(ResNo: 0)));
882}
883
884SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
885 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
886 Call_F32: RTLIB::SUB_F32,
887 Call_F64: RTLIB::SUB_F64,
888 Call_F80: RTLIB::SUB_F80,
889 Call_F128: RTLIB::SUB_F128,
890 Call_PPCF128: RTLIB::SUB_PPCF128));
891}
892
893SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
894 return SoftenFloatRes_Unary(N, LC: RTLIB::getTAN(VT: N->getValueType(ResNo: 0)));
895}
896
897SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
898 return SoftenFloatRes_Unary(N, LC: RTLIB::getTANH(VT: N->getValueType(ResNo: 0)));
899}
900
901SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
902 return SoftenFloatRes_Unary(N, LC: RTLIB::getTRUNC(VT: N->getValueType(ResNo: 0)));
903}
904
905SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
906 LoadSDNode *L = cast<LoadSDNode>(Val: N);
907 EVT VT = N->getValueType(ResNo: 0);
908 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
909 SDLoc dl(N);
910
911 auto MMOFlags =
912 L->getMemOperand()->getFlags() &
913 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
914 SDValue NewL;
915 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
916 // If softening widens the integer representation (e.g. x86_fp80 -> i96),
917 // load the original memory width and extend to the softened type.
918 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
919 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::EXTLOAD, VT: NVT, dl,
920 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
921 PtrInfo: L->getPointerInfo(), MemVT, Alignment: L->getBaseAlign(), MMOFlags,
922 Metadata: L->getAAInfo());
923 // Legalized the chain result - switch anything that used the old chain to
924 // use the new one.
925 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
926 return NewL;
927 }
928
929 // Do a non-extending load followed by FP_EXTEND.
930 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
931 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
932 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(), Alignment: L->getBaseAlign(),
933 MMOFlags, Metadata: L->getAAInfo());
934 // Legalized the chain result - switch anything that used the old chain to
935 // use the new one.
936 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
937 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
938 return BitConvertToInteger(Op: ExtendNode);
939}
940
941SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
942 AtomicSDNode *L = cast<AtomicSDNode>(Val: N);
943 EVT VT = N->getValueType(ResNo: 0);
944 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
945 SDLoc dl(N);
946
947 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
948 SDValue NewL =
949 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
950 Ops: {L->getChain(), L->getBasePtr()}, MMO: L->getMemOperand());
951
952 // Legalized the chain result - switch anything that used the old chain to
953 // use the new one.
954 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
955 return NewL;
956 }
957
958 report_fatal_error(reason: "softening fp extending atomic load not handled");
959}
960
961SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
962 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
963 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
964 return DAG.getSelect(DL: SDLoc(N),
965 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
966}
967
968SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
969 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
970 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
971 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
972 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
973 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
974}
975
976SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
977 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
978 VT: N->getValueType(ResNo: 0)));
979}
980
981SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
982 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
983 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
984 EVT VT = N->getValueType(ResNo: 0);
985 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
986 SDLoc dl(N);
987
988 SDValue NewVAARG;
989 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
990 Align: N->getConstantOperandVal(Num: 3));
991
992 // Legalized the chain result - switch anything that used the old chain to
993 // use the new one.
994 if (N != NewVAARG.getValue(R: 1).getNode())
995 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
996 return NewVAARG;
997}
998
999SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1000 bool IsStrict = N->isStrictFPOpcode();
1001 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1002 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1003 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
1004 EVT RVT = N->getValueType(ResNo: 0);
1005 EVT NVT = EVT();
1006 SDLoc dl(N);
1007
1008 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1009 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1010 // match. Look for an appropriate libcall.
1011 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1012 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1013 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1014 NVT = (MVT::SimpleValueType)t;
1015 // The source needs to big enough to hold the operand.
1016 if (NVT.bitsGE(VT: SVT))
1017 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
1018 }
1019 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1020
1021 EVT NRVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT);
1022 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1023 if (LCImpl == RTLIB::Unsupported)
1024 return SoftenFloatRes_NoLibcall(N, NVT: NRVT);
1025
1026 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1027 // Sign/zero extend the argument if the libcall takes a larger type.
1028 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
1029 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
1030 TargetLowering::MakeLibCallOptions CallOptions;
1031 CallOptions.setIsSigned(Signed);
1032 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1033 std::pair<SDValue, SDValue> Tmp =
1034 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NRVT, Ops: Op, CallOptions, dl, Chain);
1035
1036 if (IsStrict)
1037 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1038 return Tmp.first;
1039}
1040
1041SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1042 // Expand and soften recursively.
1043 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
1044 return SDValue();
1045}
1046
1047SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1048 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
1049 return SDValue();
1050}
1051
1052//===----------------------------------------------------------------------===//
1053// Convert Float Operand to Integer
1054//===----------------------------------------------------------------------===//
1055
1056bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1057 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1058 SDValue Res = SDValue();
1059
1060 switch (N->getOpcode()) {
1061 default:
1062#ifndef NDEBUG
1063 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1064 N->dump(&DAG); dbgs() << "\n";
1065#endif
1066 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
1067
1068 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1069 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1070 case ISD::STRICT_FP_TO_FP16:
1071 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1072 case ISD::FP_TO_BF16:
1073 case ISD::STRICT_FP_TO_BF16:
1074 case ISD::STRICT_FP_ROUND:
1075 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1076 case ISD::STRICT_FP_TO_SINT:
1077 case ISD::STRICT_FP_TO_UINT:
1078 case ISD::FP_TO_SINT:
1079 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1080 case ISD::FP_TO_SINT_SAT:
1081 case ISD::FP_TO_UINT_SAT:
1082 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1083 case ISD::STRICT_LROUND:
1084 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1085 case ISD::STRICT_LLROUND:
1086 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1087 case ISD::STRICT_LRINT:
1088 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1089 case ISD::STRICT_LLRINT:
1090 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1091 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1092 case ISD::STRICT_FSETCC:
1093 case ISD::STRICT_FSETCCS:
1094 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1095 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1096 case ISD::ATOMIC_STORE:
1097 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1098 break;
1099 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1100 case ISD::FAKE_USE:
1101 Res = SoftenFloatOp_FAKE_USE(N);
1102 break;
1103 case ISD::STACKMAP:
1104 Res = SoftenFloatOp_STACKMAP(N, OpNo);
1105 break;
1106 case ISD::PATCHPOINT:
1107 Res = SoftenFloatOp_PATCHPOINT(N, OpNo);
1108 break;
1109 }
1110
1111 // If the result is null, the sub-method took care of registering results etc.
1112 if (!Res.getNode()) return false;
1113
1114 // If the result is N, the sub-method updated N in place. Tell the legalizer
1115 // core about this to re-analyze.
1116 if (Res.getNode() == N)
1117 return true;
1118
1119 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1120 "Invalid operand softening");
1121
1122 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1123 return false;
1124}
1125
1126SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1127 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
1128
1129 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
1130}
1131
1132SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1133 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1134 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1135 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1136 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1137 N->getOpcode() == ISD::FP_TO_BF16 ||
1138 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1139 N->getOpcode() == ISD::STRICT_FP_ROUND);
1140
1141 bool IsStrict = N->isStrictFPOpcode();
1142 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1143 EVT SVT = Op.getValueType();
1144 EVT RVT = N->getValueType(ResNo: 0);
1145 EVT FloatRVT = RVT;
1146 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1147 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1148 FloatRVT = MVT::f16;
1149 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1150 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1151 FloatRVT = MVT::bf16;
1152
1153 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
1154 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1155
1156 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1157 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1158 if (LCImpl == RTLIB::Unsupported) {
1159 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
1160 N->getOperationName(G: &DAG));
1161 SDValue Poison = DAG.getPOISON(VT: RVT);
1162 if (IsStrict) {
1163 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1164 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1165 return SDValue();
1166 }
1167 return Poison;
1168 }
1169 Op = GetSoftenedFloat(Op);
1170 TargetLowering::MakeLibCallOptions CallOptions;
1171 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1172 std::pair<SDValue, SDValue> Tmp =
1173 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1174 if (IsStrict) {
1175 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1176 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1177 return SDValue();
1178 }
1179 return Tmp.first;
1180}
1181
1182SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1183 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1184 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1185
1186 EVT VT = NewLHS.getValueType();
1187 NewLHS = GetSoftenedFloat(Op: NewLHS);
1188 NewRHS = GetSoftenedFloat(Op: NewRHS);
1189 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1190 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1191
1192 // If softenSetCCOperands returned a scalar, we need to compare the result
1193 // against zero to select between true and false values.
1194 if (!NewRHS.getNode()) {
1195 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1196 CCCode = ISD::SETNE;
1197 }
1198
1199 // Update N to have the operands specified.
1200 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1201 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1202 Op5: N->getOperand(Num: 4)),
1203 0);
1204}
1205
1206// Even if the result type is legal, no libcall may exactly match. (e.g. We
1207// don't have FP-i8 conversions) This helper method looks for an appropriate
1208// promoted libcall.
1209static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1210 bool Signed) {
1211 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1212 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1213 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1214 ++IntVT) {
1215 Promoted = (MVT::SimpleValueType)IntVT;
1216 // The type needs to big enough to hold the result.
1217 if (Promoted.bitsGE(VT: RetVT))
1218 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1219 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1220 }
1221 return LC;
1222}
1223
1224SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1225 bool IsStrict = N->isStrictFPOpcode();
1226 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1227 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1228
1229 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1230 EVT SVT = Op.getValueType();
1231 EVT RVT = N->getValueType(ResNo: 0);
1232 EVT NVT = EVT();
1233 SDLoc dl(N);
1234
1235 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1236 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1237 // match, eg. we don't have fp -> i8 conversions.
1238 // Look for an appropriate libcall.
1239 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1240 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1241 "Unsupported FP_TO_XINT!");
1242
1243 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1244 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1245 if (LCImpl == RTLIB::Unsupported) {
1246 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
1247 N->getOperationName(G: &DAG));
1248 SDValue Poison = DAG.getPOISON(VT: RVT);
1249 if (IsStrict) {
1250 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1251 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1252 return SDValue();
1253 }
1254 return Poison;
1255 }
1256
1257 Op = GetSoftenedFloat(Op);
1258 TargetLowering::MakeLibCallOptions CallOptions;
1259 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1260 std::pair<SDValue, SDValue> Tmp =
1261 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
1262
1263 // Truncate the result if the libcall returns a larger type.
1264 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1265
1266 if (!IsStrict)
1267 return Res;
1268
1269 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1270 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1271 return SDValue();
1272}
1273
1274SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1275 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1276 return Res;
1277}
1278
1279SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1280 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1281 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1282
1283 EVT VT = NewLHS.getValueType();
1284 NewLHS = GetSoftenedFloat(Op: NewLHS);
1285 NewRHS = GetSoftenedFloat(Op: NewRHS);
1286 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1287 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1288
1289 // If softenSetCCOperands returned a scalar, we need to compare the result
1290 // against zero to select between true and false values.
1291 if (!NewRHS.getNode()) {
1292 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1293 CCCode = ISD::SETNE;
1294 }
1295
1296 // Update N to have the operands specified.
1297 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1298 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1299 Op5: DAG.getCondCode(Cond: CCCode)),
1300 0);
1301}
1302
1303SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1304 bool IsStrict = N->isStrictFPOpcode();
1305 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1306 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1307 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1308 ISD::CondCode CCCode =
1309 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1310
1311 EVT VT = Op0.getValueType();
1312 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1313 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1314 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1315 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1316
1317 // Update N to have the operands specified.
1318 if (NewRHS.getNode()) {
1319 if (IsStrict)
1320 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1321 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1322 else
1323 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1324 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1325 }
1326
1327 // Otherwise, softenSetCCOperands returned a scalar, use it.
1328 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1329 "Unexpected setcc expansion!");
1330
1331 if (IsStrict) {
1332 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1333 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1334 return SDValue();
1335 }
1336 return NewLHS;
1337}
1338
1339SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1340 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1341 assert(OpNo == 1 && "Can only soften the stored value!");
1342 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1343 SDValue Val = ST->getValue();
1344 SDLoc dl(N);
1345
1346 if (ST->isTruncatingStore())
1347 // Do an FP_ROUND followed by a non-truncating store.
1348 Val = BitConvertToInteger(
1349 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1350 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1351 else
1352 Val = GetSoftenedFloat(Op: Val);
1353
1354 // If softening widens the integer representation (e.g. x86_fp80 -> i96),
1355 // truncate the value before storing to preserve the original memory width.
1356 EVT MemVT =
1357 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ST->getMemoryVT().getSizeInBits());
1358 return DAG.getTruncStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(), SVT: MemVT,
1359 MMO: ST->getMemOperand());
1360}
1361
1362SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1363 assert(OpNo == 1 && "Can only soften the stored value!");
1364 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
1365 SDValue Val = ST->getVal();
1366 EVT VT = Val.getValueType();
1367 SDLoc dl(N);
1368
1369 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1370
1371 SDValue NewVal = GetSoftenedFloat(Op: Val);
1372 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: VT, Chain: ST->getChain(), Ptr: NewVal,
1373 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
1374}
1375
1376SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1377 SDValue LHS = N->getOperand(Num: 0);
1378 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1379 SDLoc dl(N);
1380
1381 EVT LVT = LHS.getValueType();
1382 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1383 EVT RVT = RHS.getValueType();
1384
1385 unsigned LSize = LVT.getSizeInBits();
1386 unsigned RSize = RVT.getSizeInBits();
1387
1388 // Shift right or sign-extend it if the two operands have different types.
1389 int SizeDiff = RSize - LSize;
1390 if (SizeDiff > 0) {
1391 RHS =
1392 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1393 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1394 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1395 DL: DAG.getDataLayout())));
1396 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1397 } else if (SizeDiff < 0) {
1398 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1399 RHS =
1400 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1401 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1402 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1403 DL: DAG.getDataLayout())));
1404 }
1405
1406 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1407 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1408}
1409
1410SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1411 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1412 bool IsStrict = N->isStrictFPOpcode();
1413 unsigned Offset = IsStrict ? 1 : 0;
1414 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1415 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1416 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1417 if (LCImpl == RTLIB::Unsupported) {
1418 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
1419 N->getOperationName(G: &DAG));
1420 SDValue Poison = DAG.getPOISON(VT: N->getValueType(ResNo: 0));
1421 if (IsStrict) {
1422 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1423 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1424 return SDValue();
1425 }
1426 return Poison;
1427 }
1428 TargetLowering::MakeLibCallOptions CallOptions;
1429 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1430 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
1431 std::pair<SDValue, SDValue> Tmp =
1432 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1433 if (IsStrict) {
1434 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1435 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1436 return SDValue();
1437 }
1438
1439 return Tmp.first;
1440}
1441
1442SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1443 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1444 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1445 Call_F32: RTLIB::LROUND_F32,
1446 Call_F64: RTLIB::LROUND_F64,
1447 Call_F80: RTLIB::LROUND_F80,
1448 Call_F128: RTLIB::LROUND_F128,
1449 Call_PPCF128: RTLIB::LROUND_PPCF128));
1450}
1451
1452SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1453 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1454 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1455 Call_F32: RTLIB::LLROUND_F32,
1456 Call_F64: RTLIB::LLROUND_F64,
1457 Call_F80: RTLIB::LLROUND_F80,
1458 Call_F128: RTLIB::LLROUND_F128,
1459 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1460}
1461
1462SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1463 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1464 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1465 Call_F32: RTLIB::LRINT_F32,
1466 Call_F64: RTLIB::LRINT_F64,
1467 Call_F80: RTLIB::LRINT_F80,
1468 Call_F128: RTLIB::LRINT_F128,
1469 Call_PPCF128: RTLIB::LRINT_PPCF128));
1470}
1471
1472SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1473 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1474 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1475 Call_F32: RTLIB::LLRINT_F32,
1476 Call_F64: RTLIB::LLRINT_F64,
1477 Call_F80: RTLIB::LLRINT_F80,
1478 Call_F128: RTLIB::LLRINT_F128,
1479 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1480}
1481
1482SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1483 SDValue Op1 = BitConvertToInteger(Op: N->getOperand(Num: 1));
1484 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1485 N1: N->getOperand(Num: 0), N2: Op1);
1486}
1487
1488SDValue DAGTypeLegalizer::SoftenFloatOp_STACKMAP(SDNode *N, unsigned OpNo) {
1489 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
1490 SmallVector<SDValue> NewOps(N->ops());
1491 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1492 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1493}
1494
1495SDValue DAGTypeLegalizer::SoftenFloatOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
1496 assert(OpNo >= 7);
1497 SmallVector<SDValue> NewOps(N->ops());
1498 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1499 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1500}
1501
1502//===----------------------------------------------------------------------===//
1503// Float Result Expansion
1504//===----------------------------------------------------------------------===//
1505
1506/// ExpandFloatResult - This method is called when the specified result of the
1507/// specified node is found to need expansion. At this point, the node may also
1508/// have invalid operands or may have other results that need promotion, we just
1509/// know that (at least) one result needs expansion.
1510void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1511 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1512 SDValue Lo, Hi;
1513 Lo = Hi = SDValue();
1514
1515 // See if the target wants to custom expand this node.
1516 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1517 return;
1518
1519 switch (N->getOpcode()) {
1520 default:
1521#ifndef NDEBUG
1522 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1523 N->dump(&DAG); dbgs() << "\n";
1524#endif
1525 report_fatal_error(reason: "Do not know how to expand the result of this "
1526 "operator!");
1527 // clang-format off
1528 case ISD::POISON:
1529 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1530 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1531 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1532
1533 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1534 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1535 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1536 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1537 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1538 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1539
1540 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1541 case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
1542 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1543 case ISD::STRICT_FMINNUM:
1544 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1545 case ISD::STRICT_FMAXNUM:
1546 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1547 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1548 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1549 case ISD::STRICT_FADD:
1550 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1551 case ISD::STRICT_FACOS:
1552 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1553 case ISD::STRICT_FASIN:
1554 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1555 case ISD::STRICT_FATAN:
1556 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1557 case ISD::STRICT_FATAN2:
1558 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1559 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1560 case ISD::STRICT_FCEIL:
1561 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1562 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1563 case ISD::STRICT_FCOS:
1564 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1565 case ISD::STRICT_FCOSH:
1566 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1567 case ISD::STRICT_FDIV:
1568 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1569 case ISD::STRICT_FEXP:
1570 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1571 case ISD::STRICT_FEXP2:
1572 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1573 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1574 case ISD::STRICT_FFLOOR:
1575 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1576 case ISD::STRICT_FLOG:
1577 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1578 case ISD::STRICT_FLOG2:
1579 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1580 case ISD::STRICT_FLOG10:
1581 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1582 case ISD::STRICT_FMA:
1583 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1584 case ISD::STRICT_FMUL:
1585 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1586 case ISD::STRICT_FNEARBYINT:
1587 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1588 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1589 case ISD::STRICT_FP_EXTEND:
1590 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1591 case ISD::STRICT_FPOW:
1592 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1593 case ISD::STRICT_FPOWI:
1594 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1595 case ISD::FLDEXP:
1596 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1597 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1598 case ISD::STRICT_FRINT:
1599 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1600 case ISD::STRICT_FROUND:
1601 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1602 case ISD::STRICT_FROUNDEVEN:
1603 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1604 case ISD::STRICT_FSIN:
1605 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1606 case ISD::STRICT_FSINH:
1607 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1608 case ISD::STRICT_FSQRT:
1609 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1610 case ISD::STRICT_FSUB:
1611 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1612 case ISD::STRICT_FTAN:
1613 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1614 case ISD::STRICT_FTANH:
1615 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1616 case ISD::STRICT_FTRUNC:
1617 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1618 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1619 case ISD::STRICT_SINT_TO_FP:
1620 case ISD::STRICT_UINT_TO_FP:
1621 case ISD::SINT_TO_FP:
1622 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1623 case ISD::STRICT_FREM:
1624 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1625 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1626 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1627 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1628 // clang-format on
1629 }
1630
1631 // If Lo/Hi is null, the sub-method took care of registering results etc.
1632 if (Lo.getNode())
1633 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1634}
1635
1636void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1637 SDValue &Hi) {
1638 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1639 assert(NVT.getSizeInBits() == 64 &&
1640 "Do not know how to expand this float constant!");
1641 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1642 SDLoc dl(N);
1643 const fltSemantics &Sem = NVT.getFltSemantics();
1644 Lo = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 64)), DL: dl, VT: NVT);
1645 Hi = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 0)), DL: dl, VT: NVT);
1646}
1647
1648void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1649 SDValue &Lo, SDValue &Hi) {
1650 bool IsStrict = N->isStrictFPOpcode();
1651 unsigned Offset = IsStrict ? 1 : 0;
1652 SDValue Op = N->getOperand(Num: 0 + Offset);
1653 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1654 TargetLowering::MakeLibCallOptions CallOptions;
1655 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1656 Ops: Op, CallOptions, dl: SDLoc(N),
1657 Chain);
1658 if (IsStrict)
1659 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1660 GetPairElements(Pair: Tmp.first, Lo, Hi);
1661}
1662
1663void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1664 SDValue &Lo, SDValue &Hi) {
1665 bool IsStrict = N->isStrictFPOpcode();
1666 unsigned Offset = IsStrict ? 1 : 0;
1667 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1668 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1669 TargetLowering::MakeLibCallOptions CallOptions;
1670 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1671 Ops, CallOptions, dl: SDLoc(N),
1672 Chain);
1673 if (IsStrict)
1674 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1675 GetPairElements(Pair: Tmp.first, Lo, Hi);
1676}
1677
1678void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1679 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT: N->getValueType(ResNo: 0)),
1680 /*CallRetResNo=*/0);
1681}
1682
1683void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1684 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(VT: N->getValueType(ResNo: 0)));
1685}
1686
1687void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1688 ExpandFloatRes_UnaryWithTwoFPResults(N,
1689 LC: RTLIB::getSINCOSPI(VT: N->getValueType(ResNo: 0)));
1690}
1691
1692void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1693 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1694 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1695 SmallVector<SDValue> Results;
1696 TLI.expandMultipleResultFPLibCall(DAG, LC, Node: N, Results, CallRetResNo);
1697 for (auto [ResNo, Res] : enumerate(First&: Results)) {
1698 SDValue Lo, Hi;
1699 GetPairElements(Pair: Res, Lo, Hi);
1700 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1701 }
1702}
1703
1704void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1705 SDValue &Hi) {
1706 assert(N->getValueType(0) == MVT::ppcf128 &&
1707 "Logic only correct for ppcf128!");
1708 SDLoc dl(N);
1709 SDValue Tmp;
1710 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1711 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1712 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1713 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1714 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1715 Cond: ISD::SETEQ);
1716}
1717
1718void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1719 SDValue &Hi) {
1720 ExpandFloatRes_Binary(N, LC: RTLIB::getFMIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1721}
1722
1723void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1724 SDValue &Hi) {
1725 ExpandFloatRes_Binary(N, LC: RTLIB::getFMAX(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1726}
1727
1728void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1729 SDValue &Hi) {
1730 ExpandFloatRes_Binary(N, LC: RTLIB::getFMINIMUM_NUM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1731}
1732
1733void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1734 SDValue &Hi) {
1735 ExpandFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM_NUM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1736}
1737
1738void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1739 SDValue &Hi) {
1740 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1741 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1742 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1743 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1744}
1745
1746void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1747 SDValue &Hi) {
1748 ExpandFloatRes_Unary(N, LC: RTLIB::getACOS(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1749}
1750
1751void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1752 SDValue &Hi) {
1753 ExpandFloatRes_Unary(N, LC: RTLIB::getASIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1754}
1755
1756void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1757 SDValue &Hi) {
1758 ExpandFloatRes_Unary(N, LC: RTLIB::getATAN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1759}
1760
1761void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1762 SDValue &Hi) {
1763 ExpandFloatRes_Binary(N, LC: RTLIB::getATAN2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1764}
1765
1766void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1767 SDValue &Hi) {
1768 ExpandFloatRes_Unary(N, LC: RTLIB::getCBRT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1769}
1770
1771void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1772 SDValue &Lo, SDValue &Hi) {
1773 ExpandFloatRes_Unary(N, LC: RTLIB::getCEIL(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1774}
1775
1776void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1777 SDValue &Lo, SDValue &Hi) {
1778 ExpandFloatRes_Binary(N, LC: RTLIB::getCOPYSIGN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1779}
1780
1781void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1782 SDValue &Lo, SDValue &Hi) {
1783 ExpandFloatRes_Unary(N, LC: RTLIB::getCOS(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1784}
1785
1786void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1787 SDValue &Hi) {
1788 ExpandFloatRes_Unary(N, LC: RTLIB::getCOSH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1789}
1790
1791void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1792 SDValue &Hi) {
1793 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1794 Call_F32: RTLIB::DIV_F32,
1795 Call_F64: RTLIB::DIV_F64,
1796 Call_F80: RTLIB::DIV_F80,
1797 Call_F128: RTLIB::DIV_F128,
1798 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1799}
1800
1801void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1802 SDValue &Lo, SDValue &Hi) {
1803 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1804}
1805
1806void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1807 SDValue &Lo, SDValue &Hi) {
1808 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1809}
1810
1811void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1812 SDValue &Hi) {
1813 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP10(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1814}
1815
1816void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1817 SDValue &Lo, SDValue &Hi) {
1818 ExpandFloatRes_Unary(N, LC: RTLIB::getFLOOR(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1819}
1820
1821void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1822 SDValue &Lo, SDValue &Hi) {
1823 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1824}
1825
1826void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1827 SDValue &Lo, SDValue &Hi) {
1828 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1829}
1830
1831void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1832 SDValue &Lo, SDValue &Hi) {
1833 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG10(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1834}
1835
1836void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1837 SDValue &Hi) {
1838 bool IsStrict = N->isStrictFPOpcode();
1839 unsigned Offset = IsStrict ? 1 : 0;
1840 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1841 N->getOperand(Num: 2 + Offset) };
1842 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1843 TargetLowering::MakeLibCallOptions CallOptions;
1844 std::pair<SDValue, SDValue> Tmp =
1845 TLI.makeLibCall(DAG, LC: RTLIB::getFMA(VT: N->getValueType(ResNo: 0)),
1846 RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain);
1847 if (IsStrict)
1848 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1849 GetPairElements(Pair: Tmp.first, Lo, Hi);
1850}
1851
1852void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1853 SDValue &Hi) {
1854 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1855 Call_F32: RTLIB::MUL_F32,
1856 Call_F64: RTLIB::MUL_F64,
1857 Call_F80: RTLIB::MUL_F80,
1858 Call_F128: RTLIB::MUL_F128,
1859 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1860}
1861
1862void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1863 SDValue &Lo, SDValue &Hi) {
1864 ExpandFloatRes_Unary(N, LC: RTLIB::getNEARBYINT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1865}
1866
1867void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1868 SDValue &Hi) {
1869 SDLoc dl(N);
1870 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1871 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1872 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1873}
1874
1875void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
1876 SDValue &Hi) {
1877 // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
1878 SDLoc dl(N);
1879 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1880}
1881
1882void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1883 SDValue &Hi) {
1884 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1885 SDLoc dl(N);
1886 bool IsStrict = N->isStrictFPOpcode();
1887
1888 SDValue Chain;
1889 if (IsStrict) {
1890 // If the expanded type is the same as the input type, just bypass the node.
1891 if (NVT == N->getOperand(Num: 1).getValueType()) {
1892 Hi = N->getOperand(Num: 1);
1893 Chain = N->getOperand(Num: 0);
1894 } else {
1895 // Other we need to extend.
1896 Hi = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: { NVT, MVT::Other },
1897 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
1898 Chain = Hi.getValue(R: 1);
1899 }
1900 } else {
1901 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1902 }
1903
1904 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
1905
1906 if (IsStrict)
1907 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1908}
1909
1910void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1911 SDValue &Lo, SDValue &Hi) {
1912 ExpandFloatRes_Binary(N, LC: RTLIB::getPOW(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1913}
1914
1915void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1916 SDValue &Lo, SDValue &Hi) {
1917 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1918}
1919
1920void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1921 SDValue &Hi) {
1922 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1923}
1924
1925void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1926 SDValue &Lo, SDValue &Hi) {
1927 assert(N->getValueType(0) == MVT::ppcf128 &&
1928 "Logic only correct for ppcf128!");
1929
1930 SDLoc dl(N);
1931 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1932 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1933 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1934}
1935
1936void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1937 SDValue &Lo, SDValue &Hi) {
1938 ExpandFloatRes_Binary(N, LC: RTLIB::getREM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1939}
1940
1941void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1942 SDValue &Lo, SDValue &Hi) {
1943 ExpandFloatRes_Unary(N, LC: RTLIB::getRINT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1944}
1945
1946void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1947 SDValue &Lo, SDValue &Hi) {
1948 ExpandFloatRes_Unary(N, LC: RTLIB::getROUND(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1949}
1950
1951void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1952 SDValue &Lo, SDValue &Hi) {
1953 ExpandFloatRes_Unary(N, LC: RTLIB::getROUNDEVEN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1954}
1955
1956void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1957 SDValue &Lo, SDValue &Hi) {
1958 ExpandFloatRes_Unary(N, LC: RTLIB::getSIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1959}
1960
1961void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
1962 SDValue &Hi) {
1963 ExpandFloatRes_Unary(N, LC: RTLIB::getSINH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1964}
1965
1966void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1967 SDValue &Lo, SDValue &Hi) {
1968 ExpandFloatRes_Unary(N, LC: RTLIB::getSQRT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1969}
1970
1971void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1972 SDValue &Hi) {
1973 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1974 Call_F32: RTLIB::SUB_F32,
1975 Call_F64: RTLIB::SUB_F64,
1976 Call_F80: RTLIB::SUB_F80,
1977 Call_F128: RTLIB::SUB_F128,
1978 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
1979}
1980
1981void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
1982 SDValue &Hi) {
1983 ExpandFloatRes_Unary(N, LC: RTLIB::getTAN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1984}
1985
1986void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
1987 SDValue &Hi) {
1988 ExpandFloatRes_Unary(N, LC: RTLIB::getTANH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1989}
1990
1991void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1992 SDValue &Lo, SDValue &Hi) {
1993 ExpandFloatRes_Unary(N, LC: RTLIB::getTRUNC(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1994}
1995
1996void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1997 SDValue &Hi) {
1998 if (ISD::isNormalLoad(N)) {
1999 ExpandRes_NormalLoad(N, Lo, Hi);
2000 return;
2001 }
2002
2003 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2004 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
2005 SDValue Chain = LD->getChain();
2006 SDValue Ptr = LD->getBasePtr();
2007 SDLoc dl(N);
2008
2009 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
2010 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2011 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2012
2013 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
2014 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
2015
2016 // Remember the chain.
2017 Chain = Hi.getValue(R: 1);
2018
2019 // The low part is zero.
2020 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2021
2022 // Modified the chain - switch anything that used the old chain to use the
2023 // new one.
2024 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
2025}
2026
2027void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2028 SDValue &Hi) {
2029 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2030 EVT VT = N->getValueType(ResNo: 0);
2031 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2032 bool Strict = N->isStrictFPOpcode();
2033 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
2034 EVT SrcVT = Src.getValueType();
2035 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2036 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2037 SDLoc dl(N);
2038 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
2039
2040 // TODO: Any other flags to propagate?
2041 SDNodeFlags Flags;
2042 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2043
2044 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2045 // When promoting partial word types to i32 we must honor the signedness,
2046 // though.
2047 if (SrcVT.bitsLE(VT: MVT::i32)) {
2048 // The integer can be represented exactly in an f64.
2049 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2050 if (Strict) {
2051 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
2052 Ops: {Chain, Src}, Flags);
2053 Chain = Hi.getValue(R: 1);
2054 } else
2055 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
2056 } else {
2057 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2058 if (SrcVT.bitsLE(VT: MVT::i64)) {
2059 Src = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
2060 VT: MVT::i64, Operand: Src);
2061 LC = RTLIB::SINTTOFP_I64_PPCF128;
2062 } else if (SrcVT.bitsLE(VT: MVT::i128)) {
2063 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i128, Operand: Src);
2064 LC = RTLIB::SINTTOFP_I128_PPCF128;
2065 }
2066 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2067
2068 TargetLowering::MakeLibCallOptions CallOptions;
2069 CallOptions.setIsSigned(true);
2070 std::pair<SDValue, SDValue> Tmp =
2071 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
2072 if (Strict)
2073 Chain = Tmp.second;
2074 GetPairElements(Pair: Tmp.first, Lo, Hi);
2075 }
2076
2077 // No need to complement for unsigned 32-bit integers
2078 if (isSigned || SrcVT.bitsLE(VT: MVT::i32)) {
2079 if (Strict)
2080 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2081
2082 return;
2083 }
2084
2085 // Unsigned - fix up the SINT_TO_FP value just calculated.
2086 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2087 // keep semantics correctness if the integer is not exactly representable
2088 // here. See ExpandLegalINT_TO_FP.
2089 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
2090 SrcVT = Src.getValueType();
2091
2092 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2093 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2094 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2095 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2096 ArrayRef<uint64_t> Parts;
2097
2098 switch (SrcVT.getSimpleVT().SimpleTy) {
2099 default:
2100 llvm_unreachable("Unsupported UINT_TO_FP!");
2101 case MVT::i32:
2102 Parts = TwoE32;
2103 break;
2104 case MVT::i64:
2105 Parts = TwoE64;
2106 break;
2107 case MVT::i128:
2108 Parts = TwoE128;
2109 break;
2110 }
2111
2112 // TODO: Are there other fast-math-flags to propagate to this FADD?
2113 SDValue NewLo = DAG.getConstantFP(
2114 Val: APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), DL: dl, VT: MVT::ppcf128);
2115 if (Strict) {
2116 Lo = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other),
2117 Ops: {Chain, Hi, NewLo}, Flags);
2118 Chain = Lo.getValue(R: 1);
2119 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2120 } else
2121 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
2122 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
2123 True: Lo, False: Hi, Cond: ISD::SETLT);
2124 GetPairElements(Pair: Lo, Lo, Hi);
2125}
2126
2127
2128//===----------------------------------------------------------------------===//
2129// Float Operand Expansion
2130//===----------------------------------------------------------------------===//
2131
2132/// ExpandFloatOperand - This method is called when the specified operand of the
2133/// specified node is found to need expansion. At this point, all of the result
2134/// types of the node are known to be legal, but other operands of the node may
2135/// need promotion or expansion as well as the specified one.
2136bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2137 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2138 SDValue Res = SDValue();
2139
2140 // See if the target wants to custom expand this node.
2141 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2142 return false;
2143
2144 switch (N->getOpcode()) {
2145 default:
2146#ifndef NDEBUG
2147 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2148 N->dump(&DAG); dbgs() << "\n";
2149#endif
2150 report_fatal_error(reason: "Do not know how to expand this operator's operand!");
2151
2152 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2153 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2154 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2155
2156 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2157 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2158 case ISD::STRICT_FP_ROUND:
2159 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2160 case ISD::STRICT_FP_TO_SINT:
2161 case ISD::STRICT_FP_TO_UINT:
2162 case ISD::FP_TO_SINT:
2163 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2164 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2165 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2166 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2167 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2168 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2169 case ISD::STRICT_FSETCC:
2170 case ISD::STRICT_FSETCCS:
2171 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2172 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
2173 OpNo); break;
2174 }
2175
2176 // If the result is null, the sub-method took care of registering results etc.
2177 if (!Res.getNode()) return false;
2178
2179 // If the result is N, the sub-method updated N in place. Tell the legalizer
2180 // core about this.
2181 if (Res.getNode() == N)
2182 return true;
2183
2184 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2185 "Invalid operand expansion");
2186
2187 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2188 return false;
2189}
2190
2191/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2192/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2193void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2194 SDValue &NewRHS,
2195 ISD::CondCode &CCCode,
2196 const SDLoc &dl, SDValue &Chain,
2197 bool IsSignaling) {
2198 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2199 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
2200 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
2201
2202 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2203
2204 // FIXME: This generated code sucks. We want to generate
2205 // FCMPU crN, hi1, hi2
2206 // BNE crN, L:
2207 // FCMPU crN, lo1, lo2
2208 // The following can be improved, but not that much.
2209 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2210 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2211 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
2212 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2213 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
2214 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
2215 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2216 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2217 Tmp1 =
2218 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
2219 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
2220 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2221 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2222 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
2223 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2224 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2225 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
2226 NewRHS = SDValue(); // LHS is the result, not a compare.
2227 Chain = OutputChain;
2228}
2229
2230SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2231 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
2232 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
2233 SDValue Chain;
2234 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2235
2236 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2237 // against zero to select between true and false values.
2238 if (!NewRHS.getNode()) {
2239 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2240 CCCode = ISD::SETNE;
2241 }
2242
2243 // Update N to have the operands specified.
2244 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2245 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
2246 Op5: N->getOperand(Num: 4)), 0);
2247}
2248
2249SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2250 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2251 "Logic only correct for ppcf128!");
2252 SDValue Lo, Hi;
2253 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2254 // The ppcf128 value is providing only the sign; take it from the
2255 // higher-order double (which must have the larger magnitude).
2256 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2257 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2258}
2259
2260SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2261 bool IsStrict = N->isStrictFPOpcode();
2262 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2263 "Logic only correct for ppcf128!");
2264 SDValue Lo, Hi;
2265 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2266
2267 if (!IsStrict)
2268 // Round it the rest of the way (e.g. to f32) if needed.
2269 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2270 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2271
2272 // Eliminate the node if the input float type is the same as the output float
2273 // type.
2274 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2275 // Connect the output chain to the input chain, unlinking the node.
2276 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2277 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2278 return SDValue();
2279 }
2280
2281 SDValue Expansion = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
2282 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
2283 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
2284 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2285 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2286 return SDValue();
2287}
2288
2289SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2290 EVT RVT = N->getValueType(ResNo: 0);
2291 SDLoc dl(N);
2292
2293 bool IsStrict = N->isStrictFPOpcode();
2294 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2295 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2296 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2297 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2298
2299 EVT NVT;
2300 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2301 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2302 "Unsupported FP_TO_XINT!");
2303 TargetLowering::MakeLibCallOptions CallOptions;
2304 std::pair<SDValue, SDValue> Tmp =
2305 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2306 if (!IsStrict)
2307 return Tmp.first;
2308
2309 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2310 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2311 return SDValue();
2312}
2313
2314SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2315 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2316 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2317 SDValue Chain;
2318 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2319
2320 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2321 // against zero to select between true and false values.
2322 if (!NewRHS.getNode()) {
2323 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2324 CCCode = ISD::SETNE;
2325 }
2326
2327 // Update N to have the operands specified.
2328 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2329 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2330 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2331}
2332
2333SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2334 bool IsStrict = N->isStrictFPOpcode();
2335 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2336 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2337 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2338 ISD::CondCode CCCode =
2339 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2340 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2341 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2342
2343 // FloatExpandSetCCOperands always returned a scalar.
2344 assert(!NewRHS.getNode() && "Expect to return scalar");
2345 assert(NewLHS.getValueType() == N->getValueType(0) &&
2346 "Unexpected setcc expansion!");
2347 if (Chain) {
2348 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2349 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2350 return SDValue();
2351 }
2352 return NewLHS;
2353}
2354
2355SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2356 if (ISD::isNormalStore(N))
2357 return ExpandOp_NormalStore(N, OpNo);
2358
2359 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2360 assert(OpNo == 1 && "Can only expand the stored value so far");
2361 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2362
2363 SDValue Chain = ST->getChain();
2364 SDValue Ptr = ST->getBasePtr();
2365
2366 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2367 VT: ST->getValue().getValueType());
2368 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2369 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2370 (void)NVT;
2371
2372 SDValue Lo, Hi;
2373 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2374
2375 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2376 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2377}
2378
2379SDValue DAGTypeLegalizer::ExpandFloatOp_XRINT_XROUND(SDNode *N,
2380 RTLIB::Libcall LC) {
2381 EVT RVT = N->getValueType(ResNo: 0);
2382 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2383 if (LCImpl == RTLIB::Unsupported) {
2384 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2385 N->getOperationName(G: &DAG));
2386 return DAG.getPOISON(VT: RVT);
2387 }
2388
2389 TargetLowering::MakeLibCallOptions CallOptions;
2390 return TLI
2391 .makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N))
2392 .first;
2393}
2394
2395SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2396 EVT RetVT = N->getOperand(Num: 0).getValueType();
2397 return ExpandFloatOp_XRINT_XROUND(
2398 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LROUND_F32, Call_F64: RTLIB::LROUND_F64,
2399 Call_F80: RTLIB::LROUND_F80, Call_F128: RTLIB::LROUND_F128,
2400 Call_PPCF128: RTLIB::LROUND_PPCF128));
2401}
2402
2403SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2404 EVT RetVT = N->getOperand(Num: 0).getValueType();
2405 return ExpandFloatOp_XRINT_XROUND(
2406 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LLROUND_F32, Call_F64: RTLIB::LLROUND_F64,
2407 Call_F80: RTLIB::LLROUND_F80, Call_F128: RTLIB::LLROUND_F128,
2408 Call_PPCF128: RTLIB::LLROUND_PPCF128));
2409}
2410
2411SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2412 EVT RetVT = N->getOperand(Num: 0).getValueType();
2413 return ExpandFloatOp_XRINT_XROUND(
2414 N,
2415 LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LRINT_F32, Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
2416 Call_F128: RTLIB::LRINT_F128, Call_PPCF128: RTLIB::LRINT_PPCF128));
2417}
2418
2419SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2420 EVT RetVT = N->getOperand(Num: 0).getValueType();
2421 return ExpandFloatOp_XRINT_XROUND(
2422 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LLRINT_F32, Call_F64: RTLIB::LLRINT_F64,
2423 Call_F80: RTLIB::LLRINT_F80, Call_F128: RTLIB::LLRINT_F128,
2424 Call_PPCF128: RTLIB::LLRINT_PPCF128));
2425}
2426
2427//===----------------------------------------------------------------------===//
2428// Float Operand Promotion
2429//===----------------------------------------------------------------------===//
2430//
2431
2432static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2433 if (OpVT == MVT::f16)
2434 return ISD::FP16_TO_FP;
2435 if (RetVT == MVT::f16)
2436 return ISD::FP_TO_FP16;
2437 if (OpVT == MVT::bf16)
2438 return ISD::BF16_TO_FP;
2439 if (RetVT == MVT::bf16)
2440 return ISD::FP_TO_BF16;
2441 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2442}
2443
2444static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2445 if (OpVT == MVT::f16)
2446 return ISD::STRICT_FP16_TO_FP;
2447 if (RetVT == MVT::f16)
2448 return ISD::STRICT_FP_TO_FP16;
2449 if (OpVT == MVT::bf16)
2450 return ISD::STRICT_BF16_TO_FP;
2451 if (RetVT == MVT::bf16)
2452 return ISD::STRICT_FP_TO_BF16;
2453 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2454}
2455
2456SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2457 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2458 SDLoc SL(N);
2459
2460 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2461 EVT CastVT = CastVal.getValueType();
2462
2463 SDValue NewAtomic
2464 = DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl: SL, MemVT: CastVT,
2465 VTList: DAG.getVTList(VT1: CastVT, VT2: MVT::Other),
2466 Ops: { AM->getChain(), AM->getBasePtr(), CastVal },
2467 MMO: AM->getMemOperand());
2468
2469 SDValue Result = NewAtomic;
2470
2471 // Legalize the chain result by replacing uses of the old value chain with the
2472 // new one
2473 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2474
2475 return Result;
2476}
2477
2478//===----------------------------------------------------------------------===//
2479// Half Result Soft Promotion
2480//===----------------------------------------------------------------------===//
2481
2482void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2483 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2484 N->dump(&DAG));
2485 SDValue R = SDValue();
2486
2487 // See if the target wants to custom expand this node.
2488 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2489 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2490 return;
2491 }
2492
2493 switch (N->getOpcode()) {
2494 default:
2495#ifndef NDEBUG
2496 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2497 N->dump(&DAG); dbgs() << "\n";
2498#endif
2499 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2500 "result!");
2501
2502 case ISD::ARITH_FENCE:
2503 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2504 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2505 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2506 case ISD::EXTRACT_VECTOR_ELT:
2507 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2508 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2509 case ISD::STRICT_FP_ROUND:
2510 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2511
2512 // Unary FP Operations
2513 case ISD::FACOS:
2514 case ISD::FASIN:
2515 case ISD::FATAN:
2516 case ISD::FCBRT:
2517 case ISD::FCEIL:
2518 case ISD::FCOS:
2519 case ISD::FCOSH:
2520 case ISD::FEXP:
2521 case ISD::FEXP2:
2522 case ISD::FEXP10:
2523 case ISD::FFLOOR:
2524 case ISD::FLOG:
2525 case ISD::FLOG2:
2526 case ISD::FLOG10:
2527 case ISD::FNEARBYINT:
2528 case ISD::FREEZE:
2529 case ISD::FRINT:
2530 case ISD::FROUND:
2531 case ISD::FROUNDEVEN:
2532 case ISD::FSIN:
2533 case ISD::FSINH:
2534 case ISD::FSQRT:
2535 case ISD::FTRUNC:
2536 case ISD::FTAN:
2537 case ISD::FTANH:
2538 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2539 case ISD::FABS:
2540 R = SoftPromoteHalfRes_FABS(N);
2541 break;
2542 case ISD::FNEG:
2543 R = SoftPromoteHalfRes_FNEG(N);
2544 break;
2545 case ISD::AssertNoFPClass:
2546 R = SoftPromoteHalfRes_AssertNoFPClass(N);
2547 break;
2548
2549 // Binary FP Operations
2550 case ISD::FADD:
2551 case ISD::FDIV:
2552 case ISD::FMAXIMUM:
2553 case ISD::FMINIMUM:
2554 case ISD::FMAXIMUMNUM:
2555 case ISD::FMINIMUMNUM:
2556 case ISD::FMAXNUM:
2557 case ISD::FMINNUM:
2558 case ISD::FMUL:
2559 case ISD::FPOW:
2560 case ISD::FATAN2:
2561 case ISD::FREM:
2562 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2563
2564 case ISD::FMA: // FMA is same as FMAD
2565 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2566
2567 case ISD::FPOWI:
2568 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2569
2570 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2571
2572 case ISD::FMODF:
2573 case ISD::FSINCOS:
2574 case ISD::FSINCOSPI:
2575 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
2576 break;
2577
2578 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2579 case ISD::ATOMIC_LOAD:
2580 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2581 break;
2582 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2583 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2584 case ISD::STRICT_SINT_TO_FP:
2585 case ISD::STRICT_UINT_TO_FP:
2586 case ISD::SINT_TO_FP:
2587 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2588 case ISD::CONVERT_FROM_ARBITRARY_FP:
2589 R = SoftPromoteHalfRes_CONVERT_FROM_ARBITRARY_FP(N);
2590 break;
2591 case ISD::POISON:
2592 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2593 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2594 case ISD::VECREDUCE_FADD:
2595 case ISD::VECREDUCE_FMUL:
2596 case ISD::VECREDUCE_FMIN:
2597 case ISD::VECREDUCE_FMAX:
2598 case ISD::VECREDUCE_FMAXIMUM:
2599 case ISD::VECREDUCE_FMINIMUM:
2600 case ISD::VECREDUCE_FMAXIMUMNUM:
2601 case ISD::VECREDUCE_FMINIMUMNUM:
2602 R = SoftPromoteHalfRes_VECREDUCE(N);
2603 break;
2604 case ISD::VECREDUCE_SEQ_FADD:
2605 case ISD::VECREDUCE_SEQ_FMUL:
2606 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2607 break;
2608 }
2609
2610 if (R.getNode())
2611 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
2612}
2613
2614SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2615 return DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: MVT::i16,
2616 Operand: BitConvertToInteger(Op: N->getOperand(Num: 0)));
2617}
2618
2619SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2620 return BitConvertToInteger(Op: N->getOperand(Num: 0));
2621}
2622
2623SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2624 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
2625
2626 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2627 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
2628 VT: MVT::i16);
2629}
2630
2631SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2632 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2633 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
2634 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
2635 N2: N->getOperand(Num: 1));
2636}
2637
2638SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2639 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2640 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
2641 SDLoc dl(N);
2642
2643 EVT LVT = LHS.getValueType();
2644 EVT RVT = RHS.getValueType();
2645
2646 unsigned LSize = LVT.getSizeInBits();
2647 unsigned RSize = RVT.getSizeInBits();
2648
2649 // First get the sign bit of second operand.
2650 SDValue SignBit = DAG.getNode(
2651 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
2652 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
2653 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
2654 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
2655
2656 // Shift right or sign-extend it if the two operands have different types.
2657 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2658 if (SizeDiff > 0) {
2659 SignBit =
2660 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
2661 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
2662 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2663 DL: DAG.getDataLayout())));
2664 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
2665 } else if (SizeDiff < 0) {
2666 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
2667 SignBit =
2668 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
2669 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
2670 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2671 DL: DAG.getDataLayout())));
2672 }
2673
2674 // Clear the sign bit of the first operand.
2675 SDValue Mask = DAG.getNode(
2676 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
2677 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
2678 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
2679 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
2680 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
2681
2682 // Or the value with the sign bit.
2683 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
2684}
2685
2686SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2687 EVT OVT = N->getValueType(ResNo: 0);
2688 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2689 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2690 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2691 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2692 SDNodeFlags Flags = N->getFlags();
2693 SDLoc dl(N);
2694
2695 // Promote to the larger FP type.
2696 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2697 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2698 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2699 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
2700
2701 SDValue Res;
2702 if (OVT == MVT::f16) {
2703 // If f16 fma is not natively supported, the value must be promoted to an
2704 // f64 (and not to f32!) to prevent double rounding issues.
2705 SDValue A64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op0, Flags);
2706 SDValue B64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op1, Flags);
2707 SDValue C64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op2, Flags);
2708
2709 // Prefer a wide FMA node if available; otherwise expand to mul+add.
2710 SDValue WideRes;
2711 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), MVT::f64)) {
2712 WideRes = DAG.getNode(Opcode: ISD::FMA, DL: dl, VT: MVT::f64, N1: A64, N2: B64, N3: C64, Flags);
2713 } else {
2714 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f64, N1: A64, N2: B64, Flags);
2715 WideRes = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f64, N1: Mul, N2: C64, Flags);
2716 }
2717
2718 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: MVT::f64, RetVT: OVT), DL: dl, VT: MVT::i16,
2719 Operand: WideRes);
2720 }
2721
2722 Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2, Flags);
2723 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2724}
2725
2726SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2727 EVT OVT = N->getValueType(ResNo: 0);
2728 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2729 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2730 SDValue Op1 = N->getOperand(Num: 1);
2731 SDLoc dl(N);
2732
2733 // Promote to the larger FP type.
2734 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
2735
2736 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2737
2738 // Convert back to FP16 as an integer.
2739 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2740}
2741
2742SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2743 EVT OVT = N->getValueType(ResNo: 0);
2744 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2745 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2746 SDLoc dl(N);
2747
2748 // Promote to the larger FP type.
2749 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2750
2751 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
2752 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
2753
2754 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2755
2756 // Convert back to FP16 as an integer.
2757 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2758}
2759
2760SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
2761 EVT OVT = N->getValueType(ResNo: 0);
2762 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2763 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2764 SDLoc dl(N);
2765
2766 // Promote to the larger FP type.
2767 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2768 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Op);
2769
2770 // Convert back to FP16 as an integer.
2771 ISD::NodeType Truncate = GetPromotionOpcode(OpVT: NVT, RetVT: OVT);
2772 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
2773 ++ResNum) {
2774 SDValue Trunc = DAG.getNode(Opcode: Truncate, DL: dl, VT: MVT::i16, Operand: Res.getValue(R: ResNum));
2775 SetSoftPromotedHalf(Op: SDValue(N, ResNum), Result: Trunc);
2776 }
2777
2778 return SDValue();
2779}
2780
2781SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2782 EVT RVT = N->getValueType(ResNo: 0);
2783 bool IsStrict = N->isStrictFPOpcode();
2784 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2785 EVT SVT = Op.getValueType();
2786
2787 // If the input type needs to be softened, do that now so that call lowering
2788 // will see the f16 type.
2789 if (getTypeAction(VT: SVT) == TargetLowering::TypeSoftenFloat) {
2790 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: RVT);
2791 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
2792
2793 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2794 Op = GetSoftenedFloat(Op);
2795 TargetLowering::MakeLibCallOptions CallOptions;
2796 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
2797 std::pair<SDValue, SDValue> Tmp =
2798 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
2799 if (IsStrict)
2800 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2801 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: MVT::i16, Operand: Tmp.first);
2802 }
2803
2804 if (IsStrict) {
2805 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
2806 ResultTys: {MVT::i16, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
2807 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2808 return Res;
2809 }
2810
2811 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: MVT::i16,
2812 Operand: N->getOperand(Num: 0));
2813}
2814
2815SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2816 LoadSDNode *L = cast<LoadSDNode>(Val: N);
2817
2818 // Load the value as an integer value with the same number of bits.
2819 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2820 SDValue NewL =
2821 DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: MVT::i16,
2822 dl: SDLoc(N), Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
2823 PtrInfo: L->getPointerInfo(), MemVT: MVT::i16, Alignment: L->getBaseAlign(),
2824 MMOFlags: L->getMemOperand()->getFlags(), Metadata: L->getAAInfo());
2825 // Legalize the chain result by replacing uses of the old value chain with the
2826 // new one
2827 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
2828 return NewL;
2829}
2830
2831SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
2832 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2833
2834 // Load the value as an integer value with the same number of bits.
2835 SDValue NewL = DAG.getAtomic(
2836 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: MVT::i16, VTList: DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other),
2837 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
2838
2839 // Legalize the chain result by replacing uses of the old value chain with the
2840 // new one
2841 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
2842 return NewL;
2843}
2844
2845SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2846 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2847 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2848 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1, RHS: Op2,
2849 Flags: N->getFlags());
2850}
2851
2852SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2853 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2854 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
2855 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
2856 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
2857 N5: N->getOperand(Num: 4));
2858}
2859
2860SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2861 EVT OVT = N->getValueType(ResNo: 0);
2862 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2863 SDLoc dl(N);
2864
2865 if (N->isStrictFPOpcode()) {
2866 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
2867 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
2868 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: NVT, RetVT: OVT), DL: dl,
2869 ResultTys: {MVT::i16, MVT::Other}, Ops: {Op.getValue(R: 1), Op});
2870 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
2871 return Op;
2872 }
2873
2874 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
2875
2876 // Round the value to the softened type.
2877 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2878}
2879
2880SDValue
2881DAGTypeLegalizer::SoftPromoteHalfRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
2882 EVT OVT = N->getValueType(ResNo: 0);
2883 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2884 SDLoc dl(N);
2885
2886 SDValue Res = DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: dl, VT: NVT,
2887 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
2888
2889 // Round the value to the softened type.
2890 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2891}
2892
2893SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2894 return DAG.getUNDEF(VT: MVT::i16);
2895}
2896
2897SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2898 EVT OVT = N->getValueType(ResNo: 0);
2899 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2900 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2901 SDLoc dl(N);
2902
2903 // Promote to the larger FP type.
2904 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2905
2906 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
2907
2908 // Convert back to FP16 as an integer.
2909 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2910}
2911
2912SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FABS(SDNode *N) {
2913 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2914 SDLoc dl(N);
2915
2916 // Clear the sign bit.
2917 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i16, N1: Op,
2918 N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: MVT::i16));
2919}
2920
2921SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FNEG(SDNode *N) {
2922 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2923 SDLoc dl(N);
2924
2925 // Invert the sign bit.
2926 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i16, N1: Op,
2927 N2: DAG.getConstant(Val: 0x8000, DL: dl, VT: MVT::i16));
2928}
2929
2930SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
2931 return GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2932}
2933
2934SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2935 EVT OVT = N->getValueType(ResNo: 0);
2936 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2937 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2938 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2939 SDLoc dl(N);
2940
2941 // Promote to the larger FP type.
2942 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2943 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2944 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2945
2946 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2947
2948 // Convert back to FP16 as an integer.
2949 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2950}
2951
2952SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2953 // Expand and soften recursively.
2954 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
2955 return SDValue();
2956}
2957
2958SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
2959 // Expand and soften.
2960 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
2961 return SDValue();
2962}
2963
2964//===----------------------------------------------------------------------===//
2965// Half Operand Soft Promotion
2966//===----------------------------------------------------------------------===//
2967
2968bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2969 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2970 N->dump(&DAG));
2971 SDValue Res = SDValue();
2972
2973 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
2974 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2975 return false;
2976 }
2977
2978 // Nodes that use a promotion-requiring floating point operand, but doesn't
2979 // produce a soft promotion-requiring floating point result, need to be
2980 // legalized to use the soft promoted float operand. Nodes that produce at
2981 // least one soft promotion-requiring floating point result have their
2982 // operands legalized as a part of PromoteFloatResult.
2983 switch (N->getOpcode()) {
2984 default:
2985 #ifndef NDEBUG
2986 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2987 N->dump(&DAG); dbgs() << "\n";
2988 #endif
2989 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2990 "operand!");
2991
2992 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2993 case ISD::BUILD_VECTOR:
2994 Res = SoftPromoteHalfOp_BUILD_VECTOR(N);
2995 break;
2996 case ISD::FAKE_USE:
2997 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
2998 break;
2999 case ISD::FCOPYSIGN:
3000 Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo);
3001 break;
3002 case ISD::FP_TO_SINT:
3003 case ISD::FP_TO_UINT:
3004 case ISD::STRICT_FP_TO_SINT:
3005 case ISD::STRICT_FP_TO_UINT:
3006 case ISD::LLRINT:
3007 case ISD::LLROUND:
3008 case ISD::LRINT:
3009 case ISD::LROUND:
3010 case ISD::STRICT_LLRINT:
3011 case ISD::STRICT_LLROUND:
3012 case ISD::STRICT_LRINT:
3013 case ISD::STRICT_LROUND:
3014 Res = SoftPromoteHalfOp_Op0WithStrict(N);
3015 break;
3016 case ISD::FP_TO_SINT_SAT:
3017 case ISD::FP_TO_UINT_SAT:
3018 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3019 case ISD::CONVERT_TO_ARBITRARY_FP:
3020 Res = SoftPromoteHalfOp_CONVERT_TO_ARBITRARY_FP(N);
3021 break;
3022 case ISD::STRICT_FP_EXTEND:
3023 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3024 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3025 case ISD::BR_CC:
3026 Res = SoftPromoteHalfOp_BR_CC(N);
3027 break;
3028 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3029 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3030 case ISD::ATOMIC_STORE:
3031 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3032 break;
3033 case ISD::STACKMAP:
3034 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3035 break;
3036 case ISD::PATCHPOINT:
3037 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3038 break;
3039 }
3040
3041 if (!Res.getNode())
3042 return false;
3043
3044 assert(Res.getNode() != N && "Expected a new node!");
3045
3046 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3047 "Invalid operand expansion");
3048
3049 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3050 return false;
3051}
3052
3053SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3054 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3055
3056 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3057}
3058
3059SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BUILD_VECTOR(SDNode *N) {
3060 SDLoc dl(N);
3061 EVT VT = N->getValueType(ResNo: 0);
3062
3063 SmallVector<SDValue, 8> Ops(N->getNumOperands());
3064 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I)
3065 Ops[I] = GetSoftPromotedHalf(Op: N->getOperand(Num: I));
3066
3067 EVT IVT = VT.changeVectorElementTypeToInteger();
3068 SDValue Res = DAG.getBuildVector(VT: IVT, DL: dl, Ops);
3069 return DAG.getBitcast(VT, V: Res);
3070}
3071
3072SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3073 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3074 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3075 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::Other, N1: N->getOperand(Num: 0),
3076 N2: Op);
3077}
3078
3079SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3080 unsigned OpNo) {
3081 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3082 SDValue Op1 = N->getOperand(Num: 1);
3083 EVT RVT = Op1.getValueType();
3084 SDLoc dl(N);
3085
3086 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3087
3088 Op1 = GetSoftPromotedHalf(Op: Op1);
3089 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3090
3091 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3092 N2: Op1);
3093}
3094
3095SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3096 EVT RVT = N->getValueType(ResNo: 0);
3097 bool IsStrict = N->isStrictFPOpcode();
3098 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3099 EVT SVT = Op.getValueType();
3100 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3101
3102 if (IsStrict) {
3103 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
3104 ResultTys: {RVT, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
3105 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3106 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3107 return SDValue();
3108 }
3109
3110 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3111}
3112
3113SDValue DAGTypeLegalizer::SoftPromoteHalfOp_Op0WithStrict(SDNode *N) {
3114 EVT RVT = N->getValueType(ResNo: 0);
3115 bool IsStrict = N->isStrictFPOpcode();
3116 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3117 EVT SVT = Op.getValueType();
3118 SDLoc dl(N);
3119
3120 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3121 Op = GetSoftPromotedHalf(Op);
3122
3123 if (IsStrict) {
3124 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: dl, ResultTys: {NVT, MVT::Other},
3125 Ops: {N->getOperand(Num: 0), Op});
3126 Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {RVT, MVT::Other},
3127 Ops: {Op.getValue(R: 1), Op});
3128 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3129 ReplaceValueWith(From: SDValue(N, 0), To: Op);
3130 return SDValue();
3131 }
3132
3133 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3134 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: RVT, Operand: Res);
3135}
3136
3137SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3138 EVT RVT = N->getValueType(ResNo: 0);
3139 SDValue Op = N->getOperand(Num: 0);
3140 EVT SVT = Op.getValueType();
3141 SDLoc dl(N);
3142
3143 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3144
3145 Op = GetSoftPromotedHalf(Op);
3146
3147 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3148
3149 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3150 N2: N->getOperand(Num: 1));
3151}
3152
3153// TODO: CONVERT_TO_ARBITRARY_FP also needs SoftenFloatOperand and
3154// ExpandFloatOperand handlers for targets with software float or ppcf128
3155// source types. Same gap exists for CONVERT_FROM_ARBITRARY_FP.
3156SDValue DAGTypeLegalizer::SoftPromoteHalfOp_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
3157 EVT RVT = N->getValueType(ResNo: 0);
3158 SDValue Op = N->getOperand(Num: 0);
3159 EVT SVT = Op.getValueType();
3160 SDLoc dl(N);
3161
3162 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3163 Op = GetSoftPromotedHalf(Op);
3164 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3165
3166 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3167 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
3168}
3169
3170SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BR_CC(SDNode *N) {
3171 // ISD::BR_CC node: chain(0), condcode(1), LHS(2), RHS(3), dest(4)
3172 // The comparison operands (LHS, RHS) are soft-promoted halfs.
3173 SDValue Op0 = N->getOperand(Num: 2);
3174 SDValue Op1 = N->getOperand(Num: 3);
3175 SDLoc dl(N);
3176
3177 EVT SVT = Op0.getValueType();
3178 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3179
3180 // Get the soft-promoted i16 values
3181 Op0 = GetSoftPromotedHalf(Op: Op0);
3182 Op1 = GetSoftPromotedHalf(Op: Op1);
3183
3184 // Promote both comparison operands to the larger FP type.
3185 unsigned PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3186 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3187 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3188
3189 // Create a new BR_CC node with promoted operands
3190 return DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: N->getOperand(Num: 0),
3191 N2: N->getOperand(Num: 1), N3: Op0, N4: Op1, N5: N->getOperand(Num: 4));
3192}
3193
3194SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3195 unsigned OpNo) {
3196 assert(OpNo == 0 && "Can only soften the comparison values");
3197 SDValue Op0 = N->getOperand(Num: 0);
3198 SDValue Op1 = N->getOperand(Num: 1);
3199 SDLoc dl(N);
3200
3201 EVT SVT = Op0.getValueType();
3202 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3203
3204 Op0 = GetSoftPromotedHalf(Op: Op0);
3205 Op1 = GetSoftPromotedHalf(Op: Op1);
3206
3207 // Promote to the larger FP type.
3208 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3209 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3210 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3211
3212 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3213 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3214}
3215
3216SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3217 SDValue Op0 = N->getOperand(Num: 0);
3218 SDValue Op1 = N->getOperand(Num: 1);
3219 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3220 SDLoc dl(N);
3221
3222 EVT SVT = Op0.getValueType();
3223 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3224
3225 Op0 = GetSoftPromotedHalf(Op: Op0);
3226 Op1 = GetSoftPromotedHalf(Op: Op1);
3227
3228 // Promote to the larger FP type.
3229 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3230 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3231 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3232
3233 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3234}
3235
3236SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3237 assert(OpNo == 1 && "Can only soften the stored value!");
3238 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3239 SDValue Val = ST->getValue();
3240 SDLoc dl(N);
3241
3242 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3243 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3244 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3245 MMO: ST->getMemOperand());
3246}
3247
3248SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3249 unsigned OpNo) {
3250 assert(OpNo == 1 && "Can only soften the stored value!");
3251 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
3252 SDValue Val = ST->getVal();
3253 SDLoc dl(N);
3254
3255 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3256 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: Promoted.getValueType(),
3257 Chain: ST->getChain(), Ptr: Promoted, Val: ST->getBasePtr(),
3258 MMO: ST->getMemOperand());
3259}
3260
3261SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3262 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3263 SmallVector<SDValue> NewOps(N->ops());
3264 SDValue Op = N->getOperand(Num: OpNo);
3265 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3266 SDValue NewNode =
3267 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3268
3269 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3270 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3271
3272 return SDValue(); // Signal that we replaced the node ourselves.
3273}
3274
3275SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3276 unsigned OpNo) {
3277 assert(OpNo >= 7);
3278 SmallVector<SDValue> NewOps(N->ops());
3279 SDValue Op = N->getOperand(Num: OpNo);
3280 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3281 SDValue NewNode =
3282 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3283
3284 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3285 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3286
3287 return SDValue(); // Signal that we replaced the node ourselves.
3288}
3289