1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error(reason: "Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::FCANONICALIZE:
74 R = SoftenFloatRes_FCANONICALIZE(N); break;
75 case ISD::STRICT_FMINNUM:
76 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
77 case ISD::STRICT_FMAXNUM:
78 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
79 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
80 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
81 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
82 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
83 case ISD::STRICT_FADD:
84 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
85 case ISD::STRICT_FACOS:
86 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
87 case ISD::STRICT_FASIN:
88 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
89 case ISD::STRICT_FATAN:
90 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
91 case ISD::STRICT_FATAN2:
92 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
93 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
94 case ISD::STRICT_FCEIL:
95 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
96 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
97 case ISD::STRICT_FCOS:
98 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
99 case ISD::STRICT_FCOSH:
100 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
101 case ISD::STRICT_FDIV:
102 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
103 case ISD::STRICT_FEXP:
104 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
105 case ISD::STRICT_FEXP2:
106 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
107 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
108 case ISD::STRICT_FFLOOR:
109 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
110 case ISD::STRICT_FLOG:
111 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
112 case ISD::STRICT_FLOG2:
113 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
114 case ISD::STRICT_FLOG10:
115 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
116 case ISD::STRICT_FMA:
117 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
118 case ISD::STRICT_FMUL:
119 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
120 case ISD::STRICT_FNEARBYINT:
121 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
122 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
123 case ISD::STRICT_FP_EXTEND:
124 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
125 case ISD::STRICT_FP_ROUND:
126 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
127 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
128 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
129 case ISD::STRICT_FPOW:
130 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
131 case ISD::STRICT_FPOWI:
132 case ISD::FPOWI:
133 case ISD::FLDEXP:
134 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
135 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
136 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
137 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
138 case ISD::STRICT_FREM:
139 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
140 case ISD::STRICT_FRINT:
141 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
142 case ISD::STRICT_FROUND:
143 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
144 case ISD::STRICT_FROUNDEVEN:
145 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
146 case ISD::STRICT_FSIN:
147 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
148 case ISD::STRICT_FSINH:
149 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
150 case ISD::STRICT_FSQRT:
151 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
152 case ISD::STRICT_FSUB:
153 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
154 case ISD::STRICT_FTAN:
155 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
156 case ISD::STRICT_FTANH:
157 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
158 case ISD::STRICT_FTRUNC:
159 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
160 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
161 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
162 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
163 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
164 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
165 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
166 case ISD::STRICT_SINT_TO_FP:
167 case ISD::STRICT_UINT_TO_FP:
168 case ISD::SINT_TO_FP:
169 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
170 case ISD::POISON:
171 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
172 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
173 case ISD::AssertNoFPClass: R = GetSoftenedFloat(Op: N->getOperand(Num: 0)); break;
174 case ISD::VECREDUCE_FADD:
175 case ISD::VECREDUCE_FMUL:
176 case ISD::VECREDUCE_FMIN:
177 case ISD::VECREDUCE_FMAX:
178 case ISD::VECREDUCE_FMAXIMUM:
179 case ISD::VECREDUCE_FMINIMUM:
180 case ISD::VECREDUCE_FMAXIMUMNUM:
181 case ISD::VECREDUCE_FMINIMUMNUM: R = SoftenFloatRes_VECREDUCE(N); break;
182 case ISD::VECREDUCE_SEQ_FADD:
183 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
184 // clang-format on
185 }
186
187 // If R is null, the sub-method took care of registering the result.
188 if (R.getNode()) {
189 assert(R.getNode() != N);
190 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
191 }
192}
193
194// No libcall is available to soften this operation. Emit a diagnostic and
195// produce a poison result of the softened type \p NVT.
196SDValue DAGTypeLegalizer::SoftenFloatRes_NoLibcall(SDNode *N, EVT NVT) {
197 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
198 N->getOperationName(G: &DAG));
199 if (N->isStrictFPOpcode())
200 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
201 return DAG.getPOISON(VT: NVT);
202}
203
204SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
205 bool IsStrict = N->isStrictFPOpcode();
206 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
207 unsigned Offset = IsStrict ? 1 : 0;
208 assert(N->getNumOperands() == (1 + Offset) &&
209 "Unexpected number of operands!");
210 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
211 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
212 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
213 if (LCImpl == RTLIB::Unsupported)
214 return SoftenFloatRes_NoLibcall(N, NVT);
215 TargetLowering::MakeLibCallOptions CallOptions;
216 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
217 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
218 std::pair<SDValue, SDValue> Tmp =
219 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
220 if (IsStrict)
221 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
222 return Tmp.first;
223}
224
225SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
226 bool IsStrict = N->isStrictFPOpcode();
227 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
228 unsigned Offset = IsStrict ? 1 : 0;
229 assert(N->getNumOperands() == (2 + Offset) &&
230 "Unexpected number of operands!");
231 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
232 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
233 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
234 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
235 if (LCImpl == RTLIB::Unsupported)
236 return SoftenFloatRes_NoLibcall(N, NVT);
237 TargetLowering::MakeLibCallOptions CallOptions;
238 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
239 N->getOperand(Num: 1 + Offset).getValueType() };
240 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
241 std::pair<SDValue, SDValue> Tmp =
242 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
243 if (IsStrict)
244 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
245 return Tmp.first;
246}
247
248SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
249 return BitConvertToInteger(Op: N->getOperand(Num: 0));
250}
251
252SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
253 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
254 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
255 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
256}
257
258SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
259 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
260 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
261 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
262 return NewFence;
263}
264
265SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
266 unsigned ResNo) {
267 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
268 return BitConvertToInteger(Op);
269}
270
271SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
272 // Convert the inputs to integers, and build a new pair out of them.
273 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
274 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
275 VT: N->getValueType(ResNo: 0)),
276 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
277 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
278}
279
280SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
281 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
282 // In ppcf128, the high 64 bits are always first in memory regardless
283 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
284 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
285 // way. However, APInt's are serialized in an Endian-sensitive fashion,
286 // so on big-Endian targets, the two doubles are output in the wrong
287 // order. Fix this by manually flipping the order of the high 64 bits
288 // and the low 64 bits here.
289 if (DAG.getDataLayout().isBigEndian() &&
290 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
291 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
292 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
293 APInt Val(128, words);
294 return DAG.getConstant(Val, DL: SDLoc(CN),
295 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
296 VT: CN->getValueType(ResNo: 0)));
297 } else {
298 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
299 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
300 VT: CN->getValueType(ResNo: 0)));
301 }
302}
303
304SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
305 SDValue Src = N->getOperand(Num: 0);
306 assert(Src.getValueType() == MVT::ppcf128 &&
307 "In floats only ppcf128 can be extracted by element!");
308 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(N),
309 VT: N->getValueType(ResNo: 0).changeTypeToInteger(),
310 N1: DAG.getBitcast(VT: MVT::i128, V: Src), N2: N->getOperand(Num: 1));
311}
312
313SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
314 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
315 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
316 VT: NewOp.getValueType().getVectorElementType(),
317 N1: NewOp, N2: N->getOperand(Num: 1));
318}
319
320SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
321 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
322 unsigned Size = NVT.getSizeInBits();
323
324 // Mask = ~(1 << (Size-1))
325 APInt API = APInt::getAllOnes(numBits: Size);
326 API.clearBit(BitPosition: Size - 1);
327 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
328 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
329 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
330}
331
332SDValue DAGTypeLegalizer::SoftenFloatRes_FCANONICALIZE(SDNode *N) {
333 SDLoc dl(N);
334
335 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
336 // suggested in
337 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
338 // It uses strict_fp operations even outside a strict_fp context in order
339 // to guarantee that the canonicalization is not optimized away by later
340 // passes. The result chain introduced by that is intentionally ignored
341 // since no ordering requirement is intended here.
342
343 // Create strict multiplication by 1.0.
344 SDValue Operand = N->getOperand(Num: 0);
345 EVT VT = Operand.getValueType();
346 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
347 SDValue Chain = DAG.getEntryNode();
348 // Propagate existing flags on canonicalize, and additionally set
349 // NoFPExcept.
350 SDNodeFlags CanonicalizeFlags = N->getFlags();
351 CanonicalizeFlags.setNoFPExcept(true);
352 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
353 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
354 return BitConvertToInteger(Op: Mul);
355}
356
357SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
358 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
359 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
360 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMIN(VT: N->getValueType(ResNo: 0)));
361}
362
363SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
364 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
365 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
366 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAX(VT: N->getValueType(ResNo: 0)));
367}
368
369SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
370 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMINIMUM_NUM(VT: N->getValueType(ResNo: 0)));
371}
372
373SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
374 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM_NUM(VT: N->getValueType(ResNo: 0)));
375}
376
377SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
378 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMINIMUM(VT: N->getValueType(ResNo: 0)));
379}
380
381SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
382 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM(VT: N->getValueType(ResNo: 0)));
383}
384
385SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
386 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
387 Call_F32: RTLIB::ADD_F32,
388 Call_F64: RTLIB::ADD_F64,
389 Call_F80: RTLIB::ADD_F80,
390 Call_F128: RTLIB::ADD_F128,
391 Call_PPCF128: RTLIB::ADD_PPCF128));
392}
393
394SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
395 return SoftenFloatRes_Unary(N, LC: RTLIB::getACOS(VT: N->getValueType(ResNo: 0)));
396}
397
398SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
399 return SoftenFloatRes_Unary(N, LC: RTLIB::getASIN(VT: N->getValueType(ResNo: 0)));
400}
401
402SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
403 return SoftenFloatRes_Unary(N, LC: RTLIB::getATAN(VT: N->getValueType(ResNo: 0)));
404}
405
406SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
407 return SoftenFloatRes_Binary(N, LC: RTLIB::getATAN2(VT: N->getValueType(ResNo: 0)));
408}
409
410SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
411 return SoftenFloatRes_Unary(N, LC: RTLIB::getCBRT(VT: N->getValueType(ResNo: 0)));
412}
413
414SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
415 return SoftenFloatRes_Unary(N, LC: RTLIB::getCEIL(VT: N->getValueType(ResNo: 0)));
416}
417
418SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
419 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
420 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
421 SDLoc dl(N);
422
423 EVT LVT = LHS.getValueType();
424 EVT RVT = RHS.getValueType();
425
426 unsigned LSize = LVT.getSizeInBits();
427 unsigned RSize = RVT.getSizeInBits();
428
429 // First get the sign bit of second operand.
430 SDValue SignBit = DAG.getNode(
431 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
432 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
433 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
434 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
435
436 // Shift right or sign-extend it if the two operands have different types.
437 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
438 if (SizeDiff > 0) {
439 SignBit =
440 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
441 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
442 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
443 DL: DAG.getDataLayout())));
444 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
445 } else if (SizeDiff < 0) {
446 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
447 SignBit =
448 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
449 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
450 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
451 DL: DAG.getDataLayout())));
452 }
453
454 // Clear the sign bit of the first operand.
455 SDValue Mask = DAG.getNode(
456 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
457 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
458 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
459 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
460 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
461
462 // Or the value with the sign bit.
463 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
464}
465
466SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
467 return SoftenFloatRes_Unary(N, LC: RTLIB::getCOS(VT: N->getValueType(ResNo: 0)));
468}
469
470SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
471 return SoftenFloatRes_Unary(N, LC: RTLIB::getCOSH(VT: N->getValueType(ResNo: 0)));
472}
473
474SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
475 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
476 Call_F32: RTLIB::DIV_F32,
477 Call_F64: RTLIB::DIV_F64,
478 Call_F80: RTLIB::DIV_F80,
479 Call_F128: RTLIB::DIV_F128,
480 Call_PPCF128: RTLIB::DIV_PPCF128));
481}
482
483SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
484 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP(VT: N->getValueType(ResNo: 0)));
485}
486
487SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
488 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP2(VT: N->getValueType(ResNo: 0)));
489}
490
491SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
492 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP10(VT: N->getValueType(ResNo: 0)));
493}
494
495SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
496 return SoftenFloatRes_Unary(N, LC: RTLIB::getFLOOR(VT: N->getValueType(ResNo: 0)));
497}
498
499SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
500 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG(VT: N->getValueType(ResNo: 0)));
501}
502
503SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
504 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG2(VT: N->getValueType(ResNo: 0)));
505}
506
507SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
508 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG10(VT: N->getValueType(ResNo: 0)));
509}
510
511SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
512 bool IsStrict = N->isStrictFPOpcode();
513 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
514 unsigned Offset = IsStrict ? 1 : 0;
515 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
516 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
517 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
518 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
519 TargetLowering::MakeLibCallOptions CallOptions;
520 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
521 N->getOperand(Num: 1 + Offset).getValueType(),
522 N->getOperand(Num: 2 + Offset).getValueType() };
523 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
524 std::pair<SDValue, SDValue> Tmp =
525 TLI.makeLibCall(DAG, LC: RTLIB::getFMA(VT: N->getValueType(ResNo: 0)), RetVT: NVT, Ops,
526 CallOptions, dl: SDLoc(N), Chain);
527 if (IsStrict)
528 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
529 return Tmp.first;
530}
531
532SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
533 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
534 Call_F32: RTLIB::MUL_F32,
535 Call_F64: RTLIB::MUL_F64,
536 Call_F80: RTLIB::MUL_F80,
537 Call_F128: RTLIB::MUL_F128,
538 Call_PPCF128: RTLIB::MUL_PPCF128));
539}
540
541SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
542 return SoftenFloatRes_Unary(N, LC: RTLIB::getNEARBYINT(VT: N->getValueType(ResNo: 0)));
543}
544
545SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
546 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
547 SDLoc dl(N);
548
549 // Expand Y = FNEG(X) -> Y = X ^ sign mask
550 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
551 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
552 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
553}
554
555SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
556 bool IsStrict = N->isStrictFPOpcode();
557 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
558 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
559
560 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
561
562 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
563 // -> f32, so proceed in two stages. Also, it's entirely possible for both
564 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
565 // than FP16_TO_FP.
566 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
567 N->getValueType(ResNo: 0) != MVT::f32) {
568 if (IsStrict) {
569 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
570 ResultTys: { MVT::f32, MVT::Other }, Ops: { Chain, Op });
571 Chain = Op.getValue(R: 1);
572 } else {
573 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT: MVT::f32, Operand: Op);
574 }
575 }
576
577 if (Op.getValueType() == MVT::bf16) {
578 // FIXME: Need ReplaceValueWith on chain in strict case
579 return SoftenFloatRes_BF16_TO_FP(N);
580 }
581
582 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
583 if (LC == RTLIB::UNKNOWN_LIBCALL) {
584 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fp_extend");
585 if (IsStrict)
586 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
587 return DAG.getPOISON(VT: NVT);
588 }
589 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
590 if (LCImpl == RTLIB::Unsupported)
591 return SoftenFloatRes_NoLibcall(N, NVT);
592 TargetLowering::MakeLibCallOptions CallOptions;
593 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
594 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
595 std::pair<SDValue, SDValue> Tmp =
596 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
597 if (IsStrict)
598 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
599 return Tmp.first;
600}
601
602// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
603// nodes?
604SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
605 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
606 SDValue Op = N->getOperand(Num: 0);
607 TargetLowering::MakeLibCallOptions CallOptions;
608 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
609 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
610 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
611 CallOptions, dl: SDLoc(N)).first;
612 if (N->getValueType(ResNo: 0) == MVT::f32)
613 return Res32;
614
615 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
616 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: MVT::f32, RetVT: N->getValueType(ResNo: 0));
617 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
618 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
619}
620
621// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
622// nodes?
623SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
624 assert(N->getValueType(0) == MVT::f32 &&
625 "Can only soften BF16_TO_FP with f32 result");
626 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
627 SDValue Op = N->getOperand(Num: 0);
628 SDLoc DL(N);
629 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT,
630 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op));
631 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
632 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
633 return Res;
634}
635
636SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
637 bool IsStrict = N->isStrictFPOpcode();
638 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
639 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
640 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
641 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
642 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
643 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
644 if (LCImpl == RTLIB::Unsupported)
645 return SoftenFloatRes_NoLibcall(N, NVT);
646 TargetLowering::MakeLibCallOptions CallOptions;
647 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
648 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
649 std::pair<SDValue, SDValue> Tmp =
650 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
651 if (IsStrict)
652 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
653 return Tmp.first;
654}
655
656SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
657 return SoftenFloatRes_Binary(N, LC: RTLIB::getPOW(VT: N->getValueType(ResNo: 0)));
658}
659
660SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
661 bool IsStrict = N->isStrictFPOpcode();
662 unsigned Offset = IsStrict ? 1 : 0;
663 bool IsPowI =
664 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
665 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
666
667 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(VT: N->getValueType(ResNo: 0))
668 : RTLIB::getLDEXP(VT: N->getValueType(ResNo: 0));
669 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
670 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
671 if (LCImpl == RTLIB::Unsupported) {
672 // Some targets don't have a powi libcall; use pow instead.
673 // FIXME: Implement this if some target needs it.
674 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fpowi to fpow");
675 if (IsStrict)
676 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
677 return DAG.getPOISON(VT: NVT);
678 }
679
680 if (DAG.getLibInfo().getIntSize() !=
681 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
682 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
683 // would use the wrong type for the argument.
684 DAG.getContext()->emitError(ErrorStr: "powi exponent does not match sizeof(int)");
685 if (IsStrict)
686 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
687 return DAG.getPOISON(VT: NVT);
688 }
689
690 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
691 N->getOperand(Num: 1 + Offset) };
692 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
693 TargetLowering::MakeLibCallOptions CallOptions;
694 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
695 N->getOperand(Num: 1 + Offset).getValueType() };
696 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
697 CallOptions.setIsSigned();
698 std::pair<SDValue, SDValue> Tmp =
699 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
700 if (IsStrict)
701 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
702 return Tmp.first;
703}
704
705SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
706 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
707 EVT VT0 = N->getValueType(ResNo: 0);
708 EVT VT1 = N->getValueType(ResNo: 1);
709 RTLIB::Libcall LC = RTLIB::getFREXP(VT: VT0);
710 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
711 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
712 SDLoc DL(N);
713
714 if (LCImpl == RTLIB::Unsupported) {
715 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
716 N->getOperationName(G: &DAG));
717 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
718 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
719 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
720 }
721
722 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
723 // If the exponent does not match with sizeof(int) a libcall would use the
724 // wrong type for the argument.
725 // TODO: Should be able to handle mismatches.
726 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
727 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
728 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
729 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
730 }
731
732 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
733
734 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
735 TargetLowering::MakeLibCallOptions CallOptions;
736 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
737 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
738 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
739
740 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
741 // but we only really need to handle the 0th one for softening anyway.
742 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0)
743 .setOpsTypeOverrides(CallOpsTypeOverrides);
744
745 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT0, Ops, CallOptions,
746 dl: DL, /*Chain=*/SDValue());
747 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
748 auto PtrInfo =
749 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
750
751 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
752
753 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
754 return ReturnVal;
755}
756
757bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
758 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
759 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
760 EVT VT = N->getValueType(ResNo: 0);
761
762 assert(VT == N->getValueType(1) &&
763 "expected both return values to have the same type");
764
765 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
766 if (LCImpl == RTLIB::Unsupported)
767 return false;
768
769 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
770
771 SDLoc DL(N);
772
773 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(Op: N->getOperand(Num: 0))};
774 SmallVector<EVT, 3> OpsVT = {VT};
775
776 std::array<SDValue, 2> StackSlots;
777 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
778 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
779 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
780 if (ResNum == CallRetResNo)
781 continue;
782 SDValue StackSlot = DAG.CreateStackTemporary(VT: NVT);
783 Ops.push_back(Elt: StackSlot);
784 OpsVT.push_back(Elt: StackSlot.getValueType());
785 StackSlots[ResNum] = StackSlot;
786 CallOpsTypeOverrides.push_back(Elt: PointerTy);
787 }
788
789 TargetLowering::MakeLibCallOptions CallOptions;
790 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
791 // but since both returns have the same type it should be okay.
792 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT)
793 .setOpsTypeOverrides(CallOpsTypeOverrides);
794
795 auto [ReturnVal, Chain] =
796 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: DL,
797 /*Chain=*/SDValue());
798
799 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
800 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
801 auto PtrInfo =
802 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
803 return DAG.getLoad(VT: NVT, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
804 };
805
806 for (auto [ResNum, SlackSlot] : enumerate(First&: StackSlots)) {
807 if (CallRetResNo == ResNum) {
808 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: ReturnVal);
809 continue;
810 }
811 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: CreateStackLoad(SlackSlot));
812 }
813
814 return true;
815}
816
817SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
818 EVT VT = N->getValueType(ResNo: 0);
819 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(VT)))
820 return SDValue();
821
822 // Fall back on softening the separate sin and cos calls if available.
823 RTLIB::Libcall SinLC = RTLIB::getSIN(VT);
824 RTLIB::Libcall CosLC = RTLIB::getCOS(VT);
825
826 SDValue SoftSin, SoftCos;
827 if (DAG.getLibcalls().getLibcallImpl(Call: SinLC) == RTLIB::Unsupported ||
828 DAG.getLibcalls().getLibcallImpl(Call: CosLC) == RTLIB::Unsupported) {
829 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fsincos");
830
831 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
832 SoftSin = SoftCos = DAG.getPOISON(VT: NVT);
833 } else {
834 SoftSin = SoftenFloatRes_Unary(N, LC: SinLC);
835 SoftCos = SoftenFloatRes_Unary(N, LC: CosLC);
836 }
837
838 SetSoftenedFloat(Op: SDValue(N, 0), Result: SoftSin);
839 SetSoftenedFloat(Op: SDValue(N, 1), Result: SoftCos);
840 return SDValue();
841}
842
843SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
844 EVT VT = N->getValueType(ResNo: 0);
845 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT),
846 /*CallRetResNo=*/0))
847 return SDValue();
848
849 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
850 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fmodf");
851 SDValue Poison = DAG.getPOISON(VT: NVT);
852 SetSoftenedFloat(Op: SDValue(N, 0), Result: Poison);
853 SetSoftenedFloat(Op: SDValue(N, 1), Result: Poison);
854 return SDValue();
855}
856
857SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
858 return SoftenFloatRes_Binary(N, LC: RTLIB::getREM(VT: N->getValueType(ResNo: 0)));
859}
860
861SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
862 return SoftenFloatRes_Unary(N, LC: RTLIB::getRINT(VT: N->getValueType(ResNo: 0)));
863}
864
865SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
866 return SoftenFloatRes_Unary(N, LC: RTLIB::getROUND(VT: N->getValueType(ResNo: 0)));
867}
868
869SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
870 return SoftenFloatRes_Unary(N, LC: RTLIB::getROUNDEVEN(VT: N->getValueType(ResNo: 0)));
871}
872
873SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
874 return SoftenFloatRes_Unary(N, LC: RTLIB::getSIN(VT: N->getValueType(ResNo: 0)));
875}
876
877SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
878 return SoftenFloatRes_Unary(N, LC: RTLIB::getSINH(VT: N->getValueType(ResNo: 0)));
879}
880
881SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
882 return SoftenFloatRes_Unary(N, LC: RTLIB::getSQRT(VT: N->getValueType(ResNo: 0)));
883}
884
885SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
886 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
887 Call_F32: RTLIB::SUB_F32,
888 Call_F64: RTLIB::SUB_F64,
889 Call_F80: RTLIB::SUB_F80,
890 Call_F128: RTLIB::SUB_F128,
891 Call_PPCF128: RTLIB::SUB_PPCF128));
892}
893
894SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
895 return SoftenFloatRes_Unary(N, LC: RTLIB::getTAN(VT: N->getValueType(ResNo: 0)));
896}
897
898SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
899 return SoftenFloatRes_Unary(N, LC: RTLIB::getTANH(VT: N->getValueType(ResNo: 0)));
900}
901
902SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
903 return SoftenFloatRes_Unary(N, LC: RTLIB::getTRUNC(VT: N->getValueType(ResNo: 0)));
904}
905
906SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
907 LoadSDNode *L = cast<LoadSDNode>(Val: N);
908 EVT VT = N->getValueType(ResNo: 0);
909 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
910 SDLoc dl(N);
911
912 auto MMOFlags =
913 L->getMemOperand()->getFlags() &
914 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
915 SDValue NewL;
916 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
917 // If softening widens the integer representation (e.g. x86_fp80 -> i96),
918 // load the original memory width and extend to the softened type.
919 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
920 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::EXTLOAD, VT: NVT, dl,
921 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
922 PtrInfo: L->getPointerInfo(), MemVT, Alignment: L->getBaseAlign(), MMOFlags,
923 Metadata: L->getAAInfo());
924 // Legalized the chain result - switch anything that used the old chain to
925 // use the new one.
926 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
927 return NewL;
928 }
929
930 // Do a non-extending load followed by FP_EXTEND.
931 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
932 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
933 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(), Alignment: L->getBaseAlign(),
934 MMOFlags, Metadata: L->getAAInfo());
935 // Legalized the chain result - switch anything that used the old chain to
936 // use the new one.
937 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
938 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
939 return BitConvertToInteger(Op: ExtendNode);
940}
941
942SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
943 AtomicSDNode *L = cast<AtomicSDNode>(Val: N);
944 EVT VT = N->getValueType(ResNo: 0);
945 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
946 SDLoc dl(N);
947
948 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
949 SDValue NewL =
950 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
951 Ops: {L->getChain(), L->getBasePtr()}, MMO: L->getMemOperand());
952
953 // Legalized the chain result - switch anything that used the old chain to
954 // use the new one.
955 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
956 return NewL;
957 }
958
959 report_fatal_error(reason: "softening fp extending atomic load not handled");
960}
961
962SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
963 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
964 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
965 return DAG.getSelect(DL: SDLoc(N),
966 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
967}
968
969SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
970 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
971 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
972 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
973 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
974 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
975}
976
977SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
978 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
979 VT: N->getValueType(ResNo: 0)));
980}
981
982SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
983 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
984 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
985 EVT VT = N->getValueType(ResNo: 0);
986 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
987 SDLoc dl(N);
988
989 SDValue NewVAARG;
990 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
991 Align: N->getConstantOperandVal(Num: 3));
992
993 // Legalized the chain result - switch anything that used the old chain to
994 // use the new one.
995 if (N != NewVAARG.getValue(R: 1).getNode())
996 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
997 return NewVAARG;
998}
999
1000SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1001 bool IsStrict = N->isStrictFPOpcode();
1002 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1003 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1004 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
1005 EVT RVT = N->getValueType(ResNo: 0);
1006 EVT NVT = EVT();
1007 SDLoc dl(N);
1008
1009 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1010 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1011 // match. Look for an appropriate libcall.
1012 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1013 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1014 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1015 NVT = (MVT::SimpleValueType)t;
1016 // The source needs to big enough to hold the operand.
1017 if (NVT.bitsGE(VT: SVT))
1018 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
1019 }
1020 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1021
1022 EVT NRVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT);
1023 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1024 if (LCImpl == RTLIB::Unsupported)
1025 return SoftenFloatRes_NoLibcall(N, NVT: NRVT);
1026
1027 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1028 // Sign/zero extend the argument if the libcall takes a larger type.
1029 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
1030 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
1031 TargetLowering::MakeLibCallOptions CallOptions;
1032 CallOptions.setIsSigned(Signed);
1033 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1034 std::pair<SDValue, SDValue> Tmp =
1035 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NRVT, Ops: Op, CallOptions, dl, Chain);
1036
1037 if (IsStrict)
1038 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1039 return Tmp.first;
1040}
1041
1042SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1043 // Expand and soften recursively.
1044 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
1045 return SDValue();
1046}
1047
1048SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1049 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
1050 return SDValue();
1051}
1052
1053//===----------------------------------------------------------------------===//
1054// Convert Float Operand to Integer
1055//===----------------------------------------------------------------------===//
1056
1057bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1058 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1059 SDValue Res = SDValue();
1060
1061 switch (N->getOpcode()) {
1062 default:
1063#ifndef NDEBUG
1064 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1065 N->dump(&DAG); dbgs() << "\n";
1066#endif
1067 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
1068
1069 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1070 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1071 case ISD::STRICT_FP_TO_FP16:
1072 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1073 case ISD::FP_TO_BF16:
1074 case ISD::STRICT_FP_TO_BF16:
1075 case ISD::STRICT_FP_ROUND:
1076 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1077 case ISD::STRICT_FP_TO_SINT:
1078 case ISD::STRICT_FP_TO_UINT:
1079 case ISD::FP_TO_SINT:
1080 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1081 case ISD::FP_TO_SINT_SAT:
1082 case ISD::FP_TO_UINT_SAT:
1083 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1084 case ISD::STRICT_LROUND:
1085 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1086 case ISD::STRICT_LLROUND:
1087 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1088 case ISD::STRICT_LRINT:
1089 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1090 case ISD::STRICT_LLRINT:
1091 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1092 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1093 case ISD::STRICT_FSETCC:
1094 case ISD::STRICT_FSETCCS:
1095 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1096 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1097 case ISD::ATOMIC_STORE:
1098 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1099 break;
1100 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1101 case ISD::FAKE_USE:
1102 Res = SoftenFloatOp_FAKE_USE(N);
1103 break;
1104 case ISD::STACKMAP:
1105 Res = SoftenFloatOp_STACKMAP(N, OpNo);
1106 break;
1107 case ISD::PATCHPOINT:
1108 Res = SoftenFloatOp_PATCHPOINT(N, OpNo);
1109 break;
1110 }
1111
1112 // If the result is null, the sub-method took care of registering results etc.
1113 if (!Res.getNode()) return false;
1114
1115 // If the result is N, the sub-method updated N in place. Tell the legalizer
1116 // core about this to re-analyze.
1117 if (Res.getNode() == N)
1118 return true;
1119
1120 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1121 "Invalid operand softening");
1122
1123 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1124 return false;
1125}
1126
1127SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1128 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
1129
1130 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
1131}
1132
1133SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1134 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1135 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1136 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1137 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1138 N->getOpcode() == ISD::FP_TO_BF16 ||
1139 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1140 N->getOpcode() == ISD::STRICT_FP_ROUND);
1141
1142 bool IsStrict = N->isStrictFPOpcode();
1143 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1144 EVT SVT = Op.getValueType();
1145 EVT RVT = N->getValueType(ResNo: 0);
1146 EVT FloatRVT = RVT;
1147 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1148 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1149 FloatRVT = MVT::f16;
1150 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1151 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1152 FloatRVT = MVT::bf16;
1153
1154 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
1155 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1156
1157 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1158 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1159 if (LCImpl == RTLIB::Unsupported) {
1160 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
1161 N->getOperationName(G: &DAG));
1162 SDValue Poison = DAG.getPOISON(VT: RVT);
1163 if (IsStrict) {
1164 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1165 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1166 return SDValue();
1167 }
1168 return Poison;
1169 }
1170 Op = GetSoftenedFloat(Op);
1171 TargetLowering::MakeLibCallOptions CallOptions;
1172 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1173 std::pair<SDValue, SDValue> Tmp =
1174 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1175 if (IsStrict) {
1176 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1177 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1178 return SDValue();
1179 }
1180 return Tmp.first;
1181}
1182
1183SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1184 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1185 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1186
1187 EVT VT = NewLHS.getValueType();
1188 NewLHS = GetSoftenedFloat(Op: NewLHS);
1189 NewRHS = GetSoftenedFloat(Op: NewRHS);
1190 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1191 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1192
1193 // If softenSetCCOperands returned a scalar, we need to compare the result
1194 // against zero to select between true and false values.
1195 if (!NewRHS.getNode()) {
1196 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1197 CCCode = ISD::SETNE;
1198 }
1199
1200 // Update N to have the operands specified.
1201 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1202 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1203 Op5: N->getOperand(Num: 4)),
1204 0);
1205}
1206
1207// Even if the result type is legal, no libcall may exactly match. (e.g. We
1208// don't have FP-i8 conversions) This helper method looks for an appropriate
1209// promoted libcall.
1210static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1211 bool Signed) {
1212 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1213 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1214 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1215 ++IntVT) {
1216 Promoted = (MVT::SimpleValueType)IntVT;
1217 // The type needs to big enough to hold the result.
1218 if (Promoted.bitsGE(VT: RetVT))
1219 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1220 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1221 }
1222 return LC;
1223}
1224
1225SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1226 bool IsStrict = N->isStrictFPOpcode();
1227 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1228 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1229
1230 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1231 EVT SVT = Op.getValueType();
1232 EVT RVT = N->getValueType(ResNo: 0);
1233 EVT NVT = EVT();
1234 SDLoc dl(N);
1235
1236 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1237 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1238 // match, eg. we don't have fp -> i8 conversions.
1239 // Look for an appropriate libcall.
1240 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1241 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1242 "Unsupported FP_TO_XINT!");
1243
1244 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1245 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1246 if (LCImpl == RTLIB::Unsupported) {
1247 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
1248 N->getOperationName(G: &DAG));
1249 SDValue Poison = DAG.getPOISON(VT: RVT);
1250 if (IsStrict) {
1251 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1252 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1253 return SDValue();
1254 }
1255 return Poison;
1256 }
1257
1258 Op = GetSoftenedFloat(Op);
1259 TargetLowering::MakeLibCallOptions CallOptions;
1260 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1261 std::pair<SDValue, SDValue> Tmp =
1262 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
1263
1264 // Truncate the result if the libcall returns a larger type.
1265 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1266
1267 if (!IsStrict)
1268 return Res;
1269
1270 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1271 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1272 return SDValue();
1273}
1274
1275SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1276 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1277 return Res;
1278}
1279
1280SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1281 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1282 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1283
1284 EVT VT = NewLHS.getValueType();
1285 NewLHS = GetSoftenedFloat(Op: NewLHS);
1286 NewRHS = GetSoftenedFloat(Op: NewRHS);
1287 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1288 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1289
1290 // If softenSetCCOperands returned a scalar, we need to compare the result
1291 // against zero to select between true and false values.
1292 if (!NewRHS.getNode()) {
1293 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1294 CCCode = ISD::SETNE;
1295 }
1296
1297 // Update N to have the operands specified.
1298 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1299 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1300 Op5: DAG.getCondCode(Cond: CCCode)),
1301 0);
1302}
1303
1304SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1305 bool IsStrict = N->isStrictFPOpcode();
1306 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1307 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1308 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1309 ISD::CondCode CCCode =
1310 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1311
1312 EVT VT = Op0.getValueType();
1313 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1314 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1315 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1316 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1317
1318 // Update N to have the operands specified.
1319 if (NewRHS.getNode()) {
1320 if (IsStrict)
1321 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1322 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1323 else
1324 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1325 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1326 }
1327
1328 // Otherwise, softenSetCCOperands returned a scalar, use it.
1329 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1330 "Unexpected setcc expansion!");
1331
1332 if (IsStrict) {
1333 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1334 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1335 return SDValue();
1336 }
1337 return NewLHS;
1338}
1339
1340SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1341 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1342 assert(OpNo == 1 && "Can only soften the stored value!");
1343 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1344 SDValue Val = ST->getValue();
1345 SDLoc dl(N);
1346
1347 if (ST->isTruncatingStore())
1348 // Do an FP_ROUND followed by a non-truncating store.
1349 Val = BitConvertToInteger(
1350 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1351 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1352 else
1353 Val = GetSoftenedFloat(Op: Val);
1354
1355 // If softening widens the integer representation (e.g. x86_fp80 -> i96),
1356 // truncate the value before storing to preserve the original memory width.
1357 EVT MemVT =
1358 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ST->getMemoryVT().getSizeInBits());
1359 return DAG.getTruncStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(), SVT: MemVT,
1360 MMO: ST->getMemOperand());
1361}
1362
1363SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1364 assert(OpNo == 1 && "Can only soften the stored value!");
1365 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
1366 SDValue Val = ST->getVal();
1367 EVT VT = Val.getValueType();
1368 SDLoc dl(N);
1369
1370 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1371
1372 SDValue NewVal = GetSoftenedFloat(Op: Val);
1373 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: VT, Chain: ST->getChain(), Ptr: NewVal,
1374 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
1375}
1376
1377SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1378 SDValue LHS = N->getOperand(Num: 0);
1379 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1380 SDLoc dl(N);
1381
1382 EVT LVT = LHS.getValueType();
1383 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1384 EVT RVT = RHS.getValueType();
1385
1386 unsigned LSize = LVT.getSizeInBits();
1387 unsigned RSize = RVT.getSizeInBits();
1388
1389 // Shift right or sign-extend it if the two operands have different types.
1390 int SizeDiff = RSize - LSize;
1391 if (SizeDiff > 0) {
1392 RHS =
1393 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1394 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1395 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1396 DL: DAG.getDataLayout())));
1397 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1398 } else if (SizeDiff < 0) {
1399 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1400 RHS =
1401 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1402 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1403 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1404 DL: DAG.getDataLayout())));
1405 }
1406
1407 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1408 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1409}
1410
1411SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1412 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1413 bool IsStrict = N->isStrictFPOpcode();
1414 unsigned Offset = IsStrict ? 1 : 0;
1415 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1416 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1417 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1418 if (LCImpl == RTLIB::Unsupported) {
1419 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
1420 N->getOperationName(G: &DAG));
1421 SDValue Poison = DAG.getPOISON(VT: N->getValueType(ResNo: 0));
1422 if (IsStrict) {
1423 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1424 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1425 return SDValue();
1426 }
1427 return Poison;
1428 }
1429 TargetLowering::MakeLibCallOptions CallOptions;
1430 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1431 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
1432 std::pair<SDValue, SDValue> Tmp =
1433 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1434 if (IsStrict) {
1435 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1436 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1437 return SDValue();
1438 }
1439
1440 return Tmp.first;
1441}
1442
1443SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1444 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1445 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1446 Call_F32: RTLIB::LROUND_F32,
1447 Call_F64: RTLIB::LROUND_F64,
1448 Call_F80: RTLIB::LROUND_F80,
1449 Call_F128: RTLIB::LROUND_F128,
1450 Call_PPCF128: RTLIB::LROUND_PPCF128));
1451}
1452
1453SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1454 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1455 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1456 Call_F32: RTLIB::LLROUND_F32,
1457 Call_F64: RTLIB::LLROUND_F64,
1458 Call_F80: RTLIB::LLROUND_F80,
1459 Call_F128: RTLIB::LLROUND_F128,
1460 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1461}
1462
1463SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1464 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1465 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1466 Call_F32: RTLIB::LRINT_F32,
1467 Call_F64: RTLIB::LRINT_F64,
1468 Call_F80: RTLIB::LRINT_F80,
1469 Call_F128: RTLIB::LRINT_F128,
1470 Call_PPCF128: RTLIB::LRINT_PPCF128));
1471}
1472
1473SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1474 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1475 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1476 Call_F32: RTLIB::LLRINT_F32,
1477 Call_F64: RTLIB::LLRINT_F64,
1478 Call_F80: RTLIB::LLRINT_F80,
1479 Call_F128: RTLIB::LLRINT_F128,
1480 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1481}
1482
1483SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1484 SDValue Op1 = BitConvertToInteger(Op: N->getOperand(Num: 1));
1485 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1486 N1: N->getOperand(Num: 0), N2: Op1);
1487}
1488
1489SDValue DAGTypeLegalizer::SoftenFloatOp_STACKMAP(SDNode *N, unsigned OpNo) {
1490 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
1491 SmallVector<SDValue> NewOps(N->ops());
1492 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1493 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1494}
1495
1496SDValue DAGTypeLegalizer::SoftenFloatOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
1497 assert(OpNo >= 7);
1498 SmallVector<SDValue> NewOps(N->ops());
1499 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1500 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1501}
1502
1503//===----------------------------------------------------------------------===//
1504// Float Result Expansion
1505//===----------------------------------------------------------------------===//
1506
1507/// ExpandFloatResult - This method is called when the specified result of the
1508/// specified node is found to need expansion. At this point, the node may also
1509/// have invalid operands or may have other results that need promotion, we just
1510/// know that (at least) one result needs expansion.
1511void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1512 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1513 SDValue Lo, Hi;
1514 Lo = Hi = SDValue();
1515
1516 // See if the target wants to custom expand this node.
1517 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1518 return;
1519
1520 switch (N->getOpcode()) {
1521 default:
1522#ifndef NDEBUG
1523 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1524 N->dump(&DAG); dbgs() << "\n";
1525#endif
1526 report_fatal_error(reason: "Do not know how to expand the result of this "
1527 "operator!");
1528 // clang-format off
1529 case ISD::POISON:
1530 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1531 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1532 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1533
1534 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1535 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1536 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1537 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1538 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1539 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1540
1541 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1542 case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
1543 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1544 case ISD::STRICT_FMINNUM:
1545 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1546 case ISD::STRICT_FMAXNUM:
1547 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1548 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1549 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1550 case ISD::STRICT_FADD:
1551 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1552 case ISD::STRICT_FACOS:
1553 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1554 case ISD::STRICT_FASIN:
1555 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1556 case ISD::STRICT_FATAN:
1557 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1558 case ISD::STRICT_FATAN2:
1559 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1560 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1561 case ISD::STRICT_FCEIL:
1562 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1563 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1564 case ISD::STRICT_FCOS:
1565 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1566 case ISD::STRICT_FCOSH:
1567 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1568 case ISD::STRICT_FDIV:
1569 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1570 case ISD::STRICT_FEXP:
1571 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1572 case ISD::STRICT_FEXP2:
1573 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1574 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1575 case ISD::STRICT_FFLOOR:
1576 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1577 case ISD::STRICT_FLOG:
1578 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1579 case ISD::STRICT_FLOG2:
1580 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1581 case ISD::STRICT_FLOG10:
1582 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1583 case ISD::STRICT_FMA:
1584 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1585 case ISD::STRICT_FMUL:
1586 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1587 case ISD::STRICT_FNEARBYINT:
1588 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1589 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1590 case ISD::STRICT_FP_EXTEND:
1591 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1592 case ISD::STRICT_FPOW:
1593 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1594 case ISD::STRICT_FPOWI:
1595 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1596 case ISD::FLDEXP:
1597 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1598 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1599 case ISD::STRICT_FRINT:
1600 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1601 case ISD::STRICT_FROUND:
1602 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1603 case ISD::STRICT_FROUNDEVEN:
1604 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1605 case ISD::STRICT_FSIN:
1606 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1607 case ISD::STRICT_FSINH:
1608 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1609 case ISD::STRICT_FSQRT:
1610 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1611 case ISD::STRICT_FSUB:
1612 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1613 case ISD::STRICT_FTAN:
1614 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1615 case ISD::STRICT_FTANH:
1616 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1617 case ISD::STRICT_FTRUNC:
1618 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1619 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1620 case ISD::STRICT_SINT_TO_FP:
1621 case ISD::STRICT_UINT_TO_FP:
1622 case ISD::SINT_TO_FP:
1623 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1624 case ISD::STRICT_FREM:
1625 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1626 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1627 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1628 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1629 // clang-format on
1630 }
1631
1632 // If Lo/Hi is null, the sub-method took care of registering results etc.
1633 if (Lo.getNode())
1634 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1635}
1636
1637void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1638 SDValue &Hi) {
1639 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1640 assert(NVT.getSizeInBits() == 64 &&
1641 "Do not know how to expand this float constant!");
1642 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1643 SDLoc dl(N);
1644 const fltSemantics &Sem = NVT.getFltSemantics();
1645 Lo = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 64)), DL: dl, VT: NVT);
1646 Hi = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 0)), DL: dl, VT: NVT);
1647}
1648
1649void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1650 SDValue &Lo, SDValue &Hi) {
1651 bool IsStrict = N->isStrictFPOpcode();
1652 unsigned Offset = IsStrict ? 1 : 0;
1653 SDValue Op = N->getOperand(Num: 0 + Offset);
1654 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1655 TargetLowering::MakeLibCallOptions CallOptions;
1656 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1657 Ops: Op, CallOptions, dl: SDLoc(N),
1658 Chain);
1659 if (IsStrict)
1660 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1661 GetPairElements(Pair: Tmp.first, Lo, Hi);
1662}
1663
1664void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1665 SDValue &Lo, SDValue &Hi) {
1666 bool IsStrict = N->isStrictFPOpcode();
1667 unsigned Offset = IsStrict ? 1 : 0;
1668 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1669 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1670 TargetLowering::MakeLibCallOptions CallOptions;
1671 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1672 Ops, CallOptions, dl: SDLoc(N),
1673 Chain);
1674 if (IsStrict)
1675 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1676 GetPairElements(Pair: Tmp.first, Lo, Hi);
1677}
1678
1679void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1680 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT: N->getValueType(ResNo: 0)),
1681 /*CallRetResNo=*/0);
1682}
1683
1684void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1685 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(VT: N->getValueType(ResNo: 0)));
1686}
1687
1688void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1689 ExpandFloatRes_UnaryWithTwoFPResults(N,
1690 LC: RTLIB::getSINCOSPI(VT: N->getValueType(ResNo: 0)));
1691}
1692
1693void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1694 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1695 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1696 SmallVector<SDValue> Results;
1697 TLI.expandMultipleResultFPLibCall(DAG, LC, Node: N, Results, CallRetResNo);
1698 for (auto [ResNo, Res] : enumerate(First&: Results)) {
1699 SDValue Lo, Hi;
1700 GetPairElements(Pair: Res, Lo, Hi);
1701 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1702 }
1703}
1704
1705void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1706 SDValue &Hi) {
1707 assert(N->getValueType(0) == MVT::ppcf128 &&
1708 "Logic only correct for ppcf128!");
1709 SDLoc dl(N);
1710 SDValue Tmp;
1711 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1712 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1713 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1714 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1715 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1716 Cond: ISD::SETEQ);
1717}
1718
1719void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1720 SDValue &Hi) {
1721 ExpandFloatRes_Binary(N, LC: RTLIB::getFMIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1722}
1723
1724void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1725 SDValue &Hi) {
1726 ExpandFloatRes_Binary(N, LC: RTLIB::getFMAX(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1727}
1728
1729void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1730 SDValue &Hi) {
1731 ExpandFloatRes_Binary(N, LC: RTLIB::getFMINIMUM_NUM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1732}
1733
1734void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1735 SDValue &Hi) {
1736 ExpandFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM_NUM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1737}
1738
1739void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1740 SDValue &Hi) {
1741 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1742 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1743 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1744 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1745}
1746
1747void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1748 SDValue &Hi) {
1749 ExpandFloatRes_Unary(N, LC: RTLIB::getACOS(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1750}
1751
1752void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1753 SDValue &Hi) {
1754 ExpandFloatRes_Unary(N, LC: RTLIB::getASIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1755}
1756
1757void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1758 SDValue &Hi) {
1759 ExpandFloatRes_Unary(N, LC: RTLIB::getATAN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1760}
1761
1762void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1763 SDValue &Hi) {
1764 ExpandFloatRes_Binary(N, LC: RTLIB::getATAN2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1765}
1766
1767void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1768 SDValue &Hi) {
1769 ExpandFloatRes_Unary(N, LC: RTLIB::getCBRT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1770}
1771
1772void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1773 SDValue &Lo, SDValue &Hi) {
1774 ExpandFloatRes_Unary(N, LC: RTLIB::getCEIL(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1775}
1776
1777void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1778 SDValue &Lo, SDValue &Hi) {
1779 ExpandFloatRes_Binary(N, LC: RTLIB::getCOPYSIGN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1780}
1781
1782void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1783 SDValue &Lo, SDValue &Hi) {
1784 ExpandFloatRes_Unary(N, LC: RTLIB::getCOS(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1785}
1786
1787void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1788 SDValue &Hi) {
1789 ExpandFloatRes_Unary(N, LC: RTLIB::getCOSH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1790}
1791
1792void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1793 SDValue &Hi) {
1794 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1795 Call_F32: RTLIB::DIV_F32,
1796 Call_F64: RTLIB::DIV_F64,
1797 Call_F80: RTLIB::DIV_F80,
1798 Call_F128: RTLIB::DIV_F128,
1799 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1800}
1801
1802void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1803 SDValue &Lo, SDValue &Hi) {
1804 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1805}
1806
1807void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1808 SDValue &Lo, SDValue &Hi) {
1809 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1810}
1811
1812void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1813 SDValue &Hi) {
1814 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP10(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1815}
1816
1817void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1818 SDValue &Lo, SDValue &Hi) {
1819 ExpandFloatRes_Unary(N, LC: RTLIB::getFLOOR(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1820}
1821
1822void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1823 SDValue &Lo, SDValue &Hi) {
1824 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1825}
1826
1827void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1828 SDValue &Lo, SDValue &Hi) {
1829 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1830}
1831
1832void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1833 SDValue &Lo, SDValue &Hi) {
1834 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG10(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1835}
1836
1837void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1838 SDValue &Hi) {
1839 bool IsStrict = N->isStrictFPOpcode();
1840 unsigned Offset = IsStrict ? 1 : 0;
1841 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1842 N->getOperand(Num: 2 + Offset) };
1843 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1844 TargetLowering::MakeLibCallOptions CallOptions;
1845 std::pair<SDValue, SDValue> Tmp =
1846 TLI.makeLibCall(DAG, LC: RTLIB::getFMA(VT: N->getValueType(ResNo: 0)),
1847 RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain);
1848 if (IsStrict)
1849 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1850 GetPairElements(Pair: Tmp.first, Lo, Hi);
1851}
1852
1853void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1854 SDValue &Hi) {
1855 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1856 Call_F32: RTLIB::MUL_F32,
1857 Call_F64: RTLIB::MUL_F64,
1858 Call_F80: RTLIB::MUL_F80,
1859 Call_F128: RTLIB::MUL_F128,
1860 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1861}
1862
1863void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1864 SDValue &Lo, SDValue &Hi) {
1865 ExpandFloatRes_Unary(N, LC: RTLIB::getNEARBYINT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1866}
1867
1868void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1869 SDValue &Hi) {
1870 SDLoc dl(N);
1871 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1872 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1873 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1874}
1875
1876void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
1877 SDValue &Hi) {
1878 // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
1879 SDLoc dl(N);
1880 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1881}
1882
1883void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1884 SDValue &Hi) {
1885 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1886 SDLoc dl(N);
1887 bool IsStrict = N->isStrictFPOpcode();
1888
1889 SDValue Chain;
1890 if (IsStrict) {
1891 // If the expanded type is the same as the input type, just bypass the node.
1892 if (NVT == N->getOperand(Num: 1).getValueType()) {
1893 Hi = N->getOperand(Num: 1);
1894 Chain = N->getOperand(Num: 0);
1895 } else {
1896 // Other we need to extend.
1897 Hi = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: { NVT, MVT::Other },
1898 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
1899 Chain = Hi.getValue(R: 1);
1900 }
1901 } else {
1902 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1903 }
1904
1905 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
1906
1907 if (IsStrict)
1908 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1909}
1910
1911void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1912 SDValue &Lo, SDValue &Hi) {
1913 ExpandFloatRes_Binary(N, LC: RTLIB::getPOW(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1914}
1915
1916void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1917 SDValue &Lo, SDValue &Hi) {
1918 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1919}
1920
1921void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1922 SDValue &Hi) {
1923 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1924}
1925
1926void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1927 SDValue &Lo, SDValue &Hi) {
1928 assert(N->getValueType(0) == MVT::ppcf128 &&
1929 "Logic only correct for ppcf128!");
1930
1931 SDLoc dl(N);
1932 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1933 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1934 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1935}
1936
1937void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1938 SDValue &Lo, SDValue &Hi) {
1939 ExpandFloatRes_Binary(N, LC: RTLIB::getREM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1940}
1941
1942void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1943 SDValue &Lo, SDValue &Hi) {
1944 ExpandFloatRes_Unary(N, LC: RTLIB::getRINT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1945}
1946
1947void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1948 SDValue &Lo, SDValue &Hi) {
1949 ExpandFloatRes_Unary(N, LC: RTLIB::getROUND(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1950}
1951
1952void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1953 SDValue &Lo, SDValue &Hi) {
1954 ExpandFloatRes_Unary(N, LC: RTLIB::getROUNDEVEN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1955}
1956
1957void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1958 SDValue &Lo, SDValue &Hi) {
1959 ExpandFloatRes_Unary(N, LC: RTLIB::getSIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1960}
1961
1962void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
1963 SDValue &Hi) {
1964 ExpandFloatRes_Unary(N, LC: RTLIB::getSINH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1965}
1966
1967void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1968 SDValue &Lo, SDValue &Hi) {
1969 ExpandFloatRes_Unary(N, LC: RTLIB::getSQRT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1970}
1971
1972void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1973 SDValue &Hi) {
1974 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1975 Call_F32: RTLIB::SUB_F32,
1976 Call_F64: RTLIB::SUB_F64,
1977 Call_F80: RTLIB::SUB_F80,
1978 Call_F128: RTLIB::SUB_F128,
1979 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
1980}
1981
1982void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
1983 SDValue &Hi) {
1984 ExpandFloatRes_Unary(N, LC: RTLIB::getTAN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1985}
1986
1987void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
1988 SDValue &Hi) {
1989 ExpandFloatRes_Unary(N, LC: RTLIB::getTANH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1990}
1991
1992void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1993 SDValue &Lo, SDValue &Hi) {
1994 ExpandFloatRes_Unary(N, LC: RTLIB::getTRUNC(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1995}
1996
1997void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1998 SDValue &Hi) {
1999 if (ISD::isNormalLoad(N)) {
2000 ExpandRes_NormalLoad(N, Lo, Hi);
2001 return;
2002 }
2003
2004 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2005 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
2006 SDValue Chain = LD->getChain();
2007 SDValue Ptr = LD->getBasePtr();
2008 SDLoc dl(N);
2009
2010 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
2011 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2012 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2013
2014 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
2015 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
2016
2017 // Remember the chain.
2018 Chain = Hi.getValue(R: 1);
2019
2020 // The low part is zero.
2021 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2022
2023 // Modified the chain - switch anything that used the old chain to use the
2024 // new one.
2025 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
2026}
2027
2028void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2029 SDValue &Hi) {
2030 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2031 EVT VT = N->getValueType(ResNo: 0);
2032 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2033 bool Strict = N->isStrictFPOpcode();
2034 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
2035 EVT SrcVT = Src.getValueType();
2036 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2037 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2038 SDLoc dl(N);
2039 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
2040
2041 // TODO: Any other flags to propagate?
2042 SDNodeFlags Flags;
2043 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2044
2045 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2046 // When promoting partial word types to i32 we must honor the signedness,
2047 // though.
2048 if (SrcVT.bitsLE(VT: MVT::i32)) {
2049 // The integer can be represented exactly in an f64.
2050 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2051 if (Strict) {
2052 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
2053 Ops: {Chain, Src}, Flags);
2054 Chain = Hi.getValue(R: 1);
2055 } else
2056 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
2057 } else {
2058 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2059 if (SrcVT.bitsLE(VT: MVT::i64)) {
2060 Src = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
2061 VT: MVT::i64, Operand: Src);
2062 LC = RTLIB::SINTTOFP_I64_PPCF128;
2063 } else if (SrcVT.bitsLE(VT: MVT::i128)) {
2064 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i128, Operand: Src);
2065 LC = RTLIB::SINTTOFP_I128_PPCF128;
2066 }
2067 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2068
2069 TargetLowering::MakeLibCallOptions CallOptions;
2070 CallOptions.setIsSigned(true);
2071 std::pair<SDValue, SDValue> Tmp =
2072 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
2073 if (Strict)
2074 Chain = Tmp.second;
2075 GetPairElements(Pair: Tmp.first, Lo, Hi);
2076 }
2077
2078 // No need to complement for unsigned 32-bit integers
2079 if (isSigned || SrcVT.bitsLE(VT: MVT::i32)) {
2080 if (Strict)
2081 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2082
2083 return;
2084 }
2085
2086 // Unsigned - fix up the SINT_TO_FP value just calculated.
2087 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2088 // keep semantics correctness if the integer is not exactly representable
2089 // here. See ExpandLegalINT_TO_FP.
2090 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
2091 SrcVT = Src.getValueType();
2092
2093 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2094 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2095 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2096 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2097 ArrayRef<uint64_t> Parts;
2098
2099 switch (SrcVT.getSimpleVT().SimpleTy) {
2100 default:
2101 llvm_unreachable("Unsupported UINT_TO_FP!");
2102 case MVT::i32:
2103 Parts = TwoE32;
2104 break;
2105 case MVT::i64:
2106 Parts = TwoE64;
2107 break;
2108 case MVT::i128:
2109 Parts = TwoE128;
2110 break;
2111 }
2112
2113 // TODO: Are there other fast-math-flags to propagate to this FADD?
2114 SDValue NewLo = DAG.getConstantFP(
2115 Val: APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), DL: dl, VT: MVT::ppcf128);
2116 if (Strict) {
2117 Lo = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other),
2118 Ops: {Chain, Hi, NewLo}, Flags);
2119 Chain = Lo.getValue(R: 1);
2120 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2121 } else
2122 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
2123 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
2124 True: Lo, False: Hi, Cond: ISD::SETLT);
2125 GetPairElements(Pair: Lo, Lo, Hi);
2126}
2127
2128
2129//===----------------------------------------------------------------------===//
2130// Float Operand Expansion
2131//===----------------------------------------------------------------------===//
2132
2133/// ExpandFloatOperand - This method is called when the specified operand of the
2134/// specified node is found to need expansion. At this point, all of the result
2135/// types of the node are known to be legal, but other operands of the node may
2136/// need promotion or expansion as well as the specified one.
2137bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2138 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2139 SDValue Res = SDValue();
2140
2141 // See if the target wants to custom expand this node.
2142 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2143 return false;
2144
2145 switch (N->getOpcode()) {
2146 default:
2147#ifndef NDEBUG
2148 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2149 N->dump(&DAG); dbgs() << "\n";
2150#endif
2151 report_fatal_error(reason: "Do not know how to expand this operator's operand!");
2152
2153 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2154 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2155 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2156
2157 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2158 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2159 case ISD::STRICT_FP_ROUND:
2160 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2161 case ISD::STRICT_FP_TO_SINT:
2162 case ISD::STRICT_FP_TO_UINT:
2163 case ISD::FP_TO_SINT:
2164 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2165 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2166 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2167 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2168 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2169 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2170 case ISD::STRICT_FSETCC:
2171 case ISD::STRICT_FSETCCS:
2172 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2173 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
2174 OpNo); break;
2175 }
2176
2177 // If the result is null, the sub-method took care of registering results etc.
2178 if (!Res.getNode()) return false;
2179
2180 // If the result is N, the sub-method updated N in place. Tell the legalizer
2181 // core about this.
2182 if (Res.getNode() == N)
2183 return true;
2184
2185 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2186 "Invalid operand expansion");
2187
2188 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2189 return false;
2190}
2191
2192/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2193/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2194void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2195 SDValue &NewRHS,
2196 ISD::CondCode &CCCode,
2197 const SDLoc &dl, SDValue &Chain,
2198 bool IsSignaling) {
2199 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2200 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
2201 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
2202
2203 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2204
2205 // FIXME: This generated code sucks. We want to generate
2206 // FCMPU crN, hi1, hi2
2207 // BNE crN, L:
2208 // FCMPU crN, lo1, lo2
2209 // The following can be improved, but not that much.
2210 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2211 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2212 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
2213 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2214 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
2215 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
2216 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2217 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2218 Tmp1 =
2219 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
2220 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
2221 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2222 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2223 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
2224 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2225 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2226 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
2227 NewRHS = SDValue(); // LHS is the result, not a compare.
2228 Chain = OutputChain;
2229}
2230
2231SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2232 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
2233 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
2234 SDValue Chain;
2235 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2236
2237 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2238 // against zero to select between true and false values.
2239 if (!NewRHS.getNode()) {
2240 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2241 CCCode = ISD::SETNE;
2242 }
2243
2244 // Update N to have the operands specified.
2245 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2246 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
2247 Op5: N->getOperand(Num: 4)), 0);
2248}
2249
2250SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2251 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2252 "Logic only correct for ppcf128!");
2253 SDValue Lo, Hi;
2254 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2255 // The ppcf128 value is providing only the sign; take it from the
2256 // higher-order double (which must have the larger magnitude).
2257 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2258 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2259}
2260
2261SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2262 bool IsStrict = N->isStrictFPOpcode();
2263 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2264 "Logic only correct for ppcf128!");
2265 SDValue Lo, Hi;
2266 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2267
2268 if (!IsStrict)
2269 // Round it the rest of the way (e.g. to f32) if needed.
2270 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2271 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2272
2273 // Eliminate the node if the input float type is the same as the output float
2274 // type.
2275 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2276 // Connect the output chain to the input chain, unlinking the node.
2277 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2278 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2279 return SDValue();
2280 }
2281
2282 SDValue Expansion = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
2283 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
2284 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
2285 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2286 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2287 return SDValue();
2288}
2289
2290SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2291 EVT RVT = N->getValueType(ResNo: 0);
2292 SDLoc dl(N);
2293
2294 bool IsStrict = N->isStrictFPOpcode();
2295 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2296 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2297 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2298 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2299
2300 EVT NVT;
2301 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2302 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2303 "Unsupported FP_TO_XINT!");
2304 TargetLowering::MakeLibCallOptions CallOptions;
2305 std::pair<SDValue, SDValue> Tmp =
2306 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2307 if (!IsStrict)
2308 return Tmp.first;
2309
2310 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2311 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2312 return SDValue();
2313}
2314
2315SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2316 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2317 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2318 SDValue Chain;
2319 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2320
2321 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2322 // against zero to select between true and false values.
2323 if (!NewRHS.getNode()) {
2324 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2325 CCCode = ISD::SETNE;
2326 }
2327
2328 // Update N to have the operands specified.
2329 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2330 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2331 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2332}
2333
2334SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2335 bool IsStrict = N->isStrictFPOpcode();
2336 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2337 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2338 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2339 ISD::CondCode CCCode =
2340 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2341 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2342 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2343
2344 // FloatExpandSetCCOperands always returned a scalar.
2345 assert(!NewRHS.getNode() && "Expect to return scalar");
2346 assert(NewLHS.getValueType() == N->getValueType(0) &&
2347 "Unexpected setcc expansion!");
2348 if (Chain) {
2349 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2350 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2351 return SDValue();
2352 }
2353 return NewLHS;
2354}
2355
2356SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2357 if (ISD::isNormalStore(N))
2358 return ExpandOp_NormalStore(N, OpNo);
2359
2360 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2361 assert(OpNo == 1 && "Can only expand the stored value so far");
2362 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2363
2364 SDValue Chain = ST->getChain();
2365 SDValue Ptr = ST->getBasePtr();
2366
2367 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2368 VT: ST->getValue().getValueType());
2369 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2370 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2371 (void)NVT;
2372
2373 SDValue Lo, Hi;
2374 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2375
2376 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2377 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2378}
2379
2380SDValue DAGTypeLegalizer::ExpandFloatOp_XRINT_XROUND(SDNode *N,
2381 RTLIB::Libcall LC) {
2382 EVT RVT = N->getValueType(ResNo: 0);
2383 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2384 if (LCImpl == RTLIB::Unsupported) {
2385 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2386 N->getOperationName(G: &DAG));
2387 return DAG.getPOISON(VT: RVT);
2388 }
2389
2390 TargetLowering::MakeLibCallOptions CallOptions;
2391 return TLI
2392 .makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N))
2393 .first;
2394}
2395
2396SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2397 EVT RetVT = N->getOperand(Num: 0).getValueType();
2398 return ExpandFloatOp_XRINT_XROUND(
2399 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LROUND_F32, Call_F64: RTLIB::LROUND_F64,
2400 Call_F80: RTLIB::LROUND_F80, Call_F128: RTLIB::LROUND_F128,
2401 Call_PPCF128: RTLIB::LROUND_PPCF128));
2402}
2403
2404SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2405 EVT RetVT = N->getOperand(Num: 0).getValueType();
2406 return ExpandFloatOp_XRINT_XROUND(
2407 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LLROUND_F32, Call_F64: RTLIB::LLROUND_F64,
2408 Call_F80: RTLIB::LLROUND_F80, Call_F128: RTLIB::LLROUND_F128,
2409 Call_PPCF128: RTLIB::LLROUND_PPCF128));
2410}
2411
2412SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2413 EVT RetVT = N->getOperand(Num: 0).getValueType();
2414 return ExpandFloatOp_XRINT_XROUND(
2415 N,
2416 LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LRINT_F32, Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
2417 Call_F128: RTLIB::LRINT_F128, Call_PPCF128: RTLIB::LRINT_PPCF128));
2418}
2419
2420SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2421 EVT RetVT = N->getOperand(Num: 0).getValueType();
2422 return ExpandFloatOp_XRINT_XROUND(
2423 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LLRINT_F32, Call_F64: RTLIB::LLRINT_F64,
2424 Call_F80: RTLIB::LLRINT_F80, Call_F128: RTLIB::LLRINT_F128,
2425 Call_PPCF128: RTLIB::LLRINT_PPCF128));
2426}
2427
2428//===----------------------------------------------------------------------===//
2429// Float Operand Promotion
2430//===----------------------------------------------------------------------===//
2431//
2432
2433static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2434 if (OpVT == MVT::f16)
2435 return ISD::FP16_TO_FP;
2436 if (RetVT == MVT::f16)
2437 return ISD::FP_TO_FP16;
2438 if (OpVT == MVT::bf16)
2439 return ISD::BF16_TO_FP;
2440 if (RetVT == MVT::bf16)
2441 return ISD::FP_TO_BF16;
2442 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2443}
2444
2445static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2446 if (OpVT == MVT::f16)
2447 return ISD::STRICT_FP16_TO_FP;
2448 if (RetVT == MVT::f16)
2449 return ISD::STRICT_FP_TO_FP16;
2450 if (OpVT == MVT::bf16)
2451 return ISD::STRICT_BF16_TO_FP;
2452 if (RetVT == MVT::bf16)
2453 return ISD::STRICT_FP_TO_BF16;
2454 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2455}
2456
2457SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2458 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2459 SDLoc SL(N);
2460
2461 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2462 EVT CastVT = CastVal.getValueType();
2463
2464 SDValue NewAtomic
2465 = DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl: SL, MemVT: CastVT,
2466 VTList: DAG.getVTList(VT1: CastVT, VT2: MVT::Other),
2467 Ops: { AM->getChain(), AM->getBasePtr(), CastVal },
2468 MMO: AM->getMemOperand());
2469
2470 SDValue Result = NewAtomic;
2471
2472 // Legalize the chain result by replacing uses of the old value chain with the
2473 // new one
2474 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2475
2476 return Result;
2477}
2478
2479//===----------------------------------------------------------------------===//
2480// Half Result Soft Promotion
2481//===----------------------------------------------------------------------===//
2482
2483void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2484 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2485 N->dump(&DAG));
2486 SDValue R = SDValue();
2487
2488 // See if the target wants to custom expand this node.
2489 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2490 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2491 return;
2492 }
2493
2494 switch (N->getOpcode()) {
2495 default:
2496#ifndef NDEBUG
2497 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2498 N->dump(&DAG); dbgs() << "\n";
2499#endif
2500 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2501 "result!");
2502
2503 case ISD::ARITH_FENCE:
2504 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2505 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2506 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2507 case ISD::EXTRACT_VECTOR_ELT:
2508 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2509 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2510 case ISD::STRICT_FP_ROUND:
2511 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2512
2513 // Unary FP Operations
2514 case ISD::FACOS:
2515 case ISD::FASIN:
2516 case ISD::FATAN:
2517 case ISD::FCBRT:
2518 case ISD::FCEIL:
2519 case ISD::FCOS:
2520 case ISD::FCOSH:
2521 case ISD::FEXP:
2522 case ISD::FEXP2:
2523 case ISD::FEXP10:
2524 case ISD::FFLOOR:
2525 case ISD::FLOG:
2526 case ISD::FLOG2:
2527 case ISD::FLOG10:
2528 case ISD::FNEARBYINT:
2529 case ISD::FREEZE:
2530 case ISD::FRINT:
2531 case ISD::FROUND:
2532 case ISD::FROUNDEVEN:
2533 case ISD::FSIN:
2534 case ISD::FSINH:
2535 case ISD::FSQRT:
2536 case ISD::FTRUNC:
2537 case ISD::FTAN:
2538 case ISD::FTANH:
2539 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2540 case ISD::FABS:
2541 R = SoftPromoteHalfRes_FABS(N);
2542 break;
2543 case ISD::FNEG:
2544 R = SoftPromoteHalfRes_FNEG(N);
2545 break;
2546 case ISD::AssertNoFPClass:
2547 R = SoftPromoteHalfRes_AssertNoFPClass(N);
2548 break;
2549
2550 // Binary FP Operations
2551 case ISD::FADD:
2552 case ISD::FDIV:
2553 case ISD::FMAXIMUM:
2554 case ISD::FMINIMUM:
2555 case ISD::FMAXIMUMNUM:
2556 case ISD::FMINIMUMNUM:
2557 case ISD::FMAXNUM:
2558 case ISD::FMINNUM:
2559 case ISD::FMUL:
2560 case ISD::FPOW:
2561 case ISD::FATAN2:
2562 case ISD::FREM:
2563 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2564
2565 case ISD::FMA: // FMA is same as FMAD
2566 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2567
2568 case ISD::FPOWI:
2569 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2570
2571 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2572
2573 case ISD::FMODF:
2574 case ISD::FSINCOS:
2575 case ISD::FSINCOSPI:
2576 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
2577 break;
2578
2579 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2580 case ISD::ATOMIC_LOAD:
2581 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2582 break;
2583 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2584 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2585 case ISD::STRICT_SINT_TO_FP:
2586 case ISD::STRICT_UINT_TO_FP:
2587 case ISD::SINT_TO_FP:
2588 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2589 case ISD::CONVERT_FROM_ARBITRARY_FP:
2590 R = SoftPromoteHalfRes_CONVERT_FROM_ARBITRARY_FP(N);
2591 break;
2592 case ISD::POISON:
2593 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2594 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2595 case ISD::VECREDUCE_FADD:
2596 case ISD::VECREDUCE_FMUL:
2597 case ISD::VECREDUCE_FMIN:
2598 case ISD::VECREDUCE_FMAX:
2599 case ISD::VECREDUCE_FMAXIMUM:
2600 case ISD::VECREDUCE_FMINIMUM:
2601 case ISD::VECREDUCE_FMAXIMUMNUM:
2602 case ISD::VECREDUCE_FMINIMUMNUM:
2603 R = SoftPromoteHalfRes_VECREDUCE(N);
2604 break;
2605 case ISD::VECREDUCE_SEQ_FADD:
2606 case ISD::VECREDUCE_SEQ_FMUL:
2607 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2608 break;
2609 }
2610
2611 if (R.getNode())
2612 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
2613}
2614
2615SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2616 return DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: MVT::i16,
2617 Operand: BitConvertToInteger(Op: N->getOperand(Num: 0)));
2618}
2619
2620SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2621 return BitConvertToInteger(Op: N->getOperand(Num: 0));
2622}
2623
2624SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2625 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
2626
2627 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2628 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
2629 VT: MVT::i16);
2630}
2631
2632SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2633 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2634 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
2635 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
2636 N2: N->getOperand(Num: 1));
2637}
2638
2639SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2640 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2641 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
2642 SDLoc dl(N);
2643
2644 EVT LVT = LHS.getValueType();
2645 EVT RVT = RHS.getValueType();
2646
2647 unsigned LSize = LVT.getSizeInBits();
2648 unsigned RSize = RVT.getSizeInBits();
2649
2650 // First get the sign bit of second operand.
2651 SDValue SignBit = DAG.getNode(
2652 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
2653 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
2654 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
2655 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
2656
2657 // Shift right or sign-extend it if the two operands have different types.
2658 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2659 if (SizeDiff > 0) {
2660 SignBit =
2661 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
2662 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
2663 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2664 DL: DAG.getDataLayout())));
2665 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
2666 } else if (SizeDiff < 0) {
2667 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
2668 SignBit =
2669 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
2670 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
2671 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2672 DL: DAG.getDataLayout())));
2673 }
2674
2675 // Clear the sign bit of the first operand.
2676 SDValue Mask = DAG.getNode(
2677 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
2678 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
2679 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
2680 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
2681 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
2682
2683 // Or the value with the sign bit.
2684 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
2685}
2686
2687SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2688 EVT OVT = N->getValueType(ResNo: 0);
2689 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2690 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2691 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2692 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2693 SDNodeFlags Flags = N->getFlags();
2694 SDLoc dl(N);
2695
2696 // Promote to the larger FP type.
2697 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2698 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2699 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2700 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
2701
2702 SDValue Res;
2703 if (OVT == MVT::f16) {
2704 // If f16 fma is not natively supported, the value must be promoted to an
2705 // f64 (and not to f32!) to prevent double rounding issues.
2706 SDValue A64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op0, Flags);
2707 SDValue B64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op1, Flags);
2708 SDValue C64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op2, Flags);
2709
2710 // Prefer a wide FMA node if available; otherwise expand to mul+add.
2711 SDValue WideRes;
2712 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), MVT::f64)) {
2713 WideRes = DAG.getNode(Opcode: ISD::FMA, DL: dl, VT: MVT::f64, N1: A64, N2: B64, N3: C64, Flags);
2714 } else {
2715 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f64, N1: A64, N2: B64, Flags);
2716 WideRes = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f64, N1: Mul, N2: C64, Flags);
2717 }
2718
2719 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: MVT::f64, RetVT: OVT), DL: dl, VT: MVT::i16,
2720 Operand: WideRes);
2721 }
2722
2723 Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2, Flags);
2724 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2725}
2726
2727SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2728 EVT OVT = N->getValueType(ResNo: 0);
2729 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2730 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2731 SDValue Op1 = N->getOperand(Num: 1);
2732 SDLoc dl(N);
2733
2734 // Promote to the larger FP type.
2735 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
2736
2737 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2738
2739 // Convert back to FP16 as an integer.
2740 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2741}
2742
2743SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2744 EVT OVT = N->getValueType(ResNo: 0);
2745 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2746 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2747 SDLoc dl(N);
2748
2749 // Promote to the larger FP type.
2750 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2751
2752 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
2753 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
2754
2755 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2756
2757 // Convert back to FP16 as an integer.
2758 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2759}
2760
2761SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
2762 EVT OVT = N->getValueType(ResNo: 0);
2763 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2764 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2765 SDLoc dl(N);
2766
2767 // Promote to the larger FP type.
2768 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2769 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Op);
2770
2771 // Convert back to FP16 as an integer.
2772 ISD::NodeType Truncate = GetPromotionOpcode(OpVT: NVT, RetVT: OVT);
2773 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
2774 ++ResNum) {
2775 SDValue Trunc = DAG.getNode(Opcode: Truncate, DL: dl, VT: MVT::i16, Operand: Res.getValue(R: ResNum));
2776 SetSoftPromotedHalf(Op: SDValue(N, ResNum), Result: Trunc);
2777 }
2778
2779 return SDValue();
2780}
2781
2782SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2783 EVT RVT = N->getValueType(ResNo: 0);
2784 bool IsStrict = N->isStrictFPOpcode();
2785 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2786 EVT SVT = Op.getValueType();
2787
2788 // If the input type needs to be softened, do that now so that call lowering
2789 // will see the f16 type.
2790 if (getTypeAction(VT: SVT) == TargetLowering::TypeSoftenFloat) {
2791 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: RVT);
2792 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
2793
2794 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2795 Op = GetSoftenedFloat(Op);
2796 TargetLowering::MakeLibCallOptions CallOptions;
2797 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
2798 std::pair<SDValue, SDValue> Tmp =
2799 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
2800 if (IsStrict)
2801 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2802 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: MVT::i16, Operand: Tmp.first);
2803 }
2804
2805 if (IsStrict) {
2806 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
2807 ResultTys: {MVT::i16, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
2808 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2809 return Res;
2810 }
2811
2812 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: MVT::i16,
2813 Operand: N->getOperand(Num: 0));
2814}
2815
2816SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2817 LoadSDNode *L = cast<LoadSDNode>(Val: N);
2818
2819 // Load the value as an integer value with the same number of bits.
2820 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2821 SDValue NewL =
2822 DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: MVT::i16,
2823 dl: SDLoc(N), Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
2824 PtrInfo: L->getPointerInfo(), MemVT: MVT::i16, Alignment: L->getBaseAlign(),
2825 MMOFlags: L->getMemOperand()->getFlags(), Metadata: L->getAAInfo());
2826 // Legalize the chain result by replacing uses of the old value chain with the
2827 // new one
2828 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
2829 return NewL;
2830}
2831
2832SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
2833 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2834
2835 // Load the value as an integer value with the same number of bits.
2836 SDValue NewL = DAG.getAtomic(
2837 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: MVT::i16, VTList: DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other),
2838 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
2839
2840 // Legalize the chain result by replacing uses of the old value chain with the
2841 // new one
2842 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
2843 return NewL;
2844}
2845
2846SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2847 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2848 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2849 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1, RHS: Op2,
2850 Flags: N->getFlags());
2851}
2852
2853SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2854 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2855 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
2856 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
2857 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
2858 N5: N->getOperand(Num: 4));
2859}
2860
2861SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2862 EVT OVT = N->getValueType(ResNo: 0);
2863 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2864 SDLoc dl(N);
2865
2866 if (N->isStrictFPOpcode()) {
2867 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
2868 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
2869 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: NVT, RetVT: OVT), DL: dl,
2870 ResultTys: {MVT::i16, MVT::Other}, Ops: {Op.getValue(R: 1), Op});
2871 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
2872 return Op;
2873 }
2874
2875 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
2876
2877 // Round the value to the softened type.
2878 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2879}
2880
2881SDValue
2882DAGTypeLegalizer::SoftPromoteHalfRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
2883 EVT OVT = N->getValueType(ResNo: 0);
2884 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2885 SDLoc dl(N);
2886
2887 SDValue Res = DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: dl, VT: NVT,
2888 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
2889
2890 // Round the value to the softened type.
2891 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2892}
2893
2894SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2895 return DAG.getUNDEF(VT: MVT::i16);
2896}
2897
2898SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2899 EVT OVT = N->getValueType(ResNo: 0);
2900 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2901 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2902 SDLoc dl(N);
2903
2904 // Promote to the larger FP type.
2905 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2906
2907 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
2908
2909 // Convert back to FP16 as an integer.
2910 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2911}
2912
2913SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FABS(SDNode *N) {
2914 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2915 SDLoc dl(N);
2916
2917 // Clear the sign bit.
2918 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i16, N1: Op,
2919 N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: MVT::i16));
2920}
2921
2922SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FNEG(SDNode *N) {
2923 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2924 SDLoc dl(N);
2925
2926 // Invert the sign bit.
2927 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i16, N1: Op,
2928 N2: DAG.getConstant(Val: 0x8000, DL: dl, VT: MVT::i16));
2929}
2930
2931SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
2932 return GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2933}
2934
2935SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2936 EVT OVT = N->getValueType(ResNo: 0);
2937 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2938 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2939 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2940 SDLoc dl(N);
2941
2942 // Promote to the larger FP type.
2943 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2944 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2945 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2946
2947 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2948
2949 // Convert back to FP16 as an integer.
2950 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2951}
2952
2953SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2954 // Expand and soften recursively.
2955 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
2956 return SDValue();
2957}
2958
2959SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
2960 // Expand and soften.
2961 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
2962 return SDValue();
2963}
2964
2965//===----------------------------------------------------------------------===//
2966// Half Operand Soft Promotion
2967//===----------------------------------------------------------------------===//
2968
2969bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2970 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2971 N->dump(&DAG));
2972 SDValue Res = SDValue();
2973
2974 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
2975 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2976 return false;
2977 }
2978
2979 // Nodes that use a promotion-requiring floating point operand, but doesn't
2980 // produce a soft promotion-requiring floating point result, need to be
2981 // legalized to use the soft promoted float operand. Nodes that produce at
2982 // least one soft promotion-requiring floating point result have their
2983 // operands legalized as a part of PromoteFloatResult.
2984 switch (N->getOpcode()) {
2985 default:
2986 #ifndef NDEBUG
2987 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2988 N->dump(&DAG); dbgs() << "\n";
2989 #endif
2990 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2991 "operand!");
2992
2993 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2994 case ISD::BUILD_VECTOR:
2995 Res = SoftPromoteHalfOp_BUILD_VECTOR(N);
2996 break;
2997 case ISD::INSERT_VECTOR_ELT:
2998 Res = SoftPromoteHalfOp_INSERT_VECTOR_ELT(N, OpNo);
2999 break;
3000 case ISD::FAKE_USE:
3001 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
3002 break;
3003 case ISD::FCOPYSIGN:
3004 Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo);
3005 break;
3006 case ISD::FP_TO_SINT:
3007 case ISD::FP_TO_UINT:
3008 case ISD::STRICT_FP_TO_SINT:
3009 case ISD::STRICT_FP_TO_UINT:
3010 case ISD::LLRINT:
3011 case ISD::LLROUND:
3012 case ISD::LRINT:
3013 case ISD::LROUND:
3014 case ISD::STRICT_LLRINT:
3015 case ISD::STRICT_LLROUND:
3016 case ISD::STRICT_LRINT:
3017 case ISD::STRICT_LROUND:
3018 Res = SoftPromoteHalfOp_Op0WithStrict(N);
3019 break;
3020 case ISD::FP_TO_SINT_SAT:
3021 case ISD::FP_TO_UINT_SAT:
3022 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3023 case ISD::CONVERT_TO_ARBITRARY_FP:
3024 Res = SoftPromoteHalfOp_CONVERT_TO_ARBITRARY_FP(N);
3025 break;
3026 case ISD::STRICT_FP_EXTEND:
3027 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3028 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3029 case ISD::BR_CC:
3030 Res = SoftPromoteHalfOp_BR_CC(N);
3031 break;
3032 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3033 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3034 case ISD::ATOMIC_STORE:
3035 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3036 break;
3037 case ISD::STACKMAP:
3038 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3039 break;
3040 case ISD::PATCHPOINT:
3041 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3042 break;
3043 }
3044
3045 if (!Res.getNode())
3046 return false;
3047
3048 assert(Res.getNode() != N && "Expected a new node!");
3049
3050 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3051 "Invalid operand expansion");
3052
3053 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3054 return false;
3055}
3056
3057SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3058 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3059
3060 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3061}
3062
3063SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BUILD_VECTOR(SDNode *N) {
3064 SDLoc dl(N);
3065 EVT VT = N->getValueType(ResNo: 0);
3066
3067 SmallVector<SDValue, 8> Ops(N->getNumOperands());
3068 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I)
3069 Ops[I] = GetSoftPromotedHalf(Op: N->getOperand(Num: I));
3070
3071 EVT IVT = VT.changeVectorElementTypeToInteger();
3072 SDValue Res = DAG.getBuildVector(VT: IVT, DL: dl, Ops);
3073 return DAG.getBitcast(VT, V: Res);
3074}
3075
3076SDValue DAGTypeLegalizer::SoftPromoteHalfOp_INSERT_VECTOR_ELT(SDNode *N,
3077 unsigned OpNo) {
3078 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3079 SDValue Vec = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
3080 SDValue Elt = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3081 SDValue Res = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
3082 VT: Vec.getValueType(), N1: Vec, N2: Elt, N3: N->getOperand(Num: 2));
3083 return DAG.getBitcast(VT: N->getValueType(ResNo: 0), V: Res);
3084}
3085
3086SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3087 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3088 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3089 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::Other, N1: N->getOperand(Num: 0),
3090 N2: Op);
3091}
3092
3093SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3094 unsigned OpNo) {
3095 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3096 SDValue Op1 = N->getOperand(Num: 1);
3097 EVT RVT = Op1.getValueType();
3098 SDLoc dl(N);
3099
3100 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3101
3102 Op1 = GetSoftPromotedHalf(Op: Op1);
3103 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3104
3105 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3106 N2: Op1);
3107}
3108
3109SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3110 EVT RVT = N->getValueType(ResNo: 0);
3111 bool IsStrict = N->isStrictFPOpcode();
3112 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3113 EVT SVT = Op.getValueType();
3114 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3115
3116 if (IsStrict) {
3117 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
3118 ResultTys: {RVT, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
3119 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3120 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3121 return SDValue();
3122 }
3123
3124 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3125}
3126
3127SDValue DAGTypeLegalizer::SoftPromoteHalfOp_Op0WithStrict(SDNode *N) {
3128 EVT RVT = N->getValueType(ResNo: 0);
3129 bool IsStrict = N->isStrictFPOpcode();
3130 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3131 EVT SVT = Op.getValueType();
3132 SDLoc dl(N);
3133
3134 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3135 Op = GetSoftPromotedHalf(Op);
3136
3137 if (IsStrict) {
3138 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: dl, ResultTys: {NVT, MVT::Other},
3139 Ops: {N->getOperand(Num: 0), Op});
3140 Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {RVT, MVT::Other},
3141 Ops: {Op.getValue(R: 1), Op});
3142 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3143 ReplaceValueWith(From: SDValue(N, 0), To: Op);
3144 return SDValue();
3145 }
3146
3147 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3148 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: RVT, Operand: Res);
3149}
3150
3151SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3152 EVT RVT = N->getValueType(ResNo: 0);
3153 SDValue Op = N->getOperand(Num: 0);
3154 EVT SVT = Op.getValueType();
3155 SDLoc dl(N);
3156
3157 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3158
3159 Op = GetSoftPromotedHalf(Op);
3160
3161 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3162
3163 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3164 N2: N->getOperand(Num: 1));
3165}
3166
3167// TODO: CONVERT_TO_ARBITRARY_FP also needs SoftenFloatOperand and
3168// ExpandFloatOperand handlers for targets with software float or ppcf128
3169// source types. Same gap exists for CONVERT_FROM_ARBITRARY_FP.
3170SDValue DAGTypeLegalizer::SoftPromoteHalfOp_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
3171 EVT RVT = N->getValueType(ResNo: 0);
3172 SDValue Op = N->getOperand(Num: 0);
3173 EVT SVT = Op.getValueType();
3174 SDLoc dl(N);
3175
3176 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3177 Op = GetSoftPromotedHalf(Op);
3178 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3179
3180 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3181 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
3182}
3183
3184SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BR_CC(SDNode *N) {
3185 // ISD::BR_CC node: chain(0), condcode(1), LHS(2), RHS(3), dest(4)
3186 // The comparison operands (LHS, RHS) are soft-promoted halfs.
3187 SDValue Op0 = N->getOperand(Num: 2);
3188 SDValue Op1 = N->getOperand(Num: 3);
3189 SDLoc dl(N);
3190
3191 EVT SVT = Op0.getValueType();
3192 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3193
3194 // Get the soft-promoted i16 values
3195 Op0 = GetSoftPromotedHalf(Op: Op0);
3196 Op1 = GetSoftPromotedHalf(Op: Op1);
3197
3198 // Promote both comparison operands to the larger FP type.
3199 unsigned PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3200 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3201 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3202
3203 // Create a new BR_CC node with promoted operands
3204 return DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: N->getOperand(Num: 0),
3205 N2: N->getOperand(Num: 1), N3: Op0, N4: Op1, N5: N->getOperand(Num: 4));
3206}
3207
3208SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3209 unsigned OpNo) {
3210 assert(OpNo == 0 && "Can only soften the comparison values");
3211 SDValue Op0 = N->getOperand(Num: 0);
3212 SDValue Op1 = N->getOperand(Num: 1);
3213 SDLoc dl(N);
3214
3215 EVT SVT = Op0.getValueType();
3216 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3217
3218 Op0 = GetSoftPromotedHalf(Op: Op0);
3219 Op1 = GetSoftPromotedHalf(Op: Op1);
3220
3221 // Promote to the larger FP type.
3222 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3223 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3224 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3225
3226 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3227 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3228}
3229
3230SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3231 SDValue Op0 = N->getOperand(Num: 0);
3232 SDValue Op1 = N->getOperand(Num: 1);
3233 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3234 SDLoc dl(N);
3235
3236 EVT SVT = Op0.getValueType();
3237 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3238
3239 Op0 = GetSoftPromotedHalf(Op: Op0);
3240 Op1 = GetSoftPromotedHalf(Op: Op1);
3241
3242 // Promote to the larger FP type.
3243 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3244 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3245 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3246
3247 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3248}
3249
3250SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3251 assert(OpNo == 1 && "Can only soften the stored value!");
3252 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3253 SDValue Val = ST->getValue();
3254 SDLoc dl(N);
3255
3256 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3257 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3258 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3259 MMO: ST->getMemOperand());
3260}
3261
3262SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3263 unsigned OpNo) {
3264 assert(OpNo == 1 && "Can only soften the stored value!");
3265 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
3266 SDValue Val = ST->getVal();
3267 SDLoc dl(N);
3268
3269 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3270 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: Promoted.getValueType(),
3271 Chain: ST->getChain(), Ptr: Promoted, Val: ST->getBasePtr(),
3272 MMO: ST->getMemOperand());
3273}
3274
3275SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3276 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3277 SmallVector<SDValue> NewOps(N->ops());
3278 SDValue Op = N->getOperand(Num: OpNo);
3279 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3280 SDValue NewNode =
3281 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3282
3283 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3284 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3285
3286 return SDValue(); // Signal that we replaced the node ourselves.
3287}
3288
3289SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3290 unsigned OpNo) {
3291 assert(OpNo >= 7);
3292 SmallVector<SDValue> NewOps(N->ops());
3293 SDValue Op = N->getOperand(Num: OpNo);
3294 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3295 SDValue NewNode =
3296 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3297
3298 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3299 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3300
3301 return SDValue(); // Signal that we replaced the node ourselves.
3302}
3303