1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29static void reportNoLibcall(SelectionDAG &DAG, SDNode *N, EVT VT) {
30 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
31 N->getOperationName(G: &DAG) + " with type " +
32 VT.getEVTString());
33}
34
35/// GetFPLibCall - Return the right libcall for the given floating point type.
36/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
37/// refactored away (see RTLIB::getPOWI for an example).
38static RTLIB::Libcall GetFPLibCall(EVT VT,
39 RTLIB::Libcall Call_F32,
40 RTLIB::Libcall Call_F64,
41 RTLIB::Libcall Call_F80,
42 RTLIB::Libcall Call_F128,
43 RTLIB::Libcall Call_PPCF128) {
44 return
45 VT == MVT::f32 ? Call_F32 :
46 VT == MVT::f64 ? Call_F64 :
47 VT == MVT::f80 ? Call_F80 :
48 VT == MVT::f128 ? Call_F128 :
49 VT == MVT::ppcf128 ? Call_PPCF128 :
50 RTLIB::UNKNOWN_LIBCALL;
51}
52
53//===----------------------------------------------------------------------===//
54// Convert Float Results to Integer
55//===----------------------------------------------------------------------===//
56
57void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
58 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
59 SDValue R = SDValue();
60
61 switch (N->getOpcode()) {
62 // clang-format off
63 default:
64#ifndef NDEBUG
65 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
66 N->dump(&DAG); dbgs() << "\n";
67#endif
68 report_fatal_error(reason: "Do not know how to soften the result of this "
69 "operator!");
70 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
71 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
72 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
73 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
74 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
75 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
76 case ISD::EXTRACT_VECTOR_ELT:
77 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
78 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
79 case ISD::FCANONICALIZE:
80 R = SoftenFloatRes_FCANONICALIZE(N); break;
81 case ISD::STRICT_FMINNUM:
82 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
83 case ISD::STRICT_FMAXNUM:
84 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
85 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
86 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
87 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
88 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
89 case ISD::STRICT_FADD:
90 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
91 case ISD::STRICT_FACOS:
92 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
93 case ISD::STRICT_FASIN:
94 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
95 case ISD::STRICT_FATAN:
96 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
97 case ISD::STRICT_FATAN2:
98 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
99 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
100 case ISD::STRICT_FCEIL:
101 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
102 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
103 case ISD::STRICT_FCOS:
104 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
105 case ISD::STRICT_FCOSH:
106 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
107 case ISD::STRICT_FDIV:
108 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
109 case ISD::STRICT_FEXP:
110 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
111 case ISD::STRICT_FEXP2:
112 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
113 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
114 case ISD::STRICT_FFLOOR:
115 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
116 case ISD::STRICT_FLOG:
117 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
118 case ISD::STRICT_FLOG2:
119 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
120 case ISD::STRICT_FLOG10:
121 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
122 case ISD::STRICT_FMA:
123 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
124 case ISD::STRICT_FMUL:
125 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
126 case ISD::STRICT_FNEARBYINT:
127 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
128 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
129 case ISD::STRICT_FP_EXTEND:
130 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
131 case ISD::STRICT_FP_ROUND:
132 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
133 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
134 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
135 case ISD::STRICT_FPOW:
136 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
137 case ISD::STRICT_FPOWI:
138 case ISD::FPOWI:
139 case ISD::FLDEXP:
140 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
141 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
142 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
143 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
144 case ISD::STRICT_FREM:
145 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
146 case ISD::STRICT_FRINT:
147 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
148 case ISD::STRICT_FROUND:
149 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
150 case ISD::STRICT_FROUNDEVEN:
151 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
152 case ISD::STRICT_FSIN:
153 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
154 case ISD::STRICT_FSINH:
155 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
156 case ISD::STRICT_FSQRT:
157 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
158 case ISD::STRICT_FSUB:
159 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
160 case ISD::STRICT_FTAN:
161 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
162 case ISD::STRICT_FTANH:
163 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
164 case ISD::STRICT_FTRUNC:
165 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
166 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
167 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
168 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
169 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
170 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
171 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
172 case ISD::STRICT_SINT_TO_FP:
173 case ISD::STRICT_UINT_TO_FP:
174 case ISD::SINT_TO_FP:
175 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
176 case ISD::POISON:
177 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
178 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
179 case ISD::AssertNoFPClass: R = GetSoftenedFloat(Op: N->getOperand(Num: 0)); break;
180 case ISD::VECREDUCE_FADD:
181 case ISD::VECREDUCE_FMUL:
182 case ISD::VECREDUCE_FMIN:
183 case ISD::VECREDUCE_FMAX:
184 case ISD::VECREDUCE_FMAXIMUM:
185 case ISD::VECREDUCE_FMINIMUM:
186 case ISD::VECREDUCE_FMAXIMUMNUM:
187 case ISD::VECREDUCE_FMINIMUMNUM: R = SoftenFloatRes_VECREDUCE(N); break;
188 case ISD::VECREDUCE_SEQ_FADD:
189 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
190 // clang-format on
191 }
192
193 // If R is null, the sub-method took care of registering the result.
194 if (R.getNode()) {
195 assert(R.getNode() != N);
196 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
197 }
198}
199
200// No libcall is available to soften this operation. Emit a diagnostic and
201// produce a poison result of the softened type \p NVT.
202SDValue DAGTypeLegalizer::SoftenFloatRes_NoLibcall(SDNode *N, EVT NVT) {
203 reportNoLibcall(DAG, N, VT: N->getValueType(ResNo: 0));
204 if (N->isStrictFPOpcode())
205 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
206 return DAG.getPOISON(VT: NVT);
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
210 bool IsStrict = N->isStrictFPOpcode();
211 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
212 unsigned Offset = IsStrict ? 1 : 0;
213 assert(N->getNumOperands() == (1 + Offset) &&
214 "Unexpected number of operands!");
215 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
216 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
217 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
218 if (LCImpl == RTLIB::Unsupported)
219 return SoftenFloatRes_NoLibcall(N, NVT);
220 TargetLowering::MakeLibCallOptions CallOptions;
221 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
222 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
223 std::pair<SDValue, SDValue> Tmp =
224 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
225 if (IsStrict)
226 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
227 return Tmp.first;
228}
229
230SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
231 bool IsStrict = N->isStrictFPOpcode();
232 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
233 unsigned Offset = IsStrict ? 1 : 0;
234 assert(N->getNumOperands() == (2 + Offset) &&
235 "Unexpected number of operands!");
236 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
237 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
238 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
239 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
240 if (LCImpl == RTLIB::Unsupported)
241 return SoftenFloatRes_NoLibcall(N, NVT);
242 TargetLowering::MakeLibCallOptions CallOptions;
243 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
244 N->getOperand(Num: 1 + Offset).getValueType() };
245 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
246 std::pair<SDValue, SDValue> Tmp =
247 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
248 if (IsStrict)
249 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
250 return Tmp.first;
251}
252
253SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
254 return BitConvertToInteger(Op: N->getOperand(Num: 0));
255}
256
257SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
258 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
259 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
260 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
261}
262
263SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
264 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
265 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
266 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
267 return NewFence;
268}
269
270SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
271 unsigned ResNo) {
272 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
273 return BitConvertToInteger(Op);
274}
275
276SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
277 // Convert the inputs to integers, and build a new pair out of them.
278 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
279 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
280 VT: N->getValueType(ResNo: 0)),
281 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
282 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
283}
284
285SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
286 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
287 // In ppcf128, the high 64 bits are always first in memory regardless
288 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
289 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
290 // way. However, APInt's are serialized in an Endian-sensitive fashion,
291 // so on big-Endian targets, the two doubles are output in the wrong
292 // order. Fix this by manually flipping the order of the high 64 bits
293 // and the low 64 bits here.
294 if (DAG.getDataLayout().isBigEndian() &&
295 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
296 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
297 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
298 APInt Val(128, words);
299 return DAG.getConstant(Val, DL: SDLoc(CN),
300 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
301 VT: CN->getValueType(ResNo: 0)));
302 } else {
303 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
304 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
305 VT: CN->getValueType(ResNo: 0)));
306 }
307}
308
309SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
310 SDValue Src = N->getOperand(Num: 0);
311 assert(Src.getValueType() == MVT::ppcf128 &&
312 "In floats only ppcf128 can be extracted by element!");
313 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(N),
314 VT: N->getValueType(ResNo: 0).changeTypeToInteger(),
315 N1: DAG.getBitcast(VT: MVT::i128, V: Src), N2: N->getOperand(Num: 1));
316}
317
318SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
319 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
320 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
321 VT: NewOp.getValueType().getVectorElementType(),
322 N1: NewOp, N2: N->getOperand(Num: 1));
323}
324
325SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
326 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
327 unsigned Size = NVT.getSizeInBits();
328
329 // Mask = ~(1 << (Size-1))
330 APInt API = APInt::getAllOnes(numBits: Size);
331 API.clearBit(BitPosition: Size - 1);
332 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
333 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
334 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
335}
336
337SDValue DAGTypeLegalizer::SoftenFloatRes_FCANONICALIZE(SDNode *N) {
338 SDLoc dl(N);
339
340 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
341 // suggested in
342 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
343 // It uses strict_fp operations even outside a strict_fp context in order
344 // to guarantee that the canonicalization is not optimized away by later
345 // passes. The result chain introduced by that is intentionally ignored
346 // since no ordering requirement is intended here.
347
348 // Create strict multiplication by 1.0.
349 SDValue Operand = N->getOperand(Num: 0);
350 EVT VT = Operand.getValueType();
351 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
352 SDValue Chain = DAG.getEntryNode();
353 // Propagate existing flags on canonicalize, and additionally set
354 // NoFPExcept.
355 SDNodeFlags CanonicalizeFlags = N->getFlags();
356 CanonicalizeFlags.setNoFPExcept(true);
357 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
358 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
359 return BitConvertToInteger(Op: Mul);
360}
361
362SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
363 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
364 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
365 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMIN(VT: N->getValueType(ResNo: 0)));
366}
367
368SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
369 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
370 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
371 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAX(VT: N->getValueType(ResNo: 0)));
372}
373
374SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
375 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMINIMUM_NUM(VT: N->getValueType(ResNo: 0)));
376}
377
378SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
379 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM_NUM(VT: N->getValueType(ResNo: 0)));
380}
381
382SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
383 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMINIMUM(VT: N->getValueType(ResNo: 0)));
384}
385
386SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
387 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM(VT: N->getValueType(ResNo: 0)));
388}
389
390SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
391 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
392 Call_F32: RTLIB::ADD_F32,
393 Call_F64: RTLIB::ADD_F64,
394 Call_F80: RTLIB::ADD_F80,
395 Call_F128: RTLIB::ADD_F128,
396 Call_PPCF128: RTLIB::ADD_PPCF128));
397}
398
399SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
400 return SoftenFloatRes_Unary(N, LC: RTLIB::getACOS(VT: N->getValueType(ResNo: 0)));
401}
402
403SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
404 return SoftenFloatRes_Unary(N, LC: RTLIB::getASIN(VT: N->getValueType(ResNo: 0)));
405}
406
407SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
408 return SoftenFloatRes_Unary(N, LC: RTLIB::getATAN(VT: N->getValueType(ResNo: 0)));
409}
410
411SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
412 return SoftenFloatRes_Binary(N, LC: RTLIB::getATAN2(VT: N->getValueType(ResNo: 0)));
413}
414
415SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
416 return SoftenFloatRes_Unary(N, LC: RTLIB::getCBRT(VT: N->getValueType(ResNo: 0)));
417}
418
419SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
420 return SoftenFloatRes_Unary(N, LC: RTLIB::getCEIL(VT: N->getValueType(ResNo: 0)));
421}
422
423SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
424 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
425 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
426 SDLoc dl(N);
427
428 EVT LVT = LHS.getValueType();
429 EVT RVT = RHS.getValueType();
430
431 unsigned LSize = LVT.getSizeInBits();
432 unsigned RSize = RVT.getSizeInBits();
433
434 // First get the sign bit of second operand.
435 SDValue SignBit = DAG.getNode(
436 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
437 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
438 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
439 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
440
441 // Shift right or sign-extend it if the two operands have different types.
442 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
443 if (SizeDiff > 0) {
444 SignBit =
445 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
446 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
447 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
448 DL: DAG.getDataLayout())));
449 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
450 } else if (SizeDiff < 0) {
451 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
452 SignBit =
453 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
454 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
455 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
456 DL: DAG.getDataLayout())));
457 }
458
459 // Clear the sign bit of the first operand.
460 SDValue Mask = DAG.getNode(
461 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
462 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
463 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
464 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
465 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
466
467 // Or the value with the sign bit.
468 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
469}
470
471SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
472 return SoftenFloatRes_Unary(N, LC: RTLIB::getCOS(VT: N->getValueType(ResNo: 0)));
473}
474
475SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
476 return SoftenFloatRes_Unary(N, LC: RTLIB::getCOSH(VT: N->getValueType(ResNo: 0)));
477}
478
479SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
480 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
481 Call_F32: RTLIB::DIV_F32,
482 Call_F64: RTLIB::DIV_F64,
483 Call_F80: RTLIB::DIV_F80,
484 Call_F128: RTLIB::DIV_F128,
485 Call_PPCF128: RTLIB::DIV_PPCF128));
486}
487
488SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
489 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP(VT: N->getValueType(ResNo: 0)));
490}
491
492SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
493 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP2(VT: N->getValueType(ResNo: 0)));
494}
495
496SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
497 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP10(VT: N->getValueType(ResNo: 0)));
498}
499
500SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
501 return SoftenFloatRes_Unary(N, LC: RTLIB::getFLOOR(VT: N->getValueType(ResNo: 0)));
502}
503
504SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
505 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG(VT: N->getValueType(ResNo: 0)));
506}
507
508SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
509 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG2(VT: N->getValueType(ResNo: 0)));
510}
511
512SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
513 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG10(VT: N->getValueType(ResNo: 0)));
514}
515
516SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
517 bool IsStrict = N->isStrictFPOpcode();
518 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
519 unsigned Offset = IsStrict ? 1 : 0;
520 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
521 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
522 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
523 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
524 TargetLowering::MakeLibCallOptions CallOptions;
525 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
526 N->getOperand(Num: 1 + Offset).getValueType(),
527 N->getOperand(Num: 2 + Offset).getValueType() };
528 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
529 RTLIB::Libcall LC = RTLIB::getFMA(VT: N->getValueType(ResNo: 0));
530 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
531 if (LCImpl == RTLIB::Unsupported)
532 return SoftenFloatRes_NoLibcall(N, NVT);
533 std::pair<SDValue, SDValue> Tmp =
534 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
535 if (IsStrict)
536 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
537 return Tmp.first;
538}
539
540SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
541 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
542 Call_F32: RTLIB::MUL_F32,
543 Call_F64: RTLIB::MUL_F64,
544 Call_F80: RTLIB::MUL_F80,
545 Call_F128: RTLIB::MUL_F128,
546 Call_PPCF128: RTLIB::MUL_PPCF128));
547}
548
549SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
550 return SoftenFloatRes_Unary(N, LC: RTLIB::getNEARBYINT(VT: N->getValueType(ResNo: 0)));
551}
552
553SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
554 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
555 SDLoc dl(N);
556
557 // Expand Y = FNEG(X) -> Y = X ^ sign mask
558 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
559 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
560 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
561}
562
563SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
564 bool IsStrict = N->isStrictFPOpcode();
565 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
566 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
567
568 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
569
570 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
571 // -> f32, so proceed in two stages. Also, it's entirely possible for both
572 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
573 // than FP16_TO_FP.
574 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
575 N->getValueType(ResNo: 0) != MVT::f32) {
576 if (IsStrict) {
577 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
578 ResultTys: { MVT::f32, MVT::Other }, Ops: { Chain, Op });
579 Chain = Op.getValue(R: 1);
580 } else {
581 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT: MVT::f32, Operand: Op);
582 }
583 }
584
585 if (Op.getValueType() == MVT::bf16) {
586 // FIXME: Need ReplaceValueWith on chain in strict case
587 return SoftenFloatRes_BF16_TO_FP(N);
588 }
589
590 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
591 if (LC == RTLIB::UNKNOWN_LIBCALL) {
592 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fp_extend");
593 if (IsStrict)
594 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
595 return DAG.getPOISON(VT: NVT);
596 }
597 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
598 if (LCImpl == RTLIB::Unsupported)
599 return SoftenFloatRes_NoLibcall(N, NVT);
600 TargetLowering::MakeLibCallOptions CallOptions;
601 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
602 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
603 std::pair<SDValue, SDValue> Tmp =
604 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
605 if (IsStrict)
606 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
607 return Tmp.first;
608}
609
610// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
611// nodes?
612SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
613 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
614 SDValue Op = N->getOperand(Num: 0);
615 TargetLowering::MakeLibCallOptions CallOptions;
616 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
617 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
618 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
619 CallOptions, dl: SDLoc(N)).first;
620 if (N->getValueType(ResNo: 0) == MVT::f32)
621 return Res32;
622
623 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
624 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: MVT::f32, RetVT: N->getValueType(ResNo: 0));
625 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
626 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
627}
628
629// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
630// nodes?
631SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
632 assert(N->getValueType(0) == MVT::f32 &&
633 "Can only soften BF16_TO_FP with f32 result");
634 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
635 SDValue Op = N->getOperand(Num: 0);
636 SDLoc DL(N);
637 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT,
638 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op));
639 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
640 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
641 return Res;
642}
643
644SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
645 bool IsStrict = N->isStrictFPOpcode();
646 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
647 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
648 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
649 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
650 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
651 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
652 if (LCImpl == RTLIB::Unsupported)
653 return SoftenFloatRes_NoLibcall(N, NVT);
654 TargetLowering::MakeLibCallOptions CallOptions;
655 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
656 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
657 std::pair<SDValue, SDValue> Tmp =
658 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
659 if (IsStrict)
660 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
661 return Tmp.first;
662}
663
664SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
665 return SoftenFloatRes_Binary(N, LC: RTLIB::getPOW(VT: N->getValueType(ResNo: 0)));
666}
667
668SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
669 bool IsStrict = N->isStrictFPOpcode();
670 unsigned Offset = IsStrict ? 1 : 0;
671 bool IsPowI =
672 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
673 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
674
675 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(VT: N->getValueType(ResNo: 0))
676 : RTLIB::getLDEXP(VT: N->getValueType(ResNo: 0));
677 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
678 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
679 if (LCImpl == RTLIB::Unsupported) {
680 // Some targets don't have a powi libcall; use pow instead.
681 // FIXME: Implement this if some target needs it.
682 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fpowi to fpow");
683 if (IsStrict)
684 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
685 return DAG.getPOISON(VT: NVT);
686 }
687
688 if (DAG.getLibInfo().getIntSize() !=
689 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
690 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
691 // would use the wrong type for the argument.
692 DAG.getContext()->emitError(ErrorStr: "powi exponent does not match sizeof(int)");
693 if (IsStrict)
694 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
695 return DAG.getPOISON(VT: NVT);
696 }
697
698 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
699 N->getOperand(Num: 1 + Offset) };
700 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
701 TargetLowering::MakeLibCallOptions CallOptions;
702 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
703 N->getOperand(Num: 1 + Offset).getValueType() };
704 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
705 CallOptions.setIsSigned();
706 std::pair<SDValue, SDValue> Tmp =
707 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
708 if (IsStrict)
709 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
710 return Tmp.first;
711}
712
713SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
714 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
715 EVT VT0 = N->getValueType(ResNo: 0);
716 EVT VT1 = N->getValueType(ResNo: 1);
717 RTLIB::Libcall LC = RTLIB::getFREXP(VT: VT0);
718 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
719 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
720 SDLoc DL(N);
721
722 if (LCImpl == RTLIB::Unsupported) {
723 reportNoLibcall(DAG, N, VT: VT0);
724 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
725 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
726 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
727 }
728
729 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
730 // If the exponent does not match with sizeof(int) a libcall would use the
731 // wrong type for the argument.
732 // TODO: Should be able to handle mismatches.
733 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
734 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
735 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
736 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
737 }
738
739 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
740
741 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
742 TargetLowering::MakeLibCallOptions CallOptions;
743 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
744 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
745 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
746
747 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
748 // but we only really need to handle the 0th one for softening anyway.
749 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0)
750 .setOpsTypeOverrides(CallOpsTypeOverrides);
751
752 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT0, Ops, CallOptions,
753 dl: DL, /*Chain=*/SDValue());
754 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
755 auto PtrInfo =
756 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
757
758 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
759
760 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
761 return ReturnVal;
762}
763
764bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
765 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
766 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
767 EVT VT = N->getValueType(ResNo: 0);
768
769 assert(VT == N->getValueType(1) &&
770 "expected both return values to have the same type");
771
772 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
773 if (LCImpl == RTLIB::Unsupported)
774 return false;
775
776 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
777
778 SDLoc DL(N);
779
780 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(Op: N->getOperand(Num: 0))};
781 SmallVector<EVT, 3> OpsVT = {VT};
782
783 std::array<SDValue, 2> StackSlots;
784 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
785 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
786 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
787 if (ResNum == CallRetResNo)
788 continue;
789 SDValue StackSlot = DAG.CreateStackTemporary(VT: NVT);
790 Ops.push_back(Elt: StackSlot);
791 OpsVT.push_back(Elt: StackSlot.getValueType());
792 StackSlots[ResNum] = StackSlot;
793 CallOpsTypeOverrides.push_back(Elt: PointerTy);
794 }
795
796 TargetLowering::MakeLibCallOptions CallOptions;
797 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
798 // but since both returns have the same type it should be okay.
799 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT)
800 .setOpsTypeOverrides(CallOpsTypeOverrides);
801
802 auto [ReturnVal, Chain] =
803 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: DL,
804 /*Chain=*/SDValue());
805
806 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
807 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
808 auto PtrInfo =
809 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
810 return DAG.getLoad(VT: NVT, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
811 };
812
813 for (auto [ResNum, SlackSlot] : enumerate(First&: StackSlots)) {
814 if (CallRetResNo == ResNum) {
815 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: ReturnVal);
816 continue;
817 }
818 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: CreateStackLoad(SlackSlot));
819 }
820
821 return true;
822}
823
824SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
825 EVT VT = N->getValueType(ResNo: 0);
826 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(VT)))
827 return SDValue();
828
829 // Fall back on softening the separate sin and cos calls if available.
830 RTLIB::Libcall SinLC = RTLIB::getSIN(VT);
831 RTLIB::Libcall CosLC = RTLIB::getCOS(VT);
832
833 SDValue SoftSin, SoftCos;
834 if (DAG.getLibcalls().getLibcallImpl(Call: SinLC) == RTLIB::Unsupported ||
835 DAG.getLibcalls().getLibcallImpl(Call: CosLC) == RTLIB::Unsupported) {
836 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fsincos");
837
838 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
839 SoftSin = SoftCos = DAG.getPOISON(VT: NVT);
840 } else {
841 SoftSin = SoftenFloatRes_Unary(N, LC: SinLC);
842 SoftCos = SoftenFloatRes_Unary(N, LC: CosLC);
843 }
844
845 SetSoftenedFloat(Op: SDValue(N, 0), Result: SoftSin);
846 SetSoftenedFloat(Op: SDValue(N, 1), Result: SoftCos);
847 return SDValue();
848}
849
850SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
851 EVT VT = N->getValueType(ResNo: 0);
852 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT),
853 /*CallRetResNo=*/0))
854 return SDValue();
855
856 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
857 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fmodf");
858 SDValue Poison = DAG.getPOISON(VT: NVT);
859 SetSoftenedFloat(Op: SDValue(N, 0), Result: Poison);
860 SetSoftenedFloat(Op: SDValue(N, 1), Result: Poison);
861 return SDValue();
862}
863
864SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
865 return SoftenFloatRes_Binary(N, LC: RTLIB::getREM(VT: N->getValueType(ResNo: 0)));
866}
867
868SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
869 return SoftenFloatRes_Unary(N, LC: RTLIB::getRINT(VT: N->getValueType(ResNo: 0)));
870}
871
872SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
873 return SoftenFloatRes_Unary(N, LC: RTLIB::getROUND(VT: N->getValueType(ResNo: 0)));
874}
875
876SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
877 return SoftenFloatRes_Unary(N, LC: RTLIB::getROUNDEVEN(VT: N->getValueType(ResNo: 0)));
878}
879
880SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
881 return SoftenFloatRes_Unary(N, LC: RTLIB::getSIN(VT: N->getValueType(ResNo: 0)));
882}
883
884SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
885 return SoftenFloatRes_Unary(N, LC: RTLIB::getSINH(VT: N->getValueType(ResNo: 0)));
886}
887
888SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
889 return SoftenFloatRes_Unary(N, LC: RTLIB::getSQRT(VT: N->getValueType(ResNo: 0)));
890}
891
892SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
893 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
894 Call_F32: RTLIB::SUB_F32,
895 Call_F64: RTLIB::SUB_F64,
896 Call_F80: RTLIB::SUB_F80,
897 Call_F128: RTLIB::SUB_F128,
898 Call_PPCF128: RTLIB::SUB_PPCF128));
899}
900
901SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
902 return SoftenFloatRes_Unary(N, LC: RTLIB::getTAN(VT: N->getValueType(ResNo: 0)));
903}
904
905SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
906 return SoftenFloatRes_Unary(N, LC: RTLIB::getTANH(VT: N->getValueType(ResNo: 0)));
907}
908
909SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
910 return SoftenFloatRes_Unary(N, LC: RTLIB::getTRUNC(VT: N->getValueType(ResNo: 0)));
911}
912
913SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
914 LoadSDNode *L = cast<LoadSDNode>(Val: N);
915 EVT VT = N->getValueType(ResNo: 0);
916 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
917 SDLoc dl(N);
918
919 auto MMOFlags =
920 L->getMemOperand()->getFlags() &
921 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
922 SDValue NewL;
923 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
924 // If softening widens the integer representation (e.g. x86_fp80 -> i96),
925 // load the original memory width and extend to the softened type.
926 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
927 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::EXTLOAD, VT: NVT, dl,
928 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
929 PtrInfo: L->getPointerInfo(), MemVT, Alignment: L->getBaseAlign(), MMOFlags,
930 Metadata: L->getMMOMetadataForSubAccess());
931 // Legalized the chain result - switch anything that used the old chain to
932 // use the new one.
933 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
934 return NewL;
935 }
936
937 // Do a non-extending load followed by FP_EXTEND.
938 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
939 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
940 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(), Alignment: L->getBaseAlign(),
941 MMOFlags, Metadata: L->getMMOMetadataForSubAccess());
942 // Legalized the chain result - switch anything that used the old chain to
943 // use the new one.
944 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
945 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
946 return BitConvertToInteger(Op: ExtendNode);
947}
948
949SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
950 AtomicSDNode *L = cast<AtomicSDNode>(Val: N);
951 EVT VT = N->getValueType(ResNo: 0);
952 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
953 SDLoc dl(N);
954
955 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
956 SDValue NewL =
957 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
958 Ops: {L->getChain(), L->getBasePtr()}, MMO: L->getMemOperand());
959
960 // Legalized the chain result - switch anything that used the old chain to
961 // use the new one.
962 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
963 return NewL;
964 }
965
966 report_fatal_error(reason: "softening fp extending atomic load not handled");
967}
968
969SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
970 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
971 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
972 return DAG.getSelect(DL: SDLoc(N),
973 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
974}
975
976SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
977 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
978 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
979 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
980 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
981 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
982}
983
984SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
985 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
986 VT: N->getValueType(ResNo: 0)));
987}
988
989SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
990 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
991 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
992 EVT VT = N->getValueType(ResNo: 0);
993 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
994 SDLoc dl(N);
995
996 SDValue NewVAARG;
997 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
998 Align: N->getConstantOperandVal(Num: 3));
999
1000 // Legalized the chain result - switch anything that used the old chain to
1001 // use the new one.
1002 if (N != NewVAARG.getValue(R: 1).getNode())
1003 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
1004 return NewVAARG;
1005}
1006
1007SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1008 bool IsStrict = N->isStrictFPOpcode();
1009 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1010 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1011 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
1012 EVT RVT = N->getValueType(ResNo: 0);
1013 EVT NVT = EVT();
1014 SDLoc dl(N);
1015
1016 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1017 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1018 // match. Look for an appropriate libcall.
1019 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1020 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1021 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1022 NVT = (MVT::SimpleValueType)t;
1023 // The source needs to big enough to hold the operand.
1024 if (NVT.bitsGE(VT: SVT))
1025 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
1026 }
1027 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1028
1029 EVT NRVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT);
1030 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1031 if (LCImpl == RTLIB::Unsupported)
1032 return SoftenFloatRes_NoLibcall(N, NVT: NRVT);
1033
1034 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1035 // Sign/zero extend the argument if the libcall takes a larger type.
1036 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
1037 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
1038 TargetLowering::MakeLibCallOptions CallOptions;
1039 CallOptions.setIsSigned(Signed);
1040 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1041 std::pair<SDValue, SDValue> Tmp =
1042 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NRVT, Ops: Op, CallOptions, dl, Chain);
1043
1044 if (IsStrict)
1045 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1046 return Tmp.first;
1047}
1048
1049SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1050 // Expand and soften recursively.
1051 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
1052 return SDValue();
1053}
1054
1055SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1056 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
1057 return SDValue();
1058}
1059
1060//===----------------------------------------------------------------------===//
1061// Convert Float Operand to Integer
1062//===----------------------------------------------------------------------===//
1063
1064bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1065 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1066 SDValue Res = SDValue();
1067
1068 switch (N->getOpcode()) {
1069 default:
1070#ifndef NDEBUG
1071 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1072 N->dump(&DAG); dbgs() << "\n";
1073#endif
1074 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
1075
1076 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1077 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1078 case ISD::STRICT_FP_TO_FP16:
1079 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1080 case ISD::FP_TO_BF16:
1081 case ISD::STRICT_FP_TO_BF16:
1082 case ISD::STRICT_FP_ROUND:
1083 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1084 case ISD::STRICT_FP_TO_SINT:
1085 case ISD::STRICT_FP_TO_UINT:
1086 case ISD::FP_TO_SINT:
1087 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1088 case ISD::FP_TO_SINT_SAT:
1089 case ISD::FP_TO_UINT_SAT:
1090 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1091 case ISD::STRICT_LROUND:
1092 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1093 case ISD::STRICT_LLROUND:
1094 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1095 case ISD::STRICT_LRINT:
1096 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1097 case ISD::STRICT_LLRINT:
1098 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1099 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1100 case ISD::STRICT_FSETCC:
1101 case ISD::STRICT_FSETCCS:
1102 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1103 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1104 case ISD::ATOMIC_STORE:
1105 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1106 break;
1107 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1108 case ISD::FAKE_USE:
1109 Res = SoftenFloatOp_FAKE_USE(N);
1110 break;
1111 case ISD::STACKMAP:
1112 Res = SoftenFloatOp_STACKMAP(N, OpNo);
1113 break;
1114 case ISD::PATCHPOINT:
1115 Res = SoftenFloatOp_PATCHPOINT(N, OpNo);
1116 break;
1117 }
1118
1119 // If the result is null, the sub-method took care of registering results etc.
1120 if (!Res.getNode()) return false;
1121
1122 // If the result is N, the sub-method updated N in place. Tell the legalizer
1123 // core about this to re-analyze.
1124 if (Res.getNode() == N)
1125 return true;
1126
1127 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1128 "Invalid operand softening");
1129
1130 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1131 return false;
1132}
1133
1134SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1135 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
1136
1137 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
1138}
1139
1140SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1141 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1142 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1143 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1144 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1145 N->getOpcode() == ISD::FP_TO_BF16 ||
1146 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1147 N->getOpcode() == ISD::STRICT_FP_ROUND);
1148
1149 bool IsStrict = N->isStrictFPOpcode();
1150 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1151 EVT SVT = Op.getValueType();
1152 EVT RVT = N->getValueType(ResNo: 0);
1153 EVT FloatRVT = RVT;
1154 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1155 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1156 FloatRVT = MVT::f16;
1157 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1158 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1159 FloatRVT = MVT::bf16;
1160
1161 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
1162 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1163
1164 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1165 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1166 if (LCImpl == RTLIB::Unsupported) {
1167 reportNoLibcall(DAG, N, VT: SVT);
1168 SDValue Poison = DAG.getPOISON(VT: RVT);
1169 if (IsStrict) {
1170 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1171 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1172 return SDValue();
1173 }
1174 return Poison;
1175 }
1176 Op = GetSoftenedFloat(Op);
1177 TargetLowering::MakeLibCallOptions CallOptions;
1178 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1179 std::pair<SDValue, SDValue> Tmp =
1180 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1181 if (IsStrict) {
1182 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1183 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1184 return SDValue();
1185 }
1186 return Tmp.first;
1187}
1188
1189SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1190 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1191 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1192
1193 EVT VT = NewLHS.getValueType();
1194 NewLHS = GetSoftenedFloat(Op: NewLHS);
1195 NewRHS = GetSoftenedFloat(Op: NewRHS);
1196 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1197 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1198
1199 // If softenSetCCOperands returned a scalar, we need to compare the result
1200 // against zero to select between true and false values.
1201 if (!NewRHS.getNode()) {
1202 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1203 CCCode = ISD::SETNE;
1204 }
1205
1206 // Update N to have the operands specified.
1207 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1208 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1209 Op5: N->getOperand(Num: 4)),
1210 0);
1211}
1212
1213// Even if the result type is legal, no libcall may exactly match. (e.g. We
1214// don't have FP-i8 conversions) This helper method looks for an appropriate
1215// promoted libcall.
1216static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1217 bool Signed) {
1218 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1219 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1220 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1221 ++IntVT) {
1222 Promoted = (MVT::SimpleValueType)IntVT;
1223 // The type needs to big enough to hold the result.
1224 if (Promoted.bitsGE(VT: RetVT))
1225 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1226 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1227 }
1228 return LC;
1229}
1230
1231SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1232 bool IsStrict = N->isStrictFPOpcode();
1233 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1234 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1235
1236 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1237 EVT SVT = Op.getValueType();
1238 EVT RVT = N->getValueType(ResNo: 0);
1239 EVT NVT = EVT();
1240 SDLoc dl(N);
1241
1242 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1243 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1244 // match, eg. we don't have fp -> i8 conversions.
1245 // Look for an appropriate libcall.
1246 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1247 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1248 "Unsupported FP_TO_XINT!");
1249
1250 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1251 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1252 if (LCImpl == RTLIB::Unsupported) {
1253 reportNoLibcall(DAG, N, VT: SVT);
1254 SDValue Poison = DAG.getPOISON(VT: RVT);
1255 if (IsStrict) {
1256 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1257 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1258 return SDValue();
1259 }
1260 return Poison;
1261 }
1262
1263 Op = GetSoftenedFloat(Op);
1264 TargetLowering::MakeLibCallOptions CallOptions;
1265 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1266 std::pair<SDValue, SDValue> Tmp =
1267 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
1268
1269 // Truncate the result if the libcall returns a larger type.
1270 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1271
1272 if (!IsStrict)
1273 return Res;
1274
1275 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1276 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1277 return SDValue();
1278}
1279
1280SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1281 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1282 return Res;
1283}
1284
1285SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1286 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1287 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1288
1289 EVT VT = NewLHS.getValueType();
1290 NewLHS = GetSoftenedFloat(Op: NewLHS);
1291 NewRHS = GetSoftenedFloat(Op: NewRHS);
1292 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1293 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1294
1295 // If softenSetCCOperands returned a scalar, we need to compare the result
1296 // against zero to select between true and false values.
1297 if (!NewRHS.getNode()) {
1298 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1299 CCCode = ISD::SETNE;
1300 }
1301
1302 // Update N to have the operands specified.
1303 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1304 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1305 Op5: DAG.getCondCode(Cond: CCCode)),
1306 0);
1307}
1308
1309SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1310 bool IsStrict = N->isStrictFPOpcode();
1311 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1312 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1313 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1314 ISD::CondCode CCCode =
1315 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1316
1317 EVT VT = Op0.getValueType();
1318 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1319 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1320 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1321 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1322
1323 // Update N to have the operands specified.
1324 if (NewRHS.getNode()) {
1325 if (IsStrict)
1326 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1327 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1328 else
1329 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1330 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1331 }
1332
1333 // Otherwise, softenSetCCOperands returned a scalar, use it.
1334 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1335 "Unexpected setcc expansion!");
1336
1337 if (IsStrict) {
1338 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1339 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1340 return SDValue();
1341 }
1342 return NewLHS;
1343}
1344
1345SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1346 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1347 assert(OpNo == 1 && "Can only soften the stored value!");
1348 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1349 SDValue Val = ST->getValue();
1350 SDLoc dl(N);
1351
1352 if (ST->isTruncatingStore())
1353 // Do an FP_ROUND followed by a non-truncating store.
1354 Val = BitConvertToInteger(
1355 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1356 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1357 else
1358 Val = GetSoftenedFloat(Op: Val);
1359
1360 // If softening widens the integer representation (e.g. x86_fp80 -> i96),
1361 // truncate the value before storing to preserve the original memory width.
1362 EVT MemVT =
1363 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ST->getMemoryVT().getSizeInBits());
1364 return DAG.getTruncStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(), SVT: MemVT,
1365 MMO: ST->getMemOperand());
1366}
1367
1368SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1369 assert(OpNo == 1 && "Can only soften the stored value!");
1370 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
1371 SDValue Val = ST->getVal();
1372 EVT VT = Val.getValueType();
1373 SDLoc dl(N);
1374
1375 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1376
1377 SDValue NewVal = GetSoftenedFloat(Op: Val);
1378 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: VT, Chain: ST->getChain(), Ptr: NewVal,
1379 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
1380}
1381
1382SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1383 SDValue LHS = N->getOperand(Num: 0);
1384 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1385 SDLoc dl(N);
1386
1387 EVT LVT = LHS.getValueType();
1388 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1389 EVT RVT = RHS.getValueType();
1390
1391 unsigned LSize = LVT.getSizeInBits();
1392 unsigned RSize = RVT.getSizeInBits();
1393
1394 // Shift right or sign-extend it if the two operands have different types.
1395 int SizeDiff = RSize - LSize;
1396 if (SizeDiff > 0) {
1397 RHS =
1398 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1399 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1400 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1401 DL: DAG.getDataLayout())));
1402 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1403 } else if (SizeDiff < 0) {
1404 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1405 RHS =
1406 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1407 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1408 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1409 DL: DAG.getDataLayout())));
1410 }
1411
1412 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1413 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1414}
1415
1416SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1417 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1418 bool IsStrict = N->isStrictFPOpcode();
1419 unsigned Offset = IsStrict ? 1 : 0;
1420 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1421 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1422 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1423 if (LCImpl == RTLIB::Unsupported) {
1424 reportNoLibcall(DAG, N, VT: N->getOperand(Num: 0 + Offset).getValueType());
1425 SDValue Poison = DAG.getPOISON(VT: N->getValueType(ResNo: 0));
1426 if (IsStrict) {
1427 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1428 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1429 return SDValue();
1430 }
1431 return Poison;
1432 }
1433 TargetLowering::MakeLibCallOptions CallOptions;
1434 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1435 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
1436 std::pair<SDValue, SDValue> Tmp =
1437 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1438 if (IsStrict) {
1439 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1440 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1441 return SDValue();
1442 }
1443
1444 return Tmp.first;
1445}
1446
1447SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1448 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1449 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1450 Call_F32: RTLIB::LROUND_F32,
1451 Call_F64: RTLIB::LROUND_F64,
1452 Call_F80: RTLIB::LROUND_F80,
1453 Call_F128: RTLIB::LROUND_F128,
1454 Call_PPCF128: RTLIB::LROUND_PPCF128));
1455}
1456
1457SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1458 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1459 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1460 Call_F32: RTLIB::LLROUND_F32,
1461 Call_F64: RTLIB::LLROUND_F64,
1462 Call_F80: RTLIB::LLROUND_F80,
1463 Call_F128: RTLIB::LLROUND_F128,
1464 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1465}
1466
1467SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1468 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1469 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1470 Call_F32: RTLIB::LRINT_F32,
1471 Call_F64: RTLIB::LRINT_F64,
1472 Call_F80: RTLIB::LRINT_F80,
1473 Call_F128: RTLIB::LRINT_F128,
1474 Call_PPCF128: RTLIB::LRINT_PPCF128));
1475}
1476
1477SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1478 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1479 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1480 Call_F32: RTLIB::LLRINT_F32,
1481 Call_F64: RTLIB::LLRINT_F64,
1482 Call_F80: RTLIB::LLRINT_F80,
1483 Call_F128: RTLIB::LLRINT_F128,
1484 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1485}
1486
1487SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1488 SDValue Op1 = BitConvertToInteger(Op: N->getOperand(Num: 1));
1489 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1490 N1: N->getOperand(Num: 0), N2: Op1);
1491}
1492
1493SDValue DAGTypeLegalizer::SoftenFloatOp_STACKMAP(SDNode *N, unsigned OpNo) {
1494 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
1495 SmallVector<SDValue> NewOps(N->ops());
1496 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1497 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1498}
1499
1500SDValue DAGTypeLegalizer::SoftenFloatOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
1501 assert(OpNo >= 7);
1502 SmallVector<SDValue> NewOps(N->ops());
1503 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1504 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1505}
1506
1507//===----------------------------------------------------------------------===//
1508// Float Result Expansion
1509//===----------------------------------------------------------------------===//
1510
1511/// ExpandFloatResult - This method is called when the specified result of the
1512/// specified node is found to need expansion. At this point, the node may also
1513/// have invalid operands or may have other results that need promotion, we just
1514/// know that (at least) one result needs expansion.
1515void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1516 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1517 SDValue Lo, Hi;
1518 Lo = Hi = SDValue();
1519
1520 // See if the target wants to custom expand this node.
1521 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1522 return;
1523
1524 switch (N->getOpcode()) {
1525 default:
1526#ifndef NDEBUG
1527 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1528 N->dump(&DAG); dbgs() << "\n";
1529#endif
1530 report_fatal_error(reason: "do not know how to expand the result of this "
1531 "operator!");
1532 // clang-format off
1533 case ISD::POISON:
1534 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1535 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1536 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1537
1538 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1539 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1540 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1541 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1542 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1543 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1544
1545 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1546 case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
1547 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1548 case ISD::STRICT_FMINNUM:
1549 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1550 case ISD::STRICT_FMAXNUM:
1551 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1552 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1553 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1554 case ISD::STRICT_FADD:
1555 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1556 case ISD::STRICT_FACOS:
1557 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1558 case ISD::STRICT_FASIN:
1559 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1560 case ISD::STRICT_FATAN:
1561 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1562 case ISD::STRICT_FATAN2:
1563 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1564 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1565 case ISD::STRICT_FCEIL:
1566 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1567 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1568 case ISD::STRICT_FCOS:
1569 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1570 case ISD::STRICT_FCOSH:
1571 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1572 case ISD::STRICT_FDIV:
1573 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1574 case ISD::STRICT_FEXP:
1575 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1576 case ISD::STRICT_FEXP2:
1577 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1578 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1579 case ISD::STRICT_FFLOOR:
1580 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1581 case ISD::STRICT_FLOG:
1582 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1583 case ISD::STRICT_FLOG2:
1584 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1585 case ISD::STRICT_FLOG10:
1586 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1587 case ISD::STRICT_FMA:
1588 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1589 case ISD::STRICT_FMUL:
1590 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1591 case ISD::STRICT_FNEARBYINT:
1592 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1593 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1594 case ISD::STRICT_FP_EXTEND:
1595 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1596 case ISD::STRICT_FPOW:
1597 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1598 case ISD::STRICT_FPOWI:
1599 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1600 case ISD::FLDEXP:
1601 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1602 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1603 case ISD::STRICT_FRINT:
1604 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1605 case ISD::STRICT_FROUND:
1606 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1607 case ISD::STRICT_FROUNDEVEN:
1608 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1609 case ISD::STRICT_FSIN:
1610 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1611 case ISD::STRICT_FSINH:
1612 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1613 case ISD::STRICT_FSQRT:
1614 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1615 case ISD::STRICT_FSUB:
1616 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1617 case ISD::STRICT_FTAN:
1618 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1619 case ISD::STRICT_FTANH:
1620 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1621 case ISD::STRICT_FTRUNC:
1622 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1623 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1624 case ISD::STRICT_SINT_TO_FP:
1625 case ISD::STRICT_UINT_TO_FP:
1626 case ISD::SINT_TO_FP:
1627 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1628 case ISD::STRICT_FREM:
1629 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1630 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1631 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1632 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1633 // clang-format on
1634 }
1635
1636 // If Lo/Hi is null, the sub-method took care of registering results etc.
1637 if (Lo.getNode())
1638 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1639}
1640
1641void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1642 SDValue &Hi) {
1643 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1644 assert(NVT.getSizeInBits() == 64 &&
1645 "Do not know how to expand this float constant!");
1646 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1647 SDLoc dl(N);
1648 const fltSemantics &Sem = NVT.getFltSemantics();
1649 Lo = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 64)), DL: dl, VT: NVT);
1650 Hi = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 0)), DL: dl, VT: NVT);
1651}
1652
1653// Diagnose a missing libcall and produce a poison expanded pair.
1654void DAGTypeLegalizer::ExpandFloatRes_NoLibcall(SDNode *N, SDValue &Lo,
1655 SDValue &Hi) {
1656 reportNoLibcall(DAG, N, VT: N->getValueType(ResNo: 0));
1657 if (N->isStrictFPOpcode())
1658 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
1659 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1660 Lo = Hi = DAG.getPOISON(VT: NVT);
1661}
1662
1663void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1664 SDValue &Lo, SDValue &Hi) {
1665 bool IsStrict = N->isStrictFPOpcode();
1666 unsigned Offset = IsStrict ? 1 : 0;
1667 SDValue Op = N->getOperand(Num: 0 + Offset);
1668 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1669 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1670 if (LCImpl == RTLIB::Unsupported)
1671 return ExpandFloatRes_NoLibcall(N, Lo, Hi);
1672 TargetLowering::MakeLibCallOptions CallOptions;
1673 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
1674 DAG, LibcallImpl: LCImpl, RetVT: N->getValueType(ResNo: 0), Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1675 if (IsStrict)
1676 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1677 GetPairElements(Pair: Tmp.first, Lo, Hi);
1678}
1679
1680void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1681 SDValue &Lo, SDValue &Hi) {
1682 bool IsStrict = N->isStrictFPOpcode();
1683 unsigned Offset = IsStrict ? 1 : 0;
1684 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1685 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1686 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1687 if (LCImpl == RTLIB::Unsupported)
1688 return ExpandFloatRes_NoLibcall(N, Lo, Hi);
1689 TargetLowering::MakeLibCallOptions CallOptions;
1690 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
1691 DAG, LibcallImpl: LCImpl, RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain);
1692 if (IsStrict)
1693 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1694 GetPairElements(Pair: Tmp.first, Lo, Hi);
1695}
1696
1697void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1698 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT: N->getValueType(ResNo: 0)),
1699 /*CallRetResNo=*/0);
1700}
1701
1702void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1703 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(VT: N->getValueType(ResNo: 0)));
1704}
1705
1706void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1707 ExpandFloatRes_UnaryWithTwoFPResults(N,
1708 LC: RTLIB::getSINCOSPI(VT: N->getValueType(ResNo: 0)));
1709}
1710
1711void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1712 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1713 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1714 SmallVector<SDValue> Results;
1715 if (!TLI.expandMultipleResultFPLibCall(DAG, LC, Node: N, Results, CallRetResNo)) {
1716 reportNoLibcall(DAG, N, VT: N->getValueType(ResNo: 0));
1717 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1718 SDValue Poison = DAG.getPOISON(VT: NVT);
1719 for (unsigned ResNo = 0, E = N->getNumValues(); ResNo != E; ++ResNo)
1720 SetExpandedFloat(Op: SDValue(N, ResNo), Lo: Poison, Hi: Poison);
1721 return;
1722 }
1723 for (auto [ResNo, Res] : enumerate(First&: Results)) {
1724 SDValue Lo, Hi;
1725 GetPairElements(Pair: Res, Lo, Hi);
1726 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1727 }
1728}
1729
1730void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1731 SDValue &Hi) {
1732 assert(N->getValueType(0) == MVT::ppcf128 &&
1733 "Logic only correct for ppcf128!");
1734 SDLoc dl(N);
1735 SDValue Tmp;
1736 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1737 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1738 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1739 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1740 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1741 Cond: ISD::SETEQ);
1742}
1743
1744void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1745 SDValue &Hi) {
1746 ExpandFloatRes_Binary(N, LC: RTLIB::getFMIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1747}
1748
1749void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1750 SDValue &Hi) {
1751 ExpandFloatRes_Binary(N, LC: RTLIB::getFMAX(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1752}
1753
1754void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1755 SDValue &Hi) {
1756 ExpandFloatRes_Binary(N, LC: RTLIB::getFMINIMUM_NUM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1757}
1758
1759void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1760 SDValue &Hi) {
1761 ExpandFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM_NUM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1762}
1763
1764void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1765 SDValue &Hi) {
1766 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1767 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1768 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1769 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1770}
1771
1772void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1773 SDValue &Hi) {
1774 ExpandFloatRes_Unary(N, LC: RTLIB::getACOS(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1775}
1776
1777void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1778 SDValue &Hi) {
1779 ExpandFloatRes_Unary(N, LC: RTLIB::getASIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1780}
1781
1782void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1783 SDValue &Hi) {
1784 ExpandFloatRes_Unary(N, LC: RTLIB::getATAN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1785}
1786
1787void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1788 SDValue &Hi) {
1789 ExpandFloatRes_Binary(N, LC: RTLIB::getATAN2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1790}
1791
1792void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1793 SDValue &Hi) {
1794 ExpandFloatRes_Unary(N, LC: RTLIB::getCBRT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1795}
1796
1797void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1798 SDValue &Lo, SDValue &Hi) {
1799 ExpandFloatRes_Unary(N, LC: RTLIB::getCEIL(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1800}
1801
1802void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1803 SDValue &Lo, SDValue &Hi) {
1804 ExpandFloatRes_Binary(N, LC: RTLIB::getCOPYSIGN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1805}
1806
1807void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1808 SDValue &Lo, SDValue &Hi) {
1809 ExpandFloatRes_Unary(N, LC: RTLIB::getCOS(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1810}
1811
1812void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1813 SDValue &Hi) {
1814 ExpandFloatRes_Unary(N, LC: RTLIB::getCOSH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1815}
1816
1817void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1818 SDValue &Hi) {
1819 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1820 Call_F32: RTLIB::DIV_F32,
1821 Call_F64: RTLIB::DIV_F64,
1822 Call_F80: RTLIB::DIV_F80,
1823 Call_F128: RTLIB::DIV_F128,
1824 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1825}
1826
1827void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1828 SDValue &Lo, SDValue &Hi) {
1829 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1830}
1831
1832void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1833 SDValue &Lo, SDValue &Hi) {
1834 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1835}
1836
1837void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1838 SDValue &Hi) {
1839 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP10(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1840}
1841
1842void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1843 SDValue &Lo, SDValue &Hi) {
1844 ExpandFloatRes_Unary(N, LC: RTLIB::getFLOOR(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1845}
1846
1847void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1848 SDValue &Lo, SDValue &Hi) {
1849 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1850}
1851
1852void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1853 SDValue &Lo, SDValue &Hi) {
1854 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1855}
1856
1857void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1858 SDValue &Lo, SDValue &Hi) {
1859 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG10(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1860}
1861
1862void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1863 SDValue &Hi) {
1864 bool IsStrict = N->isStrictFPOpcode();
1865 unsigned Offset = IsStrict ? 1 : 0;
1866 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1867 N->getOperand(Num: 2 + Offset) };
1868 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1869 RTLIB::Libcall LC = RTLIB::getFMA(VT: N->getValueType(ResNo: 0));
1870 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1871 if (LCImpl == RTLIB::Unsupported)
1872 return ExpandFloatRes_NoLibcall(N, Lo, Hi);
1873 TargetLowering::MakeLibCallOptions CallOptions;
1874 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
1875 DAG, LibcallImpl: LCImpl, RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain);
1876 if (IsStrict)
1877 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1878 GetPairElements(Pair: Tmp.first, Lo, Hi);
1879}
1880
1881void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1882 SDValue &Hi) {
1883 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1884 Call_F32: RTLIB::MUL_F32,
1885 Call_F64: RTLIB::MUL_F64,
1886 Call_F80: RTLIB::MUL_F80,
1887 Call_F128: RTLIB::MUL_F128,
1888 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1889}
1890
1891void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1892 SDValue &Lo, SDValue &Hi) {
1893 ExpandFloatRes_Unary(N, LC: RTLIB::getNEARBYINT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1894}
1895
1896void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1897 SDValue &Hi) {
1898 SDLoc dl(N);
1899 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1900 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1901 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1902}
1903
1904void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
1905 SDValue &Hi) {
1906 // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
1907 SDLoc dl(N);
1908 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1909}
1910
1911void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1912 SDValue &Hi) {
1913 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1914 SDLoc dl(N);
1915 bool IsStrict = N->isStrictFPOpcode();
1916
1917 SDValue Chain;
1918 if (IsStrict) {
1919 // If the expanded type is the same as the input type, just bypass the node.
1920 if (NVT == N->getOperand(Num: 1).getValueType()) {
1921 Hi = N->getOperand(Num: 1);
1922 Chain = N->getOperand(Num: 0);
1923 } else {
1924 // Other we need to extend.
1925 Hi = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: { NVT, MVT::Other },
1926 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
1927 Chain = Hi.getValue(R: 1);
1928 }
1929 } else {
1930 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1931 }
1932
1933 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
1934
1935 if (IsStrict)
1936 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1937}
1938
1939void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1940 SDValue &Lo, SDValue &Hi) {
1941 ExpandFloatRes_Binary(N, LC: RTLIB::getPOW(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1942}
1943
1944void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1945 SDValue &Lo, SDValue &Hi) {
1946 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1947}
1948
1949void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1950 SDValue &Hi) {
1951 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1952}
1953
1954void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1955 SDValue &Lo, SDValue &Hi) {
1956 assert(N->getValueType(0) == MVT::ppcf128 &&
1957 "Logic only correct for ppcf128!");
1958
1959 SDLoc dl(N);
1960 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1961 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1962 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1963}
1964
1965void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1966 SDValue &Lo, SDValue &Hi) {
1967 ExpandFloatRes_Binary(N, LC: RTLIB::getREM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1968}
1969
1970void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1971 SDValue &Lo, SDValue &Hi) {
1972 ExpandFloatRes_Unary(N, LC: RTLIB::getRINT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1973}
1974
1975void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1976 SDValue &Lo, SDValue &Hi) {
1977 ExpandFloatRes_Unary(N, LC: RTLIB::getROUND(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1978}
1979
1980void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1981 SDValue &Lo, SDValue &Hi) {
1982 ExpandFloatRes_Unary(N, LC: RTLIB::getROUNDEVEN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1983}
1984
1985void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1986 SDValue &Lo, SDValue &Hi) {
1987 ExpandFloatRes_Unary(N, LC: RTLIB::getSIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1988}
1989
1990void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
1991 SDValue &Hi) {
1992 ExpandFloatRes_Unary(N, LC: RTLIB::getSINH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1993}
1994
1995void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1996 SDValue &Lo, SDValue &Hi) {
1997 ExpandFloatRes_Unary(N, LC: RTLIB::getSQRT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1998}
1999
2000void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
2001 SDValue &Hi) {
2002 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2003 Call_F32: RTLIB::SUB_F32,
2004 Call_F64: RTLIB::SUB_F64,
2005 Call_F80: RTLIB::SUB_F80,
2006 Call_F128: RTLIB::SUB_F128,
2007 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
2008}
2009
2010void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
2011 SDValue &Hi) {
2012 ExpandFloatRes_Unary(N, LC: RTLIB::getTAN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
2013}
2014
2015void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
2016 SDValue &Hi) {
2017 ExpandFloatRes_Unary(N, LC: RTLIB::getTANH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
2018}
2019
2020void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
2021 SDValue &Lo, SDValue &Hi) {
2022 ExpandFloatRes_Unary(N, LC: RTLIB::getTRUNC(VT: N->getValueType(ResNo: 0)), Lo, Hi);
2023}
2024
2025void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
2026 SDValue &Hi) {
2027 if (ISD::isNormalLoad(N)) {
2028 ExpandRes_NormalLoad(N, Lo, Hi);
2029 return;
2030 }
2031
2032 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2033 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
2034 SDValue Chain = LD->getChain();
2035 SDValue Ptr = LD->getBasePtr();
2036 SDLoc dl(N);
2037
2038 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
2039 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2040 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2041
2042 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
2043 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
2044
2045 // Remember the chain.
2046 Chain = Hi.getValue(R: 1);
2047
2048 // The low part is zero.
2049 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2050
2051 // Modified the chain - switch anything that used the old chain to use the
2052 // new one.
2053 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
2054}
2055
2056void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2057 SDValue &Hi) {
2058 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2059 EVT VT = N->getValueType(ResNo: 0);
2060 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2061 bool Strict = N->isStrictFPOpcode();
2062 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
2063 EVT SrcVT = Src.getValueType();
2064 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2065 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2066 SDLoc dl(N);
2067 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
2068
2069 // TODO: Any other flags to propagate?
2070 SDNodeFlags Flags;
2071 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2072
2073 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2074 // When promoting partial word types to i32 we must honor the signedness,
2075 // though.
2076 if (SrcVT.bitsLE(VT: MVT::i32)) {
2077 // The integer can be represented exactly in an f64.
2078 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2079 if (Strict) {
2080 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
2081 Ops: {Chain, Src}, Flags);
2082 Chain = Hi.getValue(R: 1);
2083 } else
2084 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
2085 } else {
2086 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2087 if (SrcVT.bitsLE(VT: MVT::i64)) {
2088 Src = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
2089 VT: MVT::i64, Operand: Src);
2090 LC = RTLIB::SINTTOFP_I64_PPCF128;
2091 } else if (SrcVT.bitsLE(VT: MVT::i128)) {
2092 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i128, Operand: Src);
2093 LC = RTLIB::SINTTOFP_I128_PPCF128;
2094 }
2095 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2096
2097 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2098 if (LCImpl == RTLIB::Unsupported)
2099 return ExpandFloatRes_NoLibcall(N, Lo, Hi);
2100
2101 TargetLowering::MakeLibCallOptions CallOptions;
2102 CallOptions.setIsSigned(true);
2103 std::pair<SDValue, SDValue> Tmp =
2104 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
2105 if (Strict)
2106 Chain = Tmp.second;
2107 GetPairElements(Pair: Tmp.first, Lo, Hi);
2108 }
2109
2110 // No need to complement for unsigned 32-bit integers
2111 if (isSigned || SrcVT.bitsLE(VT: MVT::i32)) {
2112 if (Strict)
2113 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2114
2115 return;
2116 }
2117
2118 // Unsigned - fix up the SINT_TO_FP value just calculated.
2119 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2120 // keep semantics correctness if the integer is not exactly representable
2121 // here. See ExpandLegalINT_TO_FP.
2122 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
2123 SrcVT = Src.getValueType();
2124
2125 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2126 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2127 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2128 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2129 ArrayRef<uint64_t> Parts;
2130
2131 switch (SrcVT.getSimpleVT().SimpleTy) {
2132 default:
2133 llvm_unreachable("Unsupported UINT_TO_FP!");
2134 case MVT::i32:
2135 Parts = TwoE32;
2136 break;
2137 case MVT::i64:
2138 Parts = TwoE64;
2139 break;
2140 case MVT::i128:
2141 Parts = TwoE128;
2142 break;
2143 }
2144
2145 // TODO: Are there other fast-math-flags to propagate to this FADD?
2146 SDValue NewLo = DAG.getConstantFP(
2147 Val: APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), DL: dl, VT: MVT::ppcf128);
2148 if (Strict) {
2149 Lo = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other),
2150 Ops: {Chain, Hi, NewLo}, Flags);
2151 Chain = Lo.getValue(R: 1);
2152 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2153 } else
2154 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
2155 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
2156 True: Lo, False: Hi, Cond: ISD::SETLT);
2157 GetPairElements(Pair: Lo, Lo, Hi);
2158}
2159
2160
2161//===----------------------------------------------------------------------===//
2162// Float Operand Expansion
2163//===----------------------------------------------------------------------===//
2164
2165/// ExpandFloatOperand - This method is called when the specified operand of the
2166/// specified node is found to need expansion. At this point, all of the result
2167/// types of the node are known to be legal, but other operands of the node may
2168/// need promotion or expansion as well as the specified one.
2169bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2170 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2171 SDValue Res = SDValue();
2172
2173 // See if the target wants to custom expand this node.
2174 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2175 return false;
2176
2177 switch (N->getOpcode()) {
2178 default:
2179#ifndef NDEBUG
2180 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2181 N->dump(&DAG); dbgs() << "\n";
2182#endif
2183 report_fatal_error(reason: "do not know how to expand this operator's operand!");
2184
2185 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2186 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2187 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2188
2189 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2190 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2191 case ISD::STRICT_FP_ROUND:
2192 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2193 case ISD::STRICT_FP_TO_SINT:
2194 case ISD::STRICT_FP_TO_UINT:
2195 case ISD::FP_TO_SINT:
2196 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2197 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2198 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2199 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2200 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2201 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2202 case ISD::STRICT_FSETCC:
2203 case ISD::STRICT_FSETCCS:
2204 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2205 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
2206 OpNo); break;
2207 }
2208
2209 // If the result is null, the sub-method took care of registering results etc.
2210 if (!Res.getNode()) return false;
2211
2212 // If the result is N, the sub-method updated N in place. Tell the legalizer
2213 // core about this.
2214 if (Res.getNode() == N)
2215 return true;
2216
2217 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2218 "Invalid operand expansion");
2219
2220 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2221 return false;
2222}
2223
2224/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2225/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2226void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2227 SDValue &NewRHS,
2228 ISD::CondCode &CCCode,
2229 const SDLoc &dl, SDValue &Chain,
2230 bool IsSignaling) {
2231 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2232 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
2233 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
2234
2235 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2236
2237 // FIXME: This generated code sucks. We want to generate
2238 // FCMPU crN, hi1, hi2
2239 // BNE crN, L:
2240 // FCMPU crN, lo1, lo2
2241 // The following can be improved, but not that much.
2242 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2243 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2244 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
2245 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2246 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
2247 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
2248 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2249 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2250 Tmp1 =
2251 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
2252 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
2253 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2254 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2255 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
2256 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2257 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2258 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
2259 NewRHS = SDValue(); // LHS is the result, not a compare.
2260 Chain = OutputChain;
2261}
2262
2263SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2264 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
2265 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
2266 SDValue Chain;
2267 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2268
2269 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2270 // against zero to select between true and false values.
2271 if (!NewRHS.getNode()) {
2272 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2273 CCCode = ISD::SETNE;
2274 }
2275
2276 // Update N to have the operands specified.
2277 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2278 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
2279 Op5: N->getOperand(Num: 4)), 0);
2280}
2281
2282SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2283 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2284 "Logic only correct for ppcf128!");
2285 SDValue Lo, Hi;
2286 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2287 // The ppcf128 value is providing only the sign; take it from the
2288 // higher-order double (which must have the larger magnitude).
2289 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2290 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2291}
2292
2293SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2294 bool IsStrict = N->isStrictFPOpcode();
2295 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2296 "Logic only correct for ppcf128!");
2297 SDValue Lo, Hi;
2298 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2299
2300 if (!IsStrict)
2301 // Round it the rest of the way (e.g. to f32) if needed.
2302 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2303 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2304
2305 // Eliminate the node if the input float type is the same as the output float
2306 // type.
2307 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2308 // Connect the output chain to the input chain, unlinking the node.
2309 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2310 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2311 return SDValue();
2312 }
2313
2314 SDValue Expansion = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
2315 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
2316 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
2317 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2318 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2319 return SDValue();
2320}
2321
2322SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2323 EVT RVT = N->getValueType(ResNo: 0);
2324 SDLoc dl(N);
2325
2326 bool IsStrict = N->isStrictFPOpcode();
2327 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2328 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2329 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2330 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2331
2332 EVT NVT;
2333 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2334 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2335 "Unsupported FP_TO_XINT!");
2336 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2337 if (LCImpl == RTLIB::Unsupported) {
2338 reportNoLibcall(DAG, N, VT: Op.getValueType());
2339 SDValue Poison = DAG.getPOISON(VT: RVT);
2340 if (IsStrict) {
2341 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2342 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
2343 return SDValue();
2344 }
2345 return Poison;
2346 }
2347 TargetLowering::MakeLibCallOptions CallOptions;
2348 std::pair<SDValue, SDValue> Tmp =
2349 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2350 if (!IsStrict)
2351 return Tmp.first;
2352
2353 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2354 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2355 return SDValue();
2356}
2357
2358SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2359 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2360 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2361 SDValue Chain;
2362 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2363
2364 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2365 // against zero to select between true and false values.
2366 if (!NewRHS.getNode()) {
2367 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2368 CCCode = ISD::SETNE;
2369 }
2370
2371 // Update N to have the operands specified.
2372 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2373 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2374 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2375}
2376
2377SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2378 bool IsStrict = N->isStrictFPOpcode();
2379 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2380 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2381 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2382 ISD::CondCode CCCode =
2383 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2384 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2385 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2386
2387 // FloatExpandSetCCOperands always returned a scalar.
2388 assert(!NewRHS.getNode() && "Expect to return scalar");
2389 assert(NewLHS.getValueType() == N->getValueType(0) &&
2390 "Unexpected setcc expansion!");
2391 if (Chain) {
2392 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2393 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2394 return SDValue();
2395 }
2396 return NewLHS;
2397}
2398
2399SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2400 if (ISD::isNormalStore(N))
2401 return ExpandOp_NormalStore(N, OpNo);
2402
2403 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2404 assert(OpNo == 1 && "Can only expand the stored value so far");
2405 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2406
2407 SDValue Chain = ST->getChain();
2408 SDValue Ptr = ST->getBasePtr();
2409
2410 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2411 VT: ST->getValue().getValueType());
2412 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2413 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2414 (void)NVT;
2415
2416 SDValue Lo, Hi;
2417 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2418
2419 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2420 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2421}
2422
2423SDValue DAGTypeLegalizer::ExpandFloatOp_XRINT_XROUND(SDNode *N,
2424 RTLIB::Libcall LC) {
2425 EVT RVT = N->getValueType(ResNo: 0);
2426 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2427 if (LCImpl == RTLIB::Unsupported) {
2428 reportNoLibcall(DAG, N, VT: N->getOperand(Num: 0).getValueType());
2429 return DAG.getPOISON(VT: RVT);
2430 }
2431
2432 TargetLowering::MakeLibCallOptions CallOptions;
2433 return TLI
2434 .makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N))
2435 .first;
2436}
2437
2438SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2439 EVT RetVT = N->getOperand(Num: 0).getValueType();
2440 return ExpandFloatOp_XRINT_XROUND(
2441 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LROUND_F32, Call_F64: RTLIB::LROUND_F64,
2442 Call_F80: RTLIB::LROUND_F80, Call_F128: RTLIB::LROUND_F128,
2443 Call_PPCF128: RTLIB::LROUND_PPCF128));
2444}
2445
2446SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2447 EVT RetVT = N->getOperand(Num: 0).getValueType();
2448 return ExpandFloatOp_XRINT_XROUND(
2449 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LLROUND_F32, Call_F64: RTLIB::LLROUND_F64,
2450 Call_F80: RTLIB::LLROUND_F80, Call_F128: RTLIB::LLROUND_F128,
2451 Call_PPCF128: RTLIB::LLROUND_PPCF128));
2452}
2453
2454SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2455 EVT RetVT = N->getOperand(Num: 0).getValueType();
2456 return ExpandFloatOp_XRINT_XROUND(
2457 N,
2458 LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LRINT_F32, Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
2459 Call_F128: RTLIB::LRINT_F128, Call_PPCF128: RTLIB::LRINT_PPCF128));
2460}
2461
2462SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2463 EVT RetVT = N->getOperand(Num: 0).getValueType();
2464 return ExpandFloatOp_XRINT_XROUND(
2465 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LLRINT_F32, Call_F64: RTLIB::LLRINT_F64,
2466 Call_F80: RTLIB::LLRINT_F80, Call_F128: RTLIB::LLRINT_F128,
2467 Call_PPCF128: RTLIB::LLRINT_PPCF128));
2468}
2469
2470//===----------------------------------------------------------------------===//
2471// Float Operand Promotion
2472//===----------------------------------------------------------------------===//
2473//
2474
2475static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2476 if (OpVT == MVT::f16)
2477 return ISD::FP16_TO_FP;
2478 if (RetVT == MVT::f16)
2479 return ISD::FP_TO_FP16;
2480 if (OpVT == MVT::bf16)
2481 return ISD::BF16_TO_FP;
2482 if (RetVT == MVT::bf16)
2483 return ISD::FP_TO_BF16;
2484 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2485}
2486
2487static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2488 if (OpVT == MVT::f16)
2489 return ISD::STRICT_FP16_TO_FP;
2490 if (RetVT == MVT::f16)
2491 return ISD::STRICT_FP_TO_FP16;
2492 if (OpVT == MVT::bf16)
2493 return ISD::STRICT_BF16_TO_FP;
2494 if (RetVT == MVT::bf16)
2495 return ISD::STRICT_FP_TO_BF16;
2496 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2497}
2498
2499SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2500 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2501 SDLoc SL(N);
2502
2503 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2504 EVT CastVT = CastVal.getValueType();
2505
2506 SDValue NewAtomic
2507 = DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl: SL, MemVT: CastVT,
2508 VTList: DAG.getVTList(VT1: CastVT, VT2: MVT::Other),
2509 Ops: { AM->getChain(), AM->getBasePtr(), CastVal },
2510 MMO: AM->getMemOperand());
2511
2512 SDValue Result = NewAtomic;
2513
2514 // Legalize the chain result by replacing uses of the old value chain with the
2515 // new one
2516 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2517
2518 return Result;
2519}
2520
2521//===----------------------------------------------------------------------===//
2522// Half Result Soft Promotion
2523//===----------------------------------------------------------------------===//
2524
2525void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2526 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2527 N->dump(&DAG));
2528 SDValue R = SDValue();
2529
2530 // See if the target wants to custom expand this node.
2531 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2532 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2533 return;
2534 }
2535
2536 switch (N->getOpcode()) {
2537 default:
2538#ifndef NDEBUG
2539 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2540 N->dump(&DAG); dbgs() << "\n";
2541#endif
2542 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2543 "result!");
2544
2545 case ISD::ARITH_FENCE:
2546 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2547 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2548 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2549 case ISD::EXTRACT_VECTOR_ELT:
2550 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2551 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2552 case ISD::STRICT_FP_ROUND:
2553 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2554
2555 // Unary FP Operations
2556 case ISD::FACOS:
2557 case ISD::FASIN:
2558 case ISD::FATAN:
2559 case ISD::FCBRT:
2560 case ISD::FCEIL:
2561 case ISD::FCOS:
2562 case ISD::FCOSH:
2563 case ISD::FEXP:
2564 case ISD::FEXP2:
2565 case ISD::FEXP10:
2566 case ISD::FFLOOR:
2567 case ISD::FLOG:
2568 case ISD::FLOG2:
2569 case ISD::FLOG10:
2570 case ISD::FNEARBYINT:
2571 case ISD::FREEZE:
2572 case ISD::FRINT:
2573 case ISD::FROUND:
2574 case ISD::FROUNDEVEN:
2575 case ISD::FSIN:
2576 case ISD::FSINH:
2577 case ISD::FSQRT:
2578 case ISD::FTRUNC:
2579 case ISD::FTAN:
2580 case ISD::FTANH:
2581 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2582 case ISD::FABS:
2583 R = SoftPromoteHalfRes_FABS(N);
2584 break;
2585 case ISD::FNEG:
2586 R = SoftPromoteHalfRes_FNEG(N);
2587 break;
2588 case ISD::AssertNoFPClass:
2589 R = SoftPromoteHalfRes_AssertNoFPClass(N);
2590 break;
2591
2592 // Binary FP Operations
2593 case ISD::FADD:
2594 case ISD::FDIV:
2595 case ISD::FMAXIMUM:
2596 case ISD::FMINIMUM:
2597 case ISD::FMAXIMUMNUM:
2598 case ISD::FMINIMUMNUM:
2599 case ISD::FMAXNUM:
2600 case ISD::FMINNUM:
2601 case ISD::FMUL:
2602 case ISD::FPOW:
2603 case ISD::FATAN2:
2604 case ISD::FREM:
2605 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2606
2607 case ISD::FMA: // FMA is same as FMAD
2608 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2609
2610 case ISD::FPOWI:
2611 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2612
2613 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2614
2615 case ISD::FMODF:
2616 case ISD::FSINCOS:
2617 case ISD::FSINCOSPI:
2618 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
2619 break;
2620
2621 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2622 case ISD::ATOMIC_LOAD:
2623 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2624 break;
2625 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2626 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2627 case ISD::STRICT_SINT_TO_FP:
2628 case ISD::STRICT_UINT_TO_FP:
2629 case ISD::SINT_TO_FP:
2630 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2631 case ISD::CONVERT_FROM_ARBITRARY_FP:
2632 R = SoftPromoteHalfRes_CONVERT_FROM_ARBITRARY_FP(N);
2633 break;
2634 case ISD::POISON:
2635 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2636 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2637 case ISD::VECREDUCE_FADD:
2638 case ISD::VECREDUCE_FMUL:
2639 case ISD::VECREDUCE_FMIN:
2640 case ISD::VECREDUCE_FMAX:
2641 case ISD::VECREDUCE_FMAXIMUM:
2642 case ISD::VECREDUCE_FMINIMUM:
2643 case ISD::VECREDUCE_FMAXIMUMNUM:
2644 case ISD::VECREDUCE_FMINIMUMNUM:
2645 R = SoftPromoteHalfRes_VECREDUCE(N);
2646 break;
2647 case ISD::VECREDUCE_SEQ_FADD:
2648 case ISD::VECREDUCE_SEQ_FMUL:
2649 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2650 break;
2651 }
2652
2653 if (R.getNode())
2654 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
2655}
2656
2657SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2658 return DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: MVT::i16,
2659 Operand: BitConvertToInteger(Op: N->getOperand(Num: 0)));
2660}
2661
2662SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2663 return BitConvertToInteger(Op: N->getOperand(Num: 0));
2664}
2665
2666SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2667 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
2668
2669 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2670 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
2671 VT: MVT::i16);
2672}
2673
2674SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2675 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2676 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
2677 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
2678 N2: N->getOperand(Num: 1));
2679}
2680
2681SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2682 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2683 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
2684 SDLoc dl(N);
2685
2686 EVT LVT = LHS.getValueType();
2687 EVT RVT = RHS.getValueType();
2688
2689 unsigned LSize = LVT.getSizeInBits();
2690 unsigned RSize = RVT.getSizeInBits();
2691
2692 // First get the sign bit of second operand.
2693 SDValue SignBit = DAG.getNode(
2694 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
2695 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
2696 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
2697 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
2698
2699 // Shift right or sign-extend it if the two operands have different types.
2700 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2701 if (SizeDiff > 0) {
2702 SignBit =
2703 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
2704 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
2705 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2706 DL: DAG.getDataLayout())));
2707 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
2708 } else if (SizeDiff < 0) {
2709 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
2710 SignBit =
2711 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
2712 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
2713 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2714 DL: DAG.getDataLayout())));
2715 }
2716
2717 // Clear the sign bit of the first operand.
2718 SDValue Mask = DAG.getNode(
2719 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
2720 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
2721 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
2722 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
2723 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
2724
2725 // Or the value with the sign bit.
2726 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
2727}
2728
2729SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2730 EVT OVT = N->getValueType(ResNo: 0);
2731 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2732 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2733 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2734 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2735 SDNodeFlags Flags = N->getFlags();
2736 SDLoc dl(N);
2737
2738 // Promote to the larger FP type.
2739 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2740 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2741 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2742 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
2743
2744 SDValue Res;
2745 if (OVT == MVT::f16) {
2746 // If f16 fma is not natively supported, the value must be promoted to an
2747 // f64 (and not to f32!) to prevent double rounding issues.
2748 SDValue A64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op0, Flags);
2749 SDValue B64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op1, Flags);
2750 SDValue C64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op2, Flags);
2751
2752 // Prefer a wide FMA node if available; otherwise expand to mul+add.
2753 SDValue WideRes;
2754 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), MVT::f64)) {
2755 WideRes = DAG.getNode(Opcode: ISD::FMA, DL: dl, VT: MVT::f64, N1: A64, N2: B64, N3: C64, Flags);
2756 } else {
2757 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f64, N1: A64, N2: B64, Flags);
2758 WideRes = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f64, N1: Mul, N2: C64, Flags);
2759 }
2760
2761 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: MVT::f64, RetVT: OVT), DL: dl, VT: MVT::i16,
2762 Operand: WideRes);
2763 }
2764
2765 Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2, Flags);
2766 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2767}
2768
2769SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2770 EVT OVT = N->getValueType(ResNo: 0);
2771 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2772 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2773 SDValue Op1 = N->getOperand(Num: 1);
2774 SDLoc dl(N);
2775
2776 // Promote to the larger FP type.
2777 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
2778
2779 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2780
2781 // Convert back to FP16 as an integer.
2782 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2783}
2784
2785SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2786 EVT OVT = N->getValueType(ResNo: 0);
2787 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2788 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2789 SDLoc dl(N);
2790
2791 // Promote to the larger FP type.
2792 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2793
2794 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
2795 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
2796
2797 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2798
2799 // Convert back to FP16 as an integer.
2800 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2801}
2802
2803SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
2804 EVT OVT = N->getValueType(ResNo: 0);
2805 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2806 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2807 SDLoc dl(N);
2808
2809 // Promote to the larger FP type.
2810 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2811 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Op);
2812
2813 // Convert back to FP16 as an integer.
2814 ISD::NodeType Truncate = GetPromotionOpcode(OpVT: NVT, RetVT: OVT);
2815 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
2816 ++ResNum) {
2817 SDValue Trunc = DAG.getNode(Opcode: Truncate, DL: dl, VT: MVT::i16, Operand: Res.getValue(R: ResNum));
2818 SetSoftPromotedHalf(Op: SDValue(N, ResNum), Result: Trunc);
2819 }
2820
2821 return SDValue();
2822}
2823
2824SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2825 EVT RVT = N->getValueType(ResNo: 0);
2826 bool IsStrict = N->isStrictFPOpcode();
2827 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2828 EVT SVT = Op.getValueType();
2829
2830 // If the input type needs to be softened, do that now so that call lowering
2831 // will see the f16 type.
2832 if (getTypeAction(VT: SVT) == TargetLowering::TypeSoftenFloat) {
2833 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: RVT);
2834 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
2835
2836 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2837 Op = GetSoftenedFloat(Op);
2838 TargetLowering::MakeLibCallOptions CallOptions;
2839 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
2840 std::pair<SDValue, SDValue> Tmp =
2841 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
2842 if (IsStrict)
2843 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2844 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: MVT::i16, Operand: Tmp.first);
2845 }
2846
2847 if (IsStrict) {
2848 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
2849 ResultTys: {MVT::i16, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
2850 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2851 return Res;
2852 }
2853
2854 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: MVT::i16,
2855 Operand: N->getOperand(Num: 0));
2856}
2857
2858SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2859 LoadSDNode *L = cast<LoadSDNode>(Val: N);
2860
2861 // Load the value as an integer value with the same number of bits.
2862 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2863 SDValue NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(),
2864 VT: MVT::i16, dl: SDLoc(N), Chain: L->getChain(), Ptr: L->getBasePtr(),
2865 Offset: L->getOffset(), PtrInfo: L->getPointerInfo(), MemVT: MVT::i16,
2866 Alignment: L->getBaseAlign(), MMOFlags: L->getMemOperand()->getFlags(),
2867 Metadata: L->getMMOMetadataForSubAccess());
2868 // Legalize the chain result by replacing uses of the old value chain with the
2869 // new one
2870 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
2871 return NewL;
2872}
2873
2874SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
2875 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2876
2877 // Load the value as an integer value with the same number of bits.
2878 SDValue NewL = DAG.getAtomic(
2879 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: MVT::i16, VTList: DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other),
2880 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
2881
2882 // Legalize the chain result by replacing uses of the old value chain with the
2883 // new one
2884 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
2885 return NewL;
2886}
2887
2888SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2889 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2890 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2891 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1, RHS: Op2,
2892 Flags: N->getFlags());
2893}
2894
2895SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2896 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2897 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
2898 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
2899 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
2900 N5: N->getOperand(Num: 4));
2901}
2902
2903SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2904 EVT OVT = N->getValueType(ResNo: 0);
2905 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2906 SDLoc dl(N);
2907
2908 if (N->isStrictFPOpcode()) {
2909 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
2910 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
2911 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: NVT, RetVT: OVT), DL: dl,
2912 ResultTys: {MVT::i16, MVT::Other}, Ops: {Op.getValue(R: 1), Op});
2913 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
2914 return Op;
2915 }
2916
2917 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
2918
2919 // Round the value to the softened type.
2920 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2921}
2922
2923SDValue
2924DAGTypeLegalizer::SoftPromoteHalfRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
2925 EVT OVT = N->getValueType(ResNo: 0);
2926 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2927 SDLoc dl(N);
2928
2929 SDValue Res = DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: dl, VT: NVT,
2930 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
2931
2932 // Round the value to the softened type.
2933 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2934}
2935
2936SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2937 return DAG.getUNDEF(VT: MVT::i16);
2938}
2939
2940SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2941 EVT OVT = N->getValueType(ResNo: 0);
2942 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2943 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2944 SDLoc dl(N);
2945
2946 // Promote to the larger FP type.
2947 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2948
2949 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
2950
2951 // Convert back to FP16 as an integer.
2952 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2953}
2954
2955SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FABS(SDNode *N) {
2956 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2957 SDLoc dl(N);
2958
2959 // Clear the sign bit.
2960 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i16, N1: Op,
2961 N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: MVT::i16));
2962}
2963
2964SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FNEG(SDNode *N) {
2965 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2966 SDLoc dl(N);
2967
2968 // Invert the sign bit.
2969 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i16, N1: Op,
2970 N2: DAG.getConstant(Val: 0x8000, DL: dl, VT: MVT::i16));
2971}
2972
2973SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
2974 return GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2975}
2976
2977SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2978 EVT OVT = N->getValueType(ResNo: 0);
2979 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2980 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2981 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2982 SDLoc dl(N);
2983
2984 // Promote to the larger FP type.
2985 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2986 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2987 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2988
2989 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2990
2991 // Convert back to FP16 as an integer.
2992 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2993}
2994
2995SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2996 // Expand and soften recursively.
2997 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
2998 return SDValue();
2999}
3000
3001SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3002 // Expand and soften.
3003 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3004 return SDValue();
3005}
3006
3007//===----------------------------------------------------------------------===//
3008// Half Operand Soft Promotion
3009//===----------------------------------------------------------------------===//
3010
3011bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3012 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3013 N->dump(&DAG));
3014 SDValue Res = SDValue();
3015
3016 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
3017 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3018 return false;
3019 }
3020
3021 // Nodes that use a promotion-requiring floating point operand, but doesn't
3022 // produce a soft promotion-requiring floating point result, need to be
3023 // legalized to use the soft promoted float operand. Nodes that produce at
3024 // least one soft promotion-requiring floating point result have their
3025 // operands legalized as a part of PromoteFloatResult.
3026 switch (N->getOpcode()) {
3027 default:
3028 #ifndef NDEBUG
3029 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3030 N->dump(&DAG); dbgs() << "\n";
3031 #endif
3032 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3033 "operand!");
3034
3035 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3036 case ISD::BUILD_VECTOR:
3037 Res = SoftPromoteHalfOp_BUILD_VECTOR(N);
3038 break;
3039 case ISD::INSERT_VECTOR_ELT:
3040 Res = SoftPromoteHalfOp_INSERT_VECTOR_ELT(N, OpNo);
3041 break;
3042 case ISD::FAKE_USE:
3043 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
3044 break;
3045 case ISD::FCOPYSIGN:
3046 Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo);
3047 break;
3048 case ISD::FP_TO_SINT:
3049 case ISD::FP_TO_UINT:
3050 case ISD::STRICT_FP_TO_SINT:
3051 case ISD::STRICT_FP_TO_UINT:
3052 case ISD::LLRINT:
3053 case ISD::LLROUND:
3054 case ISD::LRINT:
3055 case ISD::LROUND:
3056 case ISD::STRICT_LLRINT:
3057 case ISD::STRICT_LLROUND:
3058 case ISD::STRICT_LRINT:
3059 case ISD::STRICT_LROUND:
3060 Res = SoftPromoteHalfOp_Op0WithStrict(N);
3061 break;
3062 case ISD::FP_TO_SINT_SAT:
3063 case ISD::FP_TO_UINT_SAT:
3064 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3065 case ISD::CONVERT_TO_ARBITRARY_FP:
3066 Res = SoftPromoteHalfOp_CONVERT_TO_ARBITRARY_FP(N);
3067 break;
3068 case ISD::STRICT_FP_EXTEND:
3069 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3070 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3071 case ISD::BR_CC:
3072 Res = SoftPromoteHalfOp_BR_CC(N);
3073 break;
3074 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3075 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3076 case ISD::ATOMIC_STORE:
3077 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3078 break;
3079 case ISD::STACKMAP:
3080 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3081 break;
3082 case ISD::PATCHPOINT:
3083 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3084 break;
3085 }
3086
3087 if (!Res.getNode())
3088 return false;
3089
3090 assert(Res.getNode() != N && "Expected a new node!");
3091
3092 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3093 "Invalid operand expansion");
3094
3095 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3096 return false;
3097}
3098
3099SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3100 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3101
3102 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3103}
3104
3105SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BUILD_VECTOR(SDNode *N) {
3106 SDLoc dl(N);
3107 EVT VT = N->getValueType(ResNo: 0);
3108
3109 SmallVector<SDValue, 8> Ops(N->getNumOperands());
3110 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I)
3111 Ops[I] = GetSoftPromotedHalf(Op: N->getOperand(Num: I));
3112
3113 EVT IVT = VT.changeVectorElementTypeToInteger();
3114 SDValue Res = DAG.getBuildVector(VT: IVT, DL: dl, Ops);
3115 return DAG.getBitcast(VT, V: Res);
3116}
3117
3118SDValue DAGTypeLegalizer::SoftPromoteHalfOp_INSERT_VECTOR_ELT(SDNode *N,
3119 unsigned OpNo) {
3120 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3121 SDValue Vec = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
3122 SDValue Elt = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3123 SDValue Res = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
3124 VT: Vec.getValueType(), N1: Vec, N2: Elt, N3: N->getOperand(Num: 2));
3125 return DAG.getBitcast(VT: N->getValueType(ResNo: 0), V: Res);
3126}
3127
3128SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3129 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3130 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3131 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::Other, N1: N->getOperand(Num: 0),
3132 N2: Op);
3133}
3134
3135SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3136 unsigned OpNo) {
3137 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3138 SDValue Op1 = N->getOperand(Num: 1);
3139 EVT RVT = Op1.getValueType();
3140 SDLoc dl(N);
3141
3142 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3143
3144 Op1 = GetSoftPromotedHalf(Op: Op1);
3145 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3146
3147 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3148 N2: Op1);
3149}
3150
3151SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3152 EVT RVT = N->getValueType(ResNo: 0);
3153 bool IsStrict = N->isStrictFPOpcode();
3154 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3155 EVT SVT = Op.getValueType();
3156 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3157
3158 if (IsStrict) {
3159 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
3160 ResultTys: {RVT, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
3161 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3162 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3163 return SDValue();
3164 }
3165
3166 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3167}
3168
3169SDValue DAGTypeLegalizer::SoftPromoteHalfOp_Op0WithStrict(SDNode *N) {
3170 EVT RVT = N->getValueType(ResNo: 0);
3171 bool IsStrict = N->isStrictFPOpcode();
3172 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3173 EVT SVT = Op.getValueType();
3174 SDLoc dl(N);
3175
3176 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3177 Op = GetSoftPromotedHalf(Op);
3178
3179 if (IsStrict) {
3180 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: dl, ResultTys: {NVT, MVT::Other},
3181 Ops: {N->getOperand(Num: 0), Op});
3182 Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {RVT, MVT::Other},
3183 Ops: {Op.getValue(R: 1), Op});
3184 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3185 ReplaceValueWith(From: SDValue(N, 0), To: Op);
3186 return SDValue();
3187 }
3188
3189 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3190 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: RVT, Operand: Res);
3191}
3192
3193SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3194 EVT RVT = N->getValueType(ResNo: 0);
3195 SDValue Op = N->getOperand(Num: 0);
3196 EVT SVT = Op.getValueType();
3197 SDLoc dl(N);
3198
3199 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3200
3201 Op = GetSoftPromotedHalf(Op);
3202
3203 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3204
3205 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3206 N2: N->getOperand(Num: 1));
3207}
3208
3209// TODO: CONVERT_TO_ARBITRARY_FP also needs SoftenFloatOperand and
3210// ExpandFloatOperand handlers for targets with software float or ppcf128
3211// source types. Same gap exists for CONVERT_FROM_ARBITRARY_FP.
3212SDValue DAGTypeLegalizer::SoftPromoteHalfOp_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
3213 EVT RVT = N->getValueType(ResNo: 0);
3214 SDValue Op = N->getOperand(Num: 0);
3215 EVT SVT = Op.getValueType();
3216 SDLoc dl(N);
3217
3218 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3219 Op = GetSoftPromotedHalf(Op);
3220 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3221
3222 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3223 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
3224}
3225
3226SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BR_CC(SDNode *N) {
3227 // ISD::BR_CC node: chain(0), condcode(1), LHS(2), RHS(3), dest(4)
3228 // The comparison operands (LHS, RHS) are soft-promoted halfs.
3229 SDValue Op0 = N->getOperand(Num: 2);
3230 SDValue Op1 = N->getOperand(Num: 3);
3231 SDLoc dl(N);
3232
3233 EVT SVT = Op0.getValueType();
3234 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3235
3236 // Get the soft-promoted i16 values
3237 Op0 = GetSoftPromotedHalf(Op: Op0);
3238 Op1 = GetSoftPromotedHalf(Op: Op1);
3239
3240 // Promote both comparison operands to the larger FP type.
3241 unsigned PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3242 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3243 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3244
3245 // Create a new BR_CC node with promoted operands
3246 return DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: N->getOperand(Num: 0),
3247 N2: N->getOperand(Num: 1), N3: Op0, N4: Op1, N5: N->getOperand(Num: 4));
3248}
3249
3250SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3251 unsigned OpNo) {
3252 assert(OpNo == 0 && "Can only soften the comparison values");
3253 SDValue Op0 = N->getOperand(Num: 0);
3254 SDValue Op1 = N->getOperand(Num: 1);
3255 SDLoc dl(N);
3256
3257 EVT SVT = Op0.getValueType();
3258 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3259
3260 Op0 = GetSoftPromotedHalf(Op: Op0);
3261 Op1 = GetSoftPromotedHalf(Op: Op1);
3262
3263 // Promote to the larger FP type.
3264 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3265 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3266 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3267
3268 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3269 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3270}
3271
3272SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3273 SDValue Op0 = N->getOperand(Num: 0);
3274 SDValue Op1 = N->getOperand(Num: 1);
3275 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3276 SDLoc dl(N);
3277
3278 EVT SVT = Op0.getValueType();
3279 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3280
3281 Op0 = GetSoftPromotedHalf(Op: Op0);
3282 Op1 = GetSoftPromotedHalf(Op: Op1);
3283
3284 // Promote to the larger FP type.
3285 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3286 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3287 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3288
3289 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3290}
3291
3292SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3293 assert(OpNo == 1 && "Can only soften the stored value!");
3294 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3295 SDValue Val = ST->getValue();
3296 SDLoc dl(N);
3297
3298 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3299 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3300 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3301 MMO: ST->getMemOperand());
3302}
3303
3304SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3305 unsigned OpNo) {
3306 assert(OpNo == 1 && "Can only soften the stored value!");
3307 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
3308 SDValue Val = ST->getVal();
3309 SDLoc dl(N);
3310
3311 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3312 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: Promoted.getValueType(),
3313 Chain: ST->getChain(), Ptr: Promoted, Val: ST->getBasePtr(),
3314 MMO: ST->getMemOperand());
3315}
3316
3317SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3318 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3319 SmallVector<SDValue> NewOps(N->ops());
3320 SDValue Op = N->getOperand(Num: OpNo);
3321 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3322 SDValue NewNode =
3323 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3324
3325 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3326 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3327
3328 return SDValue(); // Signal that we replaced the node ourselves.
3329}
3330
3331SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3332 unsigned OpNo) {
3333 assert(OpNo >= 7);
3334 SmallVector<SDValue> NewOps(N->ops());
3335 SDValue Op = N->getOperand(Num: OpNo);
3336 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3337 SDValue NewNode =
3338 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3339
3340 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3341 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3342
3343 return SDValue(); // Signal that we replaced the node ourselves.
3344}
3345