1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29static void reportNoLibcall(SelectionDAG &DAG, SDNode *N, EVT VT) {
30 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
31 N->getOperationName(G: &DAG) + " with type " +
32 VT.getEVTString());
33}
34
35/// GetFPLibCall - Return the right libcall for the given floating point type.
36/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
37/// refactored away (see RTLIB::getPOWI for an example).
38static RTLIB::Libcall GetFPLibCall(EVT VT,
39 RTLIB::Libcall Call_F32,
40 RTLIB::Libcall Call_F64,
41 RTLIB::Libcall Call_F80,
42 RTLIB::Libcall Call_F128,
43 RTLIB::Libcall Call_PPCF128) {
44 return
45 VT == MVT::f32 ? Call_F32 :
46 VT == MVT::f64 ? Call_F64 :
47 VT == MVT::f80 ? Call_F80 :
48 VT == MVT::f128 ? Call_F128 :
49 VT == MVT::ppcf128 ? Call_PPCF128 :
50 RTLIB::UNKNOWN_LIBCALL;
51}
52
53//===----------------------------------------------------------------------===//
54// Convert Float Results to Integer
55//===----------------------------------------------------------------------===//
56
57void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
58 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
59 SDValue R = SDValue();
60
61 switch (N->getOpcode()) {
62 // clang-format off
63 default:
64#ifndef NDEBUG
65 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
66 N->dump(&DAG); dbgs() << "\n";
67#endif
68 report_fatal_error(reason: "Do not know how to soften the result of this "
69 "operator!");
70 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
71 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
72 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
73 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
74 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
75 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
76 case ISD::EXTRACT_VECTOR_ELT:
77 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
78 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
79 case ISD::FCANONICALIZE:
80 R = SoftenFloatRes_FCANONICALIZE(N); break;
81 case ISD::STRICT_FMINNUM:
82 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
83 case ISD::STRICT_FMAXNUM:
84 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
85 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
86 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
87 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
88 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
89 case ISD::STRICT_FADD:
90 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
91 case ISD::STRICT_FACOS:
92 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
93 case ISD::STRICT_FASIN:
94 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
95 case ISD::STRICT_FATAN:
96 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
97 case ISD::STRICT_FATAN2:
98 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
99 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
100 case ISD::STRICT_FCEIL:
101 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
102 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
103 case ISD::STRICT_FCOS:
104 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
105 case ISD::STRICT_FCOSH:
106 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
107 case ISD::STRICT_FDIV:
108 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
109 case ISD::STRICT_FEXP:
110 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
111 case ISD::STRICT_FEXP2:
112 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
113 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
114 case ISD::STRICT_FFLOOR:
115 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
116 case ISD::STRICT_FLOG:
117 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
118 case ISD::STRICT_FLOG2:
119 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
120 case ISD::STRICT_FLOG10:
121 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
122 case ISD::STRICT_FMA:
123 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
124 case ISD::STRICT_FMUL:
125 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
126 case ISD::STRICT_FNEARBYINT:
127 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
128 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
129 case ISD::STRICT_FP_EXTEND:
130 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
131 case ISD::STRICT_FP_ROUND:
132 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
133 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
134 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
135 case ISD::STRICT_FPOW:
136 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
137 case ISD::STRICT_FPOWI:
138 case ISD::FPOWI:
139 case ISD::FLDEXP:
140 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
141 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
142 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
143 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
144 case ISD::STRICT_FREM:
145 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
146 case ISD::STRICT_FRINT:
147 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
148 case ISD::STRICT_FROUND:
149 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
150 case ISD::STRICT_FROUNDEVEN:
151 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
152 case ISD::STRICT_FSIN:
153 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
154 case ISD::STRICT_FSINH:
155 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
156 case ISD::STRICT_FSQRT:
157 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
158 case ISD::STRICT_FSUB:
159 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
160 case ISD::STRICT_FTAN:
161 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
162 case ISD::STRICT_FTANH:
163 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
164 case ISD::STRICT_FTRUNC:
165 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
166 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
167 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
168 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
169 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
170 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
171 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
172 case ISD::STRICT_SINT_TO_FP:
173 case ISD::STRICT_UINT_TO_FP:
174 case ISD::SINT_TO_FP:
175 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
176 case ISD::POISON:
177 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
178 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
179 case ISD::AssertNoFPClass: R = GetSoftenedFloat(Op: N->getOperand(Num: 0)); break;
180 case ISD::VECREDUCE_FADD:
181 case ISD::VECREDUCE_FMUL:
182 case ISD::VECREDUCE_FMIN:
183 case ISD::VECREDUCE_FMAX:
184 case ISD::VECREDUCE_FMAXIMUM:
185 case ISD::VECREDUCE_FMINIMUM:
186 case ISD::VECREDUCE_FMAXIMUMNUM:
187 case ISD::VECREDUCE_FMINIMUMNUM: R = SoftenFloatRes_VECREDUCE(N); break;
188 case ISD::VECREDUCE_SEQ_FADD:
189 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
190 // clang-format on
191 }
192
193 // If R is null, the sub-method took care of registering the result.
194 if (R.getNode()) {
195 assert(R.getNode() != N);
196 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
197 }
198}
199
200// No libcall is available to soften this operation. Emit a diagnostic and
201// produce a poison result of the softened type \p NVT.
202SDValue DAGTypeLegalizer::SoftenFloatRes_NoLibcall(SDNode *N, EVT NVT) {
203 reportNoLibcall(DAG, N, VT: N->getValueType(ResNo: 0));
204 if (N->isStrictFPOpcode())
205 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
206 return DAG.getPOISON(VT: NVT);
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
210 bool IsStrict = N->isStrictFPOpcode();
211 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
212 unsigned Offset = IsStrict ? 1 : 0;
213 assert(N->getNumOperands() == (1 + Offset) &&
214 "Unexpected number of operands!");
215 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
216 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
217 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
218 if (LCImpl == RTLIB::Unsupported)
219 return SoftenFloatRes_NoLibcall(N, NVT);
220 TargetLowering::MakeLibCallOptions CallOptions;
221 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
222 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
223 std::pair<SDValue, SDValue> Tmp =
224 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
225 if (IsStrict)
226 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
227 return Tmp.first;
228}
229
230SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
231 bool IsStrict = N->isStrictFPOpcode();
232 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
233 unsigned Offset = IsStrict ? 1 : 0;
234 assert(N->getNumOperands() == (2 + Offset) &&
235 "Unexpected number of operands!");
236 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
237 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
238 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
239 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
240 if (LCImpl == RTLIB::Unsupported)
241 return SoftenFloatRes_NoLibcall(N, NVT);
242 TargetLowering::MakeLibCallOptions CallOptions;
243 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
244 N->getOperand(Num: 1 + Offset).getValueType() };
245 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
246 std::pair<SDValue, SDValue> Tmp =
247 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
248 if (IsStrict)
249 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
250 return Tmp.first;
251}
252
253SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
254 return BitConvertToInteger(Op: N->getOperand(Num: 0));
255}
256
257SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
258 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
259 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
260 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
261}
262
263SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
264 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
265 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
266 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
267 return NewFence;
268}
269
270SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
271 unsigned ResNo) {
272 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
273 return BitConvertToInteger(Op);
274}
275
276SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
277 // Convert the inputs to integers, and build a new pair out of them.
278 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
279 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
280 VT: N->getValueType(ResNo: 0)),
281 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
282 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
283}
284
285SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
286 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
287 // In ppcf128, the high 64 bits are always first in memory regardless
288 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
289 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
290 // way. However, APInt's are serialized in an Endian-sensitive fashion,
291 // so on big-Endian targets, the two doubles are output in the wrong
292 // order. Fix this by manually flipping the order of the high 64 bits
293 // and the low 64 bits here.
294 if (DAG.getDataLayout().isBigEndian() &&
295 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
296 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
297 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
298 APInt Val(128, words);
299 return DAG.getConstant(Val, DL: SDLoc(CN),
300 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
301 VT: CN->getValueType(ResNo: 0)));
302 } else {
303 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
304 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
305 VT: CN->getValueType(ResNo: 0)));
306 }
307}
308
309SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
310 SDValue Src = N->getOperand(Num: 0);
311 assert(Src.getValueType() == MVT::ppcf128 &&
312 "In floats only ppcf128 can be extracted by element!");
313 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(N),
314 VT: N->getValueType(ResNo: 0).changeTypeToInteger(),
315 N1: DAG.getBitcast(VT: MVT::i128, V: Src), N2: N->getOperand(Num: 1));
316}
317
318SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
319 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
320 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
321 VT: NewOp.getValueType().getVectorElementType(),
322 N1: NewOp, N2: N->getOperand(Num: 1));
323}
324
325SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
326 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
327 unsigned Size = NVT.getSizeInBits();
328
329 // Mask = ~(1 << (Size-1))
330 APInt API = APInt::getAllOnes(numBits: Size);
331 API.clearBit(BitPosition: Size - 1);
332 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
333 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
334 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
335}
336
337SDValue DAGTypeLegalizer::SoftenFloatRes_FCANONICALIZE(SDNode *N) {
338 SDLoc dl(N);
339
340 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
341 // suggested in
342 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
343 // It uses strict_fp operations even outside a strict_fp context in order
344 // to guarantee that the canonicalization is not optimized away by later
345 // passes. The result chain introduced by that is intentionally ignored
346 // since no ordering requirement is intended here.
347
348 // Create strict multiplication by 1.0.
349 SDValue Operand = N->getOperand(Num: 0);
350 EVT VT = Operand.getValueType();
351 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
352 SDValue Chain = DAG.getEntryNode();
353 // Propagate existing flags on canonicalize, and additionally set
354 // NoFPExcept.
355 SDNodeFlags CanonicalizeFlags = N->getFlags();
356 CanonicalizeFlags.setNoFPExcept(true);
357 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
358 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
359 return BitConvertToInteger(Op: Mul);
360}
361
362SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
363 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
364 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
365 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMIN(VT: N->getValueType(ResNo: 0)));
366}
367
368SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
369 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
370 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
371 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAX(VT: N->getValueType(ResNo: 0)));
372}
373
374SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
375 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMINIMUM_NUM(VT: N->getValueType(ResNo: 0)));
376}
377
378SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
379 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM_NUM(VT: N->getValueType(ResNo: 0)));
380}
381
382SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
383 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMINIMUM(VT: N->getValueType(ResNo: 0)));
384}
385
386SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
387 return SoftenFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM(VT: N->getValueType(ResNo: 0)));
388}
389
390SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
391 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
392 Call_F32: RTLIB::ADD_F32,
393 Call_F64: RTLIB::ADD_F64,
394 Call_F80: RTLIB::ADD_F80,
395 Call_F128: RTLIB::ADD_F128,
396 Call_PPCF128: RTLIB::ADD_PPCF128));
397}
398
399SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
400 return SoftenFloatRes_Unary(N, LC: RTLIB::getACOS(VT: N->getValueType(ResNo: 0)));
401}
402
403SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
404 return SoftenFloatRes_Unary(N, LC: RTLIB::getASIN(VT: N->getValueType(ResNo: 0)));
405}
406
407SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
408 return SoftenFloatRes_Unary(N, LC: RTLIB::getATAN(VT: N->getValueType(ResNo: 0)));
409}
410
411SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
412 return SoftenFloatRes_Binary(N, LC: RTLIB::getATAN2(VT: N->getValueType(ResNo: 0)));
413}
414
415SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
416 return SoftenFloatRes_Unary(N, LC: RTLIB::getCBRT(VT: N->getValueType(ResNo: 0)));
417}
418
419SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
420 return SoftenFloatRes_Unary(N, LC: RTLIB::getCEIL(VT: N->getValueType(ResNo: 0)));
421}
422
423SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
424 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
425 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
426 SDLoc dl(N);
427
428 EVT LVT = LHS.getValueType();
429 EVT RVT = RHS.getValueType();
430
431 unsigned LSize = LVT.getSizeInBits();
432 unsigned RSize = RVT.getSizeInBits();
433
434 // First get the sign bit of second operand.
435 SDValue SignBit = DAG.getNode(
436 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
437 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
438 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
439 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
440
441 // Shift right or sign-extend it if the two operands have different types.
442 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
443 if (SizeDiff > 0) {
444 SignBit =
445 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
446 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
447 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
448 DL: DAG.getDataLayout())));
449 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
450 } else if (SizeDiff < 0) {
451 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
452 SignBit =
453 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
454 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
455 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
456 DL: DAG.getDataLayout())));
457 }
458
459 // Clear the sign bit of the first operand.
460 SDValue Mask = DAG.getNode(
461 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
462 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
463 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
464 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
465 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
466
467 // Or the value with the sign bit.
468 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
469}
470
471SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
472 return SoftenFloatRes_Unary(N, LC: RTLIB::getCOS(VT: N->getValueType(ResNo: 0)));
473}
474
475SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
476 return SoftenFloatRes_Unary(N, LC: RTLIB::getCOSH(VT: N->getValueType(ResNo: 0)));
477}
478
479SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
480 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
481 Call_F32: RTLIB::DIV_F32,
482 Call_F64: RTLIB::DIV_F64,
483 Call_F80: RTLIB::DIV_F80,
484 Call_F128: RTLIB::DIV_F128,
485 Call_PPCF128: RTLIB::DIV_PPCF128));
486}
487
488SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
489 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP(VT: N->getValueType(ResNo: 0)));
490}
491
492SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
493 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP2(VT: N->getValueType(ResNo: 0)));
494}
495
496SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
497 return SoftenFloatRes_Unary(N, LC: RTLIB::getEXP10(VT: N->getValueType(ResNo: 0)));
498}
499
500SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
501 return SoftenFloatRes_Unary(N, LC: RTLIB::getFLOOR(VT: N->getValueType(ResNo: 0)));
502}
503
504SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
505 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG(VT: N->getValueType(ResNo: 0)));
506}
507
508SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
509 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG2(VT: N->getValueType(ResNo: 0)));
510}
511
512SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
513 return SoftenFloatRes_Unary(N, LC: RTLIB::getLOG10(VT: N->getValueType(ResNo: 0)));
514}
515
516SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
517 bool IsStrict = N->isStrictFPOpcode();
518 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
519 unsigned Offset = IsStrict ? 1 : 0;
520 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
521 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
522 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
523 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
524 TargetLowering::MakeLibCallOptions CallOptions;
525 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
526 N->getOperand(Num: 1 + Offset).getValueType(),
527 N->getOperand(Num: 2 + Offset).getValueType() };
528 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
529 RTLIB::Libcall LC = RTLIB::getFMA(VT: N->getValueType(ResNo: 0));
530 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
531 if (LCImpl == RTLIB::Unsupported)
532 return SoftenFloatRes_NoLibcall(N, NVT);
533 std::pair<SDValue, SDValue> Tmp =
534 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
535 if (IsStrict)
536 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
537 return Tmp.first;
538}
539
540SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
541 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
542 Call_F32: RTLIB::MUL_F32,
543 Call_F64: RTLIB::MUL_F64,
544 Call_F80: RTLIB::MUL_F80,
545 Call_F128: RTLIB::MUL_F128,
546 Call_PPCF128: RTLIB::MUL_PPCF128));
547}
548
549SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
550 return SoftenFloatRes_Unary(N, LC: RTLIB::getNEARBYINT(VT: N->getValueType(ResNo: 0)));
551}
552
553SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
554 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
555 SDLoc dl(N);
556
557 // Expand Y = FNEG(X) -> Y = X ^ sign mask
558 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
559 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
560 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
561}
562
563SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
564 bool IsStrict = N->isStrictFPOpcode();
565 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
566 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
567
568 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
569
570 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
571 // -> f32, so proceed in two stages. Also, it's entirely possible for both
572 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
573 // than FP16_TO_FP.
574 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
575 N->getValueType(ResNo: 0) != MVT::f32) {
576 if (IsStrict) {
577 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
578 ResultTys: { MVT::f32, MVT::Other }, Ops: { Chain, Op });
579 Chain = Op.getValue(R: 1);
580 } else {
581 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT: MVT::f32, Operand: Op);
582 }
583 }
584
585 if (Op.getValueType() == MVT::bf16) {
586 // FIXME: Need ReplaceValueWith on chain in strict case
587 return SoftenFloatRes_BF16_TO_FP(N);
588 }
589
590 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
591 if (LC == RTLIB::UNKNOWN_LIBCALL) {
592 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fp_extend");
593 if (IsStrict)
594 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
595 return DAG.getPOISON(VT: NVT);
596 }
597 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
598 if (LCImpl == RTLIB::Unsupported)
599 return SoftenFloatRes_NoLibcall(N, NVT);
600 TargetLowering::MakeLibCallOptions CallOptions;
601 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
602 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
603 std::pair<SDValue, SDValue> Tmp =
604 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
605 if (IsStrict)
606 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
607 return Tmp.first;
608}
609
610// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
611// nodes?
612SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
613 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
614 SDValue Op = N->getOperand(Num: 0);
615 TargetLowering::MakeLibCallOptions CallOptions;
616 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
617 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
618 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
619 CallOptions, dl: SDLoc(N)).first;
620 if (N->getValueType(ResNo: 0) == MVT::f32)
621 return Res32;
622
623 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
624 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: MVT::f32, RetVT: N->getValueType(ResNo: 0));
625 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
626 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
627}
628
629// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
630// nodes?
631SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
632 assert(N->getValueType(0) == MVT::f32 &&
633 "Can only soften BF16_TO_FP with f32 result");
634 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
635 SDValue Op = N->getOperand(Num: 0);
636 SDLoc DL(N);
637 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT,
638 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op));
639 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
640 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
641 return Res;
642}
643
644SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
645 bool IsStrict = N->isStrictFPOpcode();
646 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
647 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
648 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
649 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
650 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
651 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
652 if (LCImpl == RTLIB::Unsupported)
653 return SoftenFloatRes_NoLibcall(N, NVT);
654 TargetLowering::MakeLibCallOptions CallOptions;
655 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
656 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
657 std::pair<SDValue, SDValue> Tmp =
658 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
659 if (IsStrict)
660 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
661 return Tmp.first;
662}
663
664SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
665 return SoftenFloatRes_Binary(N, LC: RTLIB::getPOW(VT: N->getValueType(ResNo: 0)));
666}
667
668SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
669 bool IsStrict = N->isStrictFPOpcode();
670 unsigned Offset = IsStrict ? 1 : 0;
671 bool IsPowI =
672 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
673 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
674
675 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(VT: N->getValueType(ResNo: 0))
676 : RTLIB::getLDEXP(VT: N->getValueType(ResNo: 0));
677 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
678 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
679 if (LCImpl == RTLIB::Unsupported) {
680 // Some targets don't have a powi libcall; use pow instead.
681 // FIXME: Implement this if some target needs it.
682 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fpowi to fpow");
683 if (IsStrict)
684 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
685 return DAG.getPOISON(VT: NVT);
686 }
687
688 if (DAG.getLibInfo().getIntSize() !=
689 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
690 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
691 // would use the wrong type for the argument.
692 DAG.getContext()->emitError(ErrorStr: "powi exponent does not match sizeof(int)");
693 if (IsStrict)
694 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
695 return DAG.getPOISON(VT: NVT);
696 }
697
698 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
699 N->getOperand(Num: 1 + Offset) };
700 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
701 TargetLowering::MakeLibCallOptions CallOptions;
702 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
703 N->getOperand(Num: 1 + Offset).getValueType() };
704 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0));
705 CallOptions.setIsSigned();
706 std::pair<SDValue, SDValue> Tmp =
707 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
708 if (IsStrict)
709 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
710 return Tmp.first;
711}
712
713SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
714 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
715 EVT VT0 = N->getValueType(ResNo: 0);
716 EVT VT1 = N->getValueType(ResNo: 1);
717 RTLIB::Libcall LC = RTLIB::getFREXP(VT: VT0);
718 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
719 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
720 SDLoc DL(N);
721
722 if (LCImpl == RTLIB::Unsupported) {
723 reportNoLibcall(DAG, N, VT: VT0);
724 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
725 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
726 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
727 }
728
729 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
730 // If the exponent does not match with sizeof(int) a libcall would use the
731 // wrong type for the argument.
732 // TODO: Should be able to handle mismatches.
733 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
734 SDValue PoisonExp = DAG.getPOISON(VT: VT1);
735 ReplaceValueWith(From: SDValue(N, 1), To: PoisonExp);
736 return DAG.getMergeValues(Ops: {DAG.getPOISON(VT: NVT0), PoisonExp}, dl: DL);
737 }
738
739 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
740
741 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
742 TargetLowering::MakeLibCallOptions CallOptions;
743 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
744 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
745 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
746
747 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
748 // but we only really need to handle the 0th one for softening anyway.
749 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0)
750 .setOpsTypeOverrides(CallOpsTypeOverrides);
751
752 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT0, Ops, CallOptions,
753 dl: DL, /*Chain=*/SDValue());
754 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
755 auto PtrInfo =
756 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
757
758 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
759
760 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
761 return ReturnVal;
762}
763
764bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
765 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
766 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
767 EVT VT = N->getValueType(ResNo: 0);
768
769 assert(VT == N->getValueType(1) &&
770 "expected both return values to have the same type");
771
772 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
773 if (LCImpl == RTLIB::Unsupported)
774 return false;
775
776 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
777
778 SDLoc DL(N);
779
780 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(Op: N->getOperand(Num: 0))};
781 SmallVector<EVT, 3> OpsVT = {VT};
782
783 std::array<SDValue, 2> StackSlots;
784 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
785 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
786 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
787 if (ResNum == CallRetResNo)
788 continue;
789 SDValue StackSlot = DAG.CreateStackTemporary(VT: NVT);
790 Ops.push_back(Elt: StackSlot);
791 OpsVT.push_back(Elt: StackSlot.getValueType());
792 StackSlots[ResNum] = StackSlot;
793 CallOpsTypeOverrides.push_back(Elt: PointerTy);
794 }
795
796 TargetLowering::MakeLibCallOptions CallOptions;
797 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
798 // but since both returns have the same type it should be okay.
799 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT)
800 .setOpsTypeOverrides(CallOpsTypeOverrides);
801
802 auto [ReturnVal, Chain] =
803 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops, CallOptions, dl: DL,
804 /*Chain=*/SDValue());
805
806 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
807 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
808 auto PtrInfo =
809 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
810 return DAG.getLoad(VT: NVT, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
811 };
812
813 for (auto [ResNum, SlackSlot] : enumerate(First&: StackSlots)) {
814 if (CallRetResNo == ResNum) {
815 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: ReturnVal);
816 continue;
817 }
818 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: CreateStackLoad(SlackSlot));
819 }
820
821 return true;
822}
823
824SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
825 EVT VT = N->getValueType(ResNo: 0);
826 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(VT)))
827 return SDValue();
828
829 // Fall back on softening the separate sin and cos calls if available.
830 RTLIB::Libcall SinLC = RTLIB::getSIN(VT);
831 RTLIB::Libcall CosLC = RTLIB::getCOS(VT);
832
833 SDValue SoftSin, SoftCos;
834 if (DAG.getLibcalls().getLibcallImpl(Call: SinLC) == RTLIB::Unsupported ||
835 DAG.getLibcalls().getLibcallImpl(Call: CosLC) == RTLIB::Unsupported) {
836 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fsincos");
837
838 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
839 SoftSin = SoftCos = DAG.getPOISON(VT: NVT);
840 } else {
841 SoftSin = SoftenFloatRes_Unary(N, LC: SinLC);
842 SoftCos = SoftenFloatRes_Unary(N, LC: CosLC);
843 }
844
845 SetSoftenedFloat(Op: SDValue(N, 0), Result: SoftSin);
846 SetSoftenedFloat(Op: SDValue(N, 1), Result: SoftCos);
847 return SDValue();
848}
849
850SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
851 EVT VT = N->getValueType(ResNo: 0);
852 if (SoftenFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT),
853 /*CallRetResNo=*/0))
854 return SDValue();
855
856 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
857 DAG.getContext()->emitError(ErrorStr: "do not know how to soften fmodf");
858 SDValue Poison = DAG.getPOISON(VT: NVT);
859 SetSoftenedFloat(Op: SDValue(N, 0), Result: Poison);
860 SetSoftenedFloat(Op: SDValue(N, 1), Result: Poison);
861 return SDValue();
862}
863
864SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
865 return SoftenFloatRes_Binary(N, LC: RTLIB::getREM(VT: N->getValueType(ResNo: 0)));
866}
867
868SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
869 return SoftenFloatRes_Unary(N, LC: RTLIB::getRINT(VT: N->getValueType(ResNo: 0)));
870}
871
872SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
873 return SoftenFloatRes_Unary(N, LC: RTLIB::getROUND(VT: N->getValueType(ResNo: 0)));
874}
875
876SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
877 return SoftenFloatRes_Unary(N, LC: RTLIB::getROUNDEVEN(VT: N->getValueType(ResNo: 0)));
878}
879
880SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
881 return SoftenFloatRes_Unary(N, LC: RTLIB::getSIN(VT: N->getValueType(ResNo: 0)));
882}
883
884SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
885 return SoftenFloatRes_Unary(N, LC: RTLIB::getSINH(VT: N->getValueType(ResNo: 0)));
886}
887
888SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
889 return SoftenFloatRes_Unary(N, LC: RTLIB::getSQRT(VT: N->getValueType(ResNo: 0)));
890}
891
892SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
893 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
894 Call_F32: RTLIB::SUB_F32,
895 Call_F64: RTLIB::SUB_F64,
896 Call_F80: RTLIB::SUB_F80,
897 Call_F128: RTLIB::SUB_F128,
898 Call_PPCF128: RTLIB::SUB_PPCF128));
899}
900
901SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
902 return SoftenFloatRes_Unary(N, LC: RTLIB::getTAN(VT: N->getValueType(ResNo: 0)));
903}
904
905SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
906 return SoftenFloatRes_Unary(N, LC: RTLIB::getTANH(VT: N->getValueType(ResNo: 0)));
907}
908
909SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
910 return SoftenFloatRes_Unary(N, LC: RTLIB::getTRUNC(VT: N->getValueType(ResNo: 0)));
911}
912
913SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
914 LoadSDNode *L = cast<LoadSDNode>(Val: N);
915 EVT VT = N->getValueType(ResNo: 0);
916 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
917 SDLoc dl(N);
918
919 auto MMOFlags =
920 L->getMemOperand()->getFlags() &
921 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
922 SDValue NewL;
923 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
924 // If softening widens the integer representation (e.g. x86_fp80 -> i96),
925 // load the original memory width and extend to the softened type.
926 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
927 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::EXTLOAD, VT: NVT, dl,
928 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
929 PtrInfo: L->getPointerInfo(), MemVT, Alignment: L->getBaseAlign(), MMOFlags,
930 Metadata: L->getMMOMetadataForSubAccess());
931 // Legalized the chain result - switch anything that used the old chain to
932 // use the new one.
933 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
934 return NewL;
935 }
936
937 // Do a non-extending load followed by FP_EXTEND.
938 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
939 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
940 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(), Alignment: L->getBaseAlign(),
941 MMOFlags, Metadata: L->getMMOMetadataForSubAccess());
942 // Legalized the chain result - switch anything that used the old chain to
943 // use the new one.
944 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
945 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
946 return BitConvertToInteger(Op: ExtendNode);
947}
948
949SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
950 AtomicSDNode *L = cast<AtomicSDNode>(Val: N);
951 EVT VT = N->getValueType(ResNo: 0);
952 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
953 SDLoc dl(N);
954
955 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
956 SDValue NewL =
957 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
958 Ops: {L->getChain(), L->getBasePtr()}, MMO: L->getMemOperand());
959
960 // Legalized the chain result - switch anything that used the old chain to
961 // use the new one.
962 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
963 return NewL;
964 }
965
966 report_fatal_error(reason: "softening fp extending atomic load not handled");
967}
968
969SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
970 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
971 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
972 return DAG.getSelect(DL: SDLoc(N),
973 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
974}
975
976SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
977 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
978 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
979 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
980 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
981 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
982}
983
984SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
985 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
986 VT: N->getValueType(ResNo: 0)));
987}
988
989SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
990 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
991 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
992 EVT VT = N->getValueType(ResNo: 0);
993 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
994 SDLoc dl(N);
995
996 SDValue NewVAARG;
997 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
998 Align: N->getConstantOperandVal(Num: 3));
999
1000 // Legalized the chain result - switch anything that used the old chain to
1001 // use the new one.
1002 if (N != NewVAARG.getValue(R: 1).getNode())
1003 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
1004 return NewVAARG;
1005}
1006
1007SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1008 bool IsStrict = N->isStrictFPOpcode();
1009 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1010 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1011 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
1012 EVT RVT = N->getValueType(ResNo: 0);
1013 EVT NVT = EVT();
1014 SDLoc dl(N);
1015
1016 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1017 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1018 // match. Look for an appropriate libcall.
1019 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1020 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1021 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1022 NVT = (MVT::SimpleValueType)t;
1023 // The source needs to big enough to hold the operand.
1024 if (NVT.bitsGE(VT: SVT))
1025 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
1026 }
1027 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1028
1029 EVT NRVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT);
1030 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1031 if (LCImpl == RTLIB::Unsupported)
1032 return SoftenFloatRes_NoLibcall(N, NVT: NRVT);
1033
1034 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1035 // Sign/zero extend the argument if the libcall takes a larger type.
1036 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
1037 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
1038 TargetLowering::MakeLibCallOptions CallOptions;
1039 CallOptions.setIsSigned(Signed);
1040 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1041 std::pair<SDValue, SDValue> Tmp =
1042 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NRVT, Ops: Op, CallOptions, dl, Chain);
1043
1044 if (IsStrict)
1045 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1046 return Tmp.first;
1047}
1048
1049SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1050 // Expand and soften recursively.
1051 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
1052 return SDValue();
1053}
1054
1055SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1056 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
1057 return SDValue();
1058}
1059
1060//===----------------------------------------------------------------------===//
1061// Convert Float Operand to Integer
1062//===----------------------------------------------------------------------===//
1063
1064bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1065 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1066 SDValue Res = SDValue();
1067
1068 switch (N->getOpcode()) {
1069 default:
1070#ifndef NDEBUG
1071 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1072 N->dump(&DAG); dbgs() << "\n";
1073#endif
1074 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
1075
1076 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1077 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1078 case ISD::STRICT_FP_TO_FP16:
1079 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1080 case ISD::FP_TO_BF16:
1081 case ISD::STRICT_FP_TO_BF16:
1082 case ISD::STRICT_FP_ROUND:
1083 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1084 case ISD::STRICT_FP_TO_SINT:
1085 case ISD::STRICT_FP_TO_UINT:
1086 case ISD::FP_TO_SINT:
1087 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1088 case ISD::FP_TO_SINT_SAT:
1089 case ISD::FP_TO_UINT_SAT:
1090 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1091 case ISD::STRICT_LROUND:
1092 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1093 case ISD::STRICT_LLROUND:
1094 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1095 case ISD::STRICT_LRINT:
1096 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1097 case ISD::STRICT_LLRINT:
1098 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1099 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1100 case ISD::STRICT_FSETCC:
1101 case ISD::STRICT_FSETCCS:
1102 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1103 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1104 case ISD::ATOMIC_STORE:
1105 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1106 break;
1107 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1108 case ISD::FAKE_USE:
1109 Res = SoftenFloatOp_FAKE_USE(N);
1110 break;
1111 case ISD::STACKMAP:
1112 Res = SoftenFloatOp_STACKMAP(N, OpNo);
1113 break;
1114 case ISD::PATCHPOINT:
1115 Res = SoftenFloatOp_PATCHPOINT(N, OpNo);
1116 break;
1117 }
1118
1119 // If the result is null, the sub-method took care of registering results etc.
1120 if (!Res.getNode()) return false;
1121
1122 // If the result is N, the sub-method updated N in place. Tell the legalizer
1123 // core about this to re-analyze.
1124 if (Res.getNode() == N)
1125 return true;
1126
1127 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1128 "Invalid operand softening");
1129
1130 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1131 return false;
1132}
1133
1134SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1135 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
1136
1137 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
1138}
1139
1140SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1141 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1142 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1143 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1144 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1145 N->getOpcode() == ISD::FP_TO_BF16 ||
1146 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1147 N->getOpcode() == ISD::STRICT_FP_ROUND);
1148
1149 bool IsStrict = N->isStrictFPOpcode();
1150 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1151 EVT SVT = Op.getValueType();
1152 EVT RVT = N->getValueType(ResNo: 0);
1153 EVT FloatRVT = RVT;
1154 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1155 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1156 FloatRVT = MVT::f16;
1157 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1158 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1159 FloatRVT = MVT::bf16;
1160
1161 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
1162 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1163
1164 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1165 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1166 if (LCImpl == RTLIB::Unsupported) {
1167 reportNoLibcall(DAG, N, VT: SVT);
1168 SDValue Poison = DAG.getPOISON(VT: RVT);
1169 if (IsStrict) {
1170 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1171 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1172 return SDValue();
1173 }
1174 return Poison;
1175 }
1176 Op = GetSoftenedFloat(Op);
1177 TargetLowering::MakeLibCallOptions CallOptions;
1178 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1179 std::pair<SDValue, SDValue> Tmp =
1180 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1181 if (IsStrict) {
1182 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1183 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1184 return SDValue();
1185 }
1186 return Tmp.first;
1187}
1188
1189SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1190 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1191 ISD::CondCode CCCode = N->getOperandCondCode(Num: 1);
1192
1193 EVT VT = NewLHS.getValueType();
1194 NewLHS = GetSoftenedFloat(Op: NewLHS);
1195 NewRHS = GetSoftenedFloat(Op: NewRHS);
1196 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1197 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1198
1199 // If softenSetCCOperands returned a scalar, we need to compare the result
1200 // against zero to select between true and false values.
1201 if (!NewRHS.getNode()) {
1202 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1203 CCCode = ISD::SETNE;
1204 }
1205
1206 // Update N to have the operands specified.
1207 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1208 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1209 Op5: N->getOperand(Num: 4)),
1210 0);
1211}
1212
1213// Even if the result type is legal, no libcall may exactly match. (e.g. We
1214// don't have FP-i8 conversions) This helper method looks for an appropriate
1215// promoted libcall.
1216static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1217 bool Signed) {
1218 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1219 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1220 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1221 ++IntVT) {
1222 Promoted = (MVT::SimpleValueType)IntVT;
1223 // The type needs to big enough to hold the result.
1224 if (Promoted.bitsGE(VT: RetVT))
1225 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1226 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1227 }
1228 return LC;
1229}
1230
1231SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1232 bool IsStrict = N->isStrictFPOpcode();
1233 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1234 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1235
1236 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1237 EVT SVT = Op.getValueType();
1238 EVT RVT = N->getValueType(ResNo: 0);
1239 EVT NVT = EVT();
1240 SDLoc dl(N);
1241
1242 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1243 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1244 // match, eg. we don't have fp -> i8 conversions.
1245 // Look for an appropriate libcall.
1246 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1247 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1248 "Unsupported FP_TO_XINT!");
1249
1250 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1251 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1252 if (LCImpl == RTLIB::Unsupported) {
1253 reportNoLibcall(DAG, N, VT: SVT);
1254 SDValue Poison = DAG.getPOISON(VT: RVT);
1255 if (IsStrict) {
1256 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1257 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1258 return SDValue();
1259 }
1260 return Poison;
1261 }
1262
1263 Op = GetSoftenedFloat(Op);
1264 TargetLowering::MakeLibCallOptions CallOptions;
1265 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
1266 std::pair<SDValue, SDValue> Tmp =
1267 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
1268
1269 // Truncate the result if the libcall returns a larger type.
1270 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1271
1272 if (!IsStrict)
1273 return Res;
1274
1275 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1276 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1277 return SDValue();
1278}
1279
1280SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1281 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1282 return Res;
1283}
1284
1285SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1286 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1287 ISD::CondCode CCCode = N->getOperandCondCode(Num: 4);
1288
1289 EVT VT = NewLHS.getValueType();
1290 NewLHS = GetSoftenedFloat(Op: NewLHS);
1291 NewRHS = GetSoftenedFloat(Op: NewRHS);
1292 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1293 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1294
1295 // If softenSetCCOperands returned a scalar, we need to compare the result
1296 // against zero to select between true and false values.
1297 if (!NewRHS.getNode()) {
1298 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1299 CCCode = ISD::SETNE;
1300 }
1301
1302 // Update N to have the operands specified.
1303 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1304 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1305 Op5: DAG.getCondCode(Cond: CCCode)),
1306 0);
1307}
1308
1309SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1310 bool IsStrict = N->isStrictFPOpcode();
1311 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1312 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1313 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1314 ISD::CondCode CCCode = N->getOperandCondCode(Num: IsStrict ? 3 : 2);
1315
1316 EVT VT = Op0.getValueType();
1317 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1318 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1319 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1320 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1321
1322 // Update N to have the operands specified.
1323 if (NewRHS.getNode()) {
1324 if (IsStrict)
1325 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1326 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1327 else
1328 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1329 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1330 }
1331
1332 // Otherwise, softenSetCCOperands returned a scalar, use it.
1333 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1334 "Unexpected setcc expansion!");
1335
1336 if (IsStrict) {
1337 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1338 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1339 return SDValue();
1340 }
1341 return NewLHS;
1342}
1343
1344SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1345 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1346 assert(OpNo == 1 && "Can only soften the stored value!");
1347 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1348 SDValue Val = ST->getValue();
1349 SDLoc dl(N);
1350
1351 if (ST->isTruncatingStore())
1352 // Do an FP_ROUND followed by a non-truncating store.
1353 Val = BitConvertToInteger(
1354 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1355 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1356 else
1357 Val = GetSoftenedFloat(Op: Val);
1358
1359 // If softening widens the integer representation (e.g. x86_fp80 -> i96),
1360 // truncate the value before storing to preserve the original memory width.
1361 EVT MemVT =
1362 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ST->getMemoryVT().getSizeInBits());
1363 return DAG.getTruncStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(), SVT: MemVT,
1364 MMO: ST->getMemOperand());
1365}
1366
1367SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1368 assert(OpNo == 1 && "Can only soften the stored value!");
1369 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
1370 SDValue Val = ST->getVal();
1371 EVT VT = Val.getValueType();
1372 SDLoc dl(N);
1373
1374 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1375
1376 SDValue NewVal = GetSoftenedFloat(Op: Val);
1377 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: VT, Chain: ST->getChain(), Ptr: NewVal,
1378 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
1379}
1380
1381SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1382 SDValue LHS = N->getOperand(Num: 0);
1383 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1384 SDLoc dl(N);
1385
1386 EVT LVT = LHS.getValueType();
1387 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1388 EVT RVT = RHS.getValueType();
1389
1390 unsigned LSize = LVT.getSizeInBits();
1391 unsigned RSize = RVT.getSizeInBits();
1392
1393 // Shift right or sign-extend it if the two operands have different types.
1394 int SizeDiff = RSize - LSize;
1395 if (SizeDiff > 0) {
1396 RHS =
1397 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1398 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1399 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1400 DL: DAG.getDataLayout())));
1401 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1402 } else if (SizeDiff < 0) {
1403 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1404 RHS =
1405 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1406 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1407 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1408 DL: DAG.getDataLayout())));
1409 }
1410
1411 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1412 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1413}
1414
1415SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1416 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1417 bool IsStrict = N->isStrictFPOpcode();
1418 unsigned Offset = IsStrict ? 1 : 0;
1419 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1420 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1421 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1422 if (LCImpl == RTLIB::Unsupported) {
1423 reportNoLibcall(DAG, N, VT: N->getOperand(Num: 0 + Offset).getValueType());
1424 SDValue Poison = DAG.getPOISON(VT: N->getValueType(ResNo: 0));
1425 if (IsStrict) {
1426 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1427 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
1428 return SDValue();
1429 }
1430 return Poison;
1431 }
1432 TargetLowering::MakeLibCallOptions CallOptions;
1433 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1434 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
1435 std::pair<SDValue, SDValue> Tmp =
1436 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1437 if (IsStrict) {
1438 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1439 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1440 return SDValue();
1441 }
1442
1443 return Tmp.first;
1444}
1445
1446SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1447 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1448 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1449 Call_F32: RTLIB::LROUND_F32,
1450 Call_F64: RTLIB::LROUND_F64,
1451 Call_F80: RTLIB::LROUND_F80,
1452 Call_F128: RTLIB::LROUND_F128,
1453 Call_PPCF128: RTLIB::LROUND_PPCF128));
1454}
1455
1456SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1457 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1458 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1459 Call_F32: RTLIB::LLROUND_F32,
1460 Call_F64: RTLIB::LLROUND_F64,
1461 Call_F80: RTLIB::LLROUND_F80,
1462 Call_F128: RTLIB::LLROUND_F128,
1463 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1464}
1465
1466SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1467 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1468 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1469 Call_F32: RTLIB::LRINT_F32,
1470 Call_F64: RTLIB::LRINT_F64,
1471 Call_F80: RTLIB::LRINT_F80,
1472 Call_F128: RTLIB::LRINT_F128,
1473 Call_PPCF128: RTLIB::LRINT_PPCF128));
1474}
1475
1476SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1477 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1478 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1479 Call_F32: RTLIB::LLRINT_F32,
1480 Call_F64: RTLIB::LLRINT_F64,
1481 Call_F80: RTLIB::LLRINT_F80,
1482 Call_F128: RTLIB::LLRINT_F128,
1483 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1484}
1485
1486SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1487 SDValue Op1 = BitConvertToInteger(Op: N->getOperand(Num: 1));
1488 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1489 N1: N->getOperand(Num: 0), N2: Op1);
1490}
1491
1492SDValue DAGTypeLegalizer::SoftenFloatOp_STACKMAP(SDNode *N, unsigned OpNo) {
1493 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
1494 SmallVector<SDValue> NewOps(N->ops());
1495 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1496 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1497}
1498
1499SDValue DAGTypeLegalizer::SoftenFloatOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
1500 assert(OpNo >= 7);
1501 SmallVector<SDValue> NewOps(N->ops());
1502 NewOps[OpNo] = GetSoftenedFloat(Op: NewOps[OpNo]);
1503 return SDValue(DAG.UpdateNodeOperands(N, Ops: NewOps), 0);
1504}
1505
1506//===----------------------------------------------------------------------===//
1507// Float Result Expansion
1508//===----------------------------------------------------------------------===//
1509
1510/// ExpandFloatResult - This method is called when the specified result of the
1511/// specified node is found to need expansion. At this point, the node may also
1512/// have invalid operands or may have other results that need promotion, we just
1513/// know that (at least) one result needs expansion.
1514void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1515 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1516 SDValue Lo, Hi;
1517 Lo = Hi = SDValue();
1518
1519 // See if the target wants to custom expand this node.
1520 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1521 return;
1522
1523 switch (N->getOpcode()) {
1524 default:
1525#ifndef NDEBUG
1526 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1527 N->dump(&DAG); dbgs() << "\n";
1528#endif
1529 report_fatal_error(reason: "do not know how to expand the result of this "
1530 "operator!");
1531 // clang-format off
1532 case ISD::POISON:
1533 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1534 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1535 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1536
1537 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1538 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1539 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1540 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1541 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1542 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1543
1544 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1545 case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
1546 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1547 case ISD::STRICT_FMINNUM:
1548 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1549 case ISD::STRICT_FMAXNUM:
1550 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1551 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1552 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1553 case ISD::STRICT_FADD:
1554 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1555 case ISD::STRICT_FACOS:
1556 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1557 case ISD::STRICT_FASIN:
1558 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1559 case ISD::STRICT_FATAN:
1560 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1561 case ISD::STRICT_FATAN2:
1562 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1563 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1564 case ISD::STRICT_FCEIL:
1565 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1566 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1567 case ISD::STRICT_FCOS:
1568 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1569 case ISD::STRICT_FCOSH:
1570 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1571 case ISD::STRICT_FDIV:
1572 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1573 case ISD::STRICT_FEXP:
1574 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1575 case ISD::STRICT_FEXP2:
1576 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1577 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1578 case ISD::STRICT_FFLOOR:
1579 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1580 case ISD::STRICT_FLOG:
1581 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1582 case ISD::STRICT_FLOG2:
1583 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1584 case ISD::STRICT_FLOG10:
1585 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1586 case ISD::STRICT_FMA:
1587 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1588 case ISD::STRICT_FMUL:
1589 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1590 case ISD::STRICT_FNEARBYINT:
1591 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1592 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1593 case ISD::STRICT_FP_EXTEND:
1594 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1595 case ISD::STRICT_FPOW:
1596 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1597 case ISD::STRICT_FPOWI:
1598 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1599 case ISD::FLDEXP:
1600 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1601 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1602 case ISD::STRICT_FRINT:
1603 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1604 case ISD::STRICT_FROUND:
1605 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1606 case ISD::STRICT_FROUNDEVEN:
1607 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1608 case ISD::STRICT_FSIN:
1609 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1610 case ISD::STRICT_FSINH:
1611 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1612 case ISD::STRICT_FSQRT:
1613 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1614 case ISD::STRICT_FSUB:
1615 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1616 case ISD::STRICT_FTAN:
1617 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1618 case ISD::STRICT_FTANH:
1619 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1620 case ISD::STRICT_FTRUNC:
1621 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1622 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1623 case ISD::STRICT_SINT_TO_FP:
1624 case ISD::STRICT_UINT_TO_FP:
1625 case ISD::SINT_TO_FP:
1626 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1627 case ISD::STRICT_FREM:
1628 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1629 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1630 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1631 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1632 // clang-format on
1633 }
1634
1635 // If Lo/Hi is null, the sub-method took care of registering results etc.
1636 if (Lo.getNode())
1637 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1638}
1639
1640void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1641 SDValue &Hi) {
1642 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1643 assert(NVT.getSizeInBits() == 64 &&
1644 "Do not know how to expand this float constant!");
1645 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1646 SDLoc dl(N);
1647 const fltSemantics &Sem = NVT.getFltSemantics();
1648 Lo = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 64)), DL: dl, VT: NVT);
1649 Hi = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 0)), DL: dl, VT: NVT);
1650}
1651
1652// Diagnose a missing libcall and produce a poison expanded pair.
1653void DAGTypeLegalizer::ExpandFloatRes_NoLibcall(SDNode *N, SDValue &Lo,
1654 SDValue &Hi) {
1655 reportNoLibcall(DAG, N, VT: N->getValueType(ResNo: 0));
1656 if (N->isStrictFPOpcode())
1657 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
1658 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1659 Lo = Hi = DAG.getPOISON(VT: NVT);
1660}
1661
1662void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1663 SDValue &Lo, SDValue &Hi) {
1664 bool IsStrict = N->isStrictFPOpcode();
1665 unsigned Offset = IsStrict ? 1 : 0;
1666 SDValue Op = N->getOperand(Num: 0 + Offset);
1667 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1668 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1669 if (LCImpl == RTLIB::Unsupported)
1670 return ExpandFloatRes_NoLibcall(N, Lo, Hi);
1671 TargetLowering::MakeLibCallOptions CallOptions;
1672 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
1673 DAG, LibcallImpl: LCImpl, RetVT: N->getValueType(ResNo: 0), Ops: Op, CallOptions, dl: SDLoc(N), Chain);
1674 if (IsStrict)
1675 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1676 GetPairElements(Pair: Tmp.first, Lo, Hi);
1677}
1678
1679void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1680 SDValue &Lo, SDValue &Hi) {
1681 bool IsStrict = N->isStrictFPOpcode();
1682 unsigned Offset = IsStrict ? 1 : 0;
1683 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1684 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1685 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1686 if (LCImpl == RTLIB::Unsupported)
1687 return ExpandFloatRes_NoLibcall(N, Lo, Hi);
1688 TargetLowering::MakeLibCallOptions CallOptions;
1689 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
1690 DAG, LibcallImpl: LCImpl, RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain);
1691 if (IsStrict)
1692 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1693 GetPairElements(Pair: Tmp.first, Lo, Hi);
1694}
1695
1696void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1697 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(VT: N->getValueType(ResNo: 0)),
1698 /*CallRetResNo=*/0);
1699}
1700
1701void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1702 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(VT: N->getValueType(ResNo: 0)));
1703}
1704
1705void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1706 ExpandFloatRes_UnaryWithTwoFPResults(N,
1707 LC: RTLIB::getSINCOSPI(VT: N->getValueType(ResNo: 0)));
1708}
1709
1710void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1711 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1712 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1713 SmallVector<SDValue> Results;
1714 if (!TLI.expandMultipleResultFPLibCall(DAG, LC, Node: N, Results, CallRetResNo)) {
1715 reportNoLibcall(DAG, N, VT: N->getValueType(ResNo: 0));
1716 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1717 SDValue Poison = DAG.getPOISON(VT: NVT);
1718 for (unsigned ResNo = 0, E = N->getNumValues(); ResNo != E; ++ResNo)
1719 SetExpandedFloat(Op: SDValue(N, ResNo), Lo: Poison, Hi: Poison);
1720 return;
1721 }
1722 for (auto [ResNo, Res] : enumerate(First&: Results)) {
1723 SDValue Lo, Hi;
1724 GetPairElements(Pair: Res, Lo, Hi);
1725 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1726 }
1727}
1728
1729void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1730 SDValue &Hi) {
1731 assert(N->getValueType(0) == MVT::ppcf128 &&
1732 "Logic only correct for ppcf128!");
1733 SDLoc dl(N);
1734 SDValue Tmp;
1735 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1736 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1737 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1738 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1739 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1740 Cond: ISD::SETEQ);
1741}
1742
1743void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1744 SDValue &Hi) {
1745 ExpandFloatRes_Binary(N, LC: RTLIB::getFMIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1746}
1747
1748void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1749 SDValue &Hi) {
1750 ExpandFloatRes_Binary(N, LC: RTLIB::getFMAX(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1751}
1752
1753void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1754 SDValue &Hi) {
1755 ExpandFloatRes_Binary(N, LC: RTLIB::getFMINIMUM_NUM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1756}
1757
1758void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1759 SDValue &Hi) {
1760 ExpandFloatRes_Binary(N, LC: RTLIB::getFMAXIMUM_NUM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1761}
1762
1763void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1764 SDValue &Hi) {
1765 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1766 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1767 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1768 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1769}
1770
1771void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1772 SDValue &Hi) {
1773 ExpandFloatRes_Unary(N, LC: RTLIB::getACOS(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1774}
1775
1776void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1777 SDValue &Hi) {
1778 ExpandFloatRes_Unary(N, LC: RTLIB::getASIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1779}
1780
1781void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1782 SDValue &Hi) {
1783 ExpandFloatRes_Unary(N, LC: RTLIB::getATAN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1784}
1785
1786void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1787 SDValue &Hi) {
1788 ExpandFloatRes_Binary(N, LC: RTLIB::getATAN2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1789}
1790
1791void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1792 SDValue &Hi) {
1793 ExpandFloatRes_Unary(N, LC: RTLIB::getCBRT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1794}
1795
1796void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1797 SDValue &Lo, SDValue &Hi) {
1798 ExpandFloatRes_Unary(N, LC: RTLIB::getCEIL(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1799}
1800
1801void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1802 SDValue &Lo, SDValue &Hi) {
1803 ExpandFloatRes_Binary(N, LC: RTLIB::getCOPYSIGN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1804}
1805
1806void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1807 SDValue &Lo, SDValue &Hi) {
1808 ExpandFloatRes_Unary(N, LC: RTLIB::getCOS(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1809}
1810
1811void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1812 SDValue &Hi) {
1813 ExpandFloatRes_Unary(N, LC: RTLIB::getCOSH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1814}
1815
1816void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1817 SDValue &Hi) {
1818 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1819 Call_F32: RTLIB::DIV_F32,
1820 Call_F64: RTLIB::DIV_F64,
1821 Call_F80: RTLIB::DIV_F80,
1822 Call_F128: RTLIB::DIV_F128,
1823 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1824}
1825
1826void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1827 SDValue &Lo, SDValue &Hi) {
1828 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1829}
1830
1831void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1832 SDValue &Lo, SDValue &Hi) {
1833 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1834}
1835
1836void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1837 SDValue &Hi) {
1838 ExpandFloatRes_Unary(N, LC: RTLIB::getEXP10(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1839}
1840
1841void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1842 SDValue &Lo, SDValue &Hi) {
1843 ExpandFloatRes_Unary(N, LC: RTLIB::getFLOOR(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1844}
1845
1846void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1847 SDValue &Lo, SDValue &Hi) {
1848 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1849}
1850
1851void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1852 SDValue &Lo, SDValue &Hi) {
1853 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG2(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1854}
1855
1856void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1857 SDValue &Lo, SDValue &Hi) {
1858 ExpandFloatRes_Unary(N, LC: RTLIB::getLOG10(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1859}
1860
1861void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1862 SDValue &Hi) {
1863 bool IsStrict = N->isStrictFPOpcode();
1864 unsigned Offset = IsStrict ? 1 : 0;
1865 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1866 N->getOperand(Num: 2 + Offset) };
1867 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1868 RTLIB::Libcall LC = RTLIB::getFMA(VT: N->getValueType(ResNo: 0));
1869 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
1870 if (LCImpl == RTLIB::Unsupported)
1871 return ExpandFloatRes_NoLibcall(N, Lo, Hi);
1872 TargetLowering::MakeLibCallOptions CallOptions;
1873 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
1874 DAG, LibcallImpl: LCImpl, RetVT: N->getValueType(ResNo: 0), Ops, CallOptions, dl: SDLoc(N), Chain);
1875 if (IsStrict)
1876 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1877 GetPairElements(Pair: Tmp.first, Lo, Hi);
1878}
1879
1880void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1881 SDValue &Hi) {
1882 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1883 Call_F32: RTLIB::MUL_F32,
1884 Call_F64: RTLIB::MUL_F64,
1885 Call_F80: RTLIB::MUL_F80,
1886 Call_F128: RTLIB::MUL_F128,
1887 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1888}
1889
1890void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1891 SDValue &Lo, SDValue &Hi) {
1892 ExpandFloatRes_Unary(N, LC: RTLIB::getNEARBYINT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1893}
1894
1895void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1896 SDValue &Hi) {
1897 SDLoc dl(N);
1898 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1899 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1900 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1901}
1902
1903void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
1904 SDValue &Hi) {
1905 // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
1906 SDLoc dl(N);
1907 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1908}
1909
1910void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1911 SDValue &Hi) {
1912 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1913 SDLoc dl(N);
1914 bool IsStrict = N->isStrictFPOpcode();
1915
1916 SDValue Chain;
1917 if (IsStrict) {
1918 // If the expanded type is the same as the input type, just bypass the node.
1919 if (NVT == N->getOperand(Num: 1).getValueType()) {
1920 Hi = N->getOperand(Num: 1);
1921 Chain = N->getOperand(Num: 0);
1922 } else {
1923 // Other we need to extend.
1924 Hi = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: { NVT, MVT::Other },
1925 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
1926 Chain = Hi.getValue(R: 1);
1927 }
1928 } else {
1929 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1930 }
1931
1932 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
1933
1934 if (IsStrict)
1935 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1936}
1937
1938void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1939 SDValue &Lo, SDValue &Hi) {
1940 ExpandFloatRes_Binary(N, LC: RTLIB::getPOW(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1941}
1942
1943void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1944 SDValue &Lo, SDValue &Hi) {
1945 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1946}
1947
1948void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1949 SDValue &Hi) {
1950 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1951}
1952
1953void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1954 SDValue &Lo, SDValue &Hi) {
1955 assert(N->getValueType(0) == MVT::ppcf128 &&
1956 "Logic only correct for ppcf128!");
1957
1958 SDLoc dl(N);
1959 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1960 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1961 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1962}
1963
1964void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1965 SDValue &Lo, SDValue &Hi) {
1966 ExpandFloatRes_Binary(N, LC: RTLIB::getREM(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1967}
1968
1969void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1970 SDValue &Lo, SDValue &Hi) {
1971 ExpandFloatRes_Unary(N, LC: RTLIB::getRINT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1972}
1973
1974void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1975 SDValue &Lo, SDValue &Hi) {
1976 ExpandFloatRes_Unary(N, LC: RTLIB::getROUND(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1977}
1978
1979void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1980 SDValue &Lo, SDValue &Hi) {
1981 ExpandFloatRes_Unary(N, LC: RTLIB::getROUNDEVEN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1982}
1983
1984void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1985 SDValue &Lo, SDValue &Hi) {
1986 ExpandFloatRes_Unary(N, LC: RTLIB::getSIN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1987}
1988
1989void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
1990 SDValue &Hi) {
1991 ExpandFloatRes_Unary(N, LC: RTLIB::getSINH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1992}
1993
1994void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1995 SDValue &Lo, SDValue &Hi) {
1996 ExpandFloatRes_Unary(N, LC: RTLIB::getSQRT(VT: N->getValueType(ResNo: 0)), Lo, Hi);
1997}
1998
1999void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
2000 SDValue &Hi) {
2001 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2002 Call_F32: RTLIB::SUB_F32,
2003 Call_F64: RTLIB::SUB_F64,
2004 Call_F80: RTLIB::SUB_F80,
2005 Call_F128: RTLIB::SUB_F128,
2006 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
2007}
2008
2009void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
2010 SDValue &Hi) {
2011 ExpandFloatRes_Unary(N, LC: RTLIB::getTAN(VT: N->getValueType(ResNo: 0)), Lo, Hi);
2012}
2013
2014void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
2015 SDValue &Hi) {
2016 ExpandFloatRes_Unary(N, LC: RTLIB::getTANH(VT: N->getValueType(ResNo: 0)), Lo, Hi);
2017}
2018
2019void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
2020 SDValue &Lo, SDValue &Hi) {
2021 ExpandFloatRes_Unary(N, LC: RTLIB::getTRUNC(VT: N->getValueType(ResNo: 0)), Lo, Hi);
2022}
2023
2024void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
2025 SDValue &Hi) {
2026 if (ISD::isNormalLoad(N)) {
2027 ExpandRes_NormalLoad(N, Lo, Hi);
2028 return;
2029 }
2030
2031 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2032 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
2033 SDValue Chain = LD->getChain();
2034 SDValue Ptr = LD->getBasePtr();
2035 SDLoc dl(N);
2036
2037 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
2038 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2039 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2040
2041 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
2042 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
2043
2044 // Remember the chain.
2045 Chain = Hi.getValue(R: 1);
2046
2047 // The low part is zero.
2048 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2049
2050 // Modified the chain - switch anything that used the old chain to use the
2051 // new one.
2052 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
2053}
2054
2055void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2056 SDValue &Hi) {
2057 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2058 EVT VT = N->getValueType(ResNo: 0);
2059 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2060 bool Strict = N->isStrictFPOpcode();
2061 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
2062 EVT SrcVT = Src.getValueType();
2063 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2064 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2065 SDLoc dl(N);
2066 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
2067
2068 // TODO: Any other flags to propagate?
2069 SDNodeFlags Flags;
2070 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2071
2072 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2073 // When promoting partial word types to i32 we must honor the signedness,
2074 // though.
2075 if (SrcVT.bitsLE(VT: MVT::i32)) {
2076 // The integer can be represented exactly in an f64.
2077 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2078 if (Strict) {
2079 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
2080 Ops: {Chain, Src}, Flags);
2081 Chain = Hi.getValue(R: 1);
2082 } else
2083 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
2084 } else {
2085 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2086 if (SrcVT.bitsLE(VT: MVT::i64)) {
2087 Src = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
2088 VT: MVT::i64, Operand: Src);
2089 LC = RTLIB::SINTTOFP_I64_PPCF128;
2090 } else if (SrcVT.bitsLE(VT: MVT::i128)) {
2091 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i128, Operand: Src);
2092 LC = RTLIB::SINTTOFP_I128_PPCF128;
2093 }
2094 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2095
2096 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2097 if (LCImpl == RTLIB::Unsupported)
2098 return ExpandFloatRes_NoLibcall(N, Lo, Hi);
2099
2100 TargetLowering::MakeLibCallOptions CallOptions;
2101 CallOptions.setIsSigned(true);
2102 std::pair<SDValue, SDValue> Tmp =
2103 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
2104 if (Strict)
2105 Chain = Tmp.second;
2106 GetPairElements(Pair: Tmp.first, Lo, Hi);
2107 }
2108
2109 // No need to complement for unsigned 32-bit integers
2110 if (isSigned || SrcVT.bitsLE(VT: MVT::i32)) {
2111 if (Strict)
2112 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2113
2114 return;
2115 }
2116
2117 // Unsigned - fix up the SINT_TO_FP value just calculated.
2118 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2119 // keep semantics correctness if the integer is not exactly representable
2120 // here. See ExpandLegalINT_TO_FP.
2121 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
2122 SrcVT = Src.getValueType();
2123
2124 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2125 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2126 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2127 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2128 ArrayRef<uint64_t> Parts;
2129
2130 switch (SrcVT.getSimpleVT().SimpleTy) {
2131 default:
2132 llvm_unreachable("Unsupported UINT_TO_FP!");
2133 case MVT::i32:
2134 Parts = TwoE32;
2135 break;
2136 case MVT::i64:
2137 Parts = TwoE64;
2138 break;
2139 case MVT::i128:
2140 Parts = TwoE128;
2141 break;
2142 }
2143
2144 // TODO: Are there other fast-math-flags to propagate to this FADD?
2145 SDValue NewLo = DAG.getConstantFP(
2146 Val: APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), DL: dl, VT: MVT::ppcf128);
2147 if (Strict) {
2148 Lo = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other),
2149 Ops: {Chain, Hi, NewLo}, Flags);
2150 Chain = Lo.getValue(R: 1);
2151 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2152 } else
2153 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
2154 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
2155 True: Lo, False: Hi, Cond: ISD::SETLT);
2156 GetPairElements(Pair: Lo, Lo, Hi);
2157}
2158
2159
2160//===----------------------------------------------------------------------===//
2161// Float Operand Expansion
2162//===----------------------------------------------------------------------===//
2163
2164/// ExpandFloatOperand - This method is called when the specified operand of the
2165/// specified node is found to need expansion. At this point, all of the result
2166/// types of the node are known to be legal, but other operands of the node may
2167/// need promotion or expansion as well as the specified one.
2168bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2169 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2170 SDValue Res = SDValue();
2171
2172 // See if the target wants to custom expand this node.
2173 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2174 return false;
2175
2176 switch (N->getOpcode()) {
2177 default:
2178#ifndef NDEBUG
2179 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2180 N->dump(&DAG); dbgs() << "\n";
2181#endif
2182 report_fatal_error(reason: "do not know how to expand this operator's operand!");
2183
2184 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2185 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2186 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2187
2188 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2189 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2190 case ISD::STRICT_FP_ROUND:
2191 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2192 case ISD::STRICT_FP_TO_SINT:
2193 case ISD::STRICT_FP_TO_UINT:
2194 case ISD::FP_TO_SINT:
2195 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2196 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2197 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2198 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2199 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2200 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2201 case ISD::STRICT_FSETCC:
2202 case ISD::STRICT_FSETCCS:
2203 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2204 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
2205 OpNo); break;
2206 }
2207
2208 // If the result is null, the sub-method took care of registering results etc.
2209 if (!Res.getNode()) return false;
2210
2211 // If the result is N, the sub-method updated N in place. Tell the legalizer
2212 // core about this.
2213 if (Res.getNode() == N)
2214 return true;
2215
2216 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2217 "Invalid operand expansion");
2218
2219 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2220 return false;
2221}
2222
2223/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2224/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2225void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2226 SDValue &NewRHS,
2227 ISD::CondCode &CCCode,
2228 const SDLoc &dl, SDValue &Chain,
2229 bool IsSignaling) {
2230 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2231 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
2232 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
2233
2234 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2235
2236 // FIXME: This generated code sucks. We want to generate
2237 // FCMPU crN, hi1, hi2
2238 // BNE crN, L:
2239 // FCMPU crN, lo1, lo2
2240 // The following can be improved, but not that much.
2241 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2242 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2243 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
2244 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2245 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
2246 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
2247 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2248 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2249 Tmp1 =
2250 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
2251 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
2252 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2253 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2254 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
2255 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2256 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2257 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
2258 NewRHS = SDValue(); // LHS is the result, not a compare.
2259 Chain = OutputChain;
2260}
2261
2262SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2263 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
2264 ISD::CondCode CCCode = N->getOperandCondCode(Num: 1);
2265 SDValue Chain;
2266 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2267
2268 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2269 // against zero to select between true and false values.
2270 if (!NewRHS.getNode()) {
2271 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2272 CCCode = ISD::SETNE;
2273 }
2274
2275 // Update N to have the operands specified.
2276 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2277 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
2278 Op5: N->getOperand(Num: 4)), 0);
2279}
2280
2281SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2282 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2283 "Logic only correct for ppcf128!");
2284 SDValue Lo, Hi;
2285 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2286 // The ppcf128 value is providing only the sign; take it from the
2287 // higher-order double (which must have the larger magnitude).
2288 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2289 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2290}
2291
2292SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2293 bool IsStrict = N->isStrictFPOpcode();
2294 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2295 "Logic only correct for ppcf128!");
2296 SDValue Lo, Hi;
2297 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2298
2299 if (!IsStrict)
2300 // Round it the rest of the way (e.g. to f32) if needed.
2301 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2302 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2303
2304 // Eliminate the node if the input float type is the same as the output float
2305 // type.
2306 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2307 // Connect the output chain to the input chain, unlinking the node.
2308 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2309 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2310 return SDValue();
2311 }
2312
2313 SDValue Expansion = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
2314 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
2315 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
2316 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2317 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2318 return SDValue();
2319}
2320
2321SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2322 EVT RVT = N->getValueType(ResNo: 0);
2323 SDLoc dl(N);
2324
2325 bool IsStrict = N->isStrictFPOpcode();
2326 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2327 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2328 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2329 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2330
2331 EVT NVT;
2332 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2333 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2334 "Unsupported FP_TO_XINT!");
2335 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2336 if (LCImpl == RTLIB::Unsupported) {
2337 reportNoLibcall(DAG, N, VT: Op.getValueType());
2338 SDValue Poison = DAG.getPOISON(VT: RVT);
2339 if (IsStrict) {
2340 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2341 ReplaceValueWith(From: SDValue(N, 0), To: Poison);
2342 return SDValue();
2343 }
2344 return Poison;
2345 }
2346 TargetLowering::MakeLibCallOptions CallOptions;
2347 std::pair<SDValue, SDValue> Tmp =
2348 TLI.makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2349 if (!IsStrict)
2350 return Tmp.first;
2351
2352 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2353 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2354 return SDValue();
2355}
2356
2357SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2358 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2359 ISD::CondCode CCCode = N->getOperandCondCode(Num: 4);
2360 SDValue Chain;
2361 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2362
2363 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2364 // against zero to select between true and false values.
2365 if (!NewRHS.getNode()) {
2366 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2367 CCCode = ISD::SETNE;
2368 }
2369
2370 // Update N to have the operands specified.
2371 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2372 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2373 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2374}
2375
2376SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2377 bool IsStrict = N->isStrictFPOpcode();
2378 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2379 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2380 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2381 ISD::CondCode CCCode = N->getOperandCondCode(Num: IsStrict ? 3 : 2);
2382 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2383 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2384
2385 // FloatExpandSetCCOperands always returned a scalar.
2386 assert(!NewRHS.getNode() && "Expect to return scalar");
2387 assert(NewLHS.getValueType() == N->getValueType(0) &&
2388 "Unexpected setcc expansion!");
2389 if (Chain) {
2390 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2391 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2392 return SDValue();
2393 }
2394 return NewLHS;
2395}
2396
2397SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2398 if (ISD::isNormalStore(N))
2399 return ExpandOp_NormalStore(N, OpNo);
2400
2401 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2402 assert(OpNo == 1 && "Can only expand the stored value so far");
2403 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2404
2405 SDValue Chain = ST->getChain();
2406 SDValue Ptr = ST->getBasePtr();
2407
2408 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2409 VT: ST->getValue().getValueType());
2410 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2411 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2412 (void)NVT;
2413
2414 SDValue Lo, Hi;
2415 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2416
2417 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2418 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2419}
2420
2421SDValue DAGTypeLegalizer::ExpandFloatOp_XRINT_XROUND(SDNode *N,
2422 RTLIB::Libcall LC) {
2423 EVT RVT = N->getValueType(ResNo: 0);
2424 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2425 if (LCImpl == RTLIB::Unsupported) {
2426 reportNoLibcall(DAG, N, VT: N->getOperand(Num: 0).getValueType());
2427 return DAG.getPOISON(VT: RVT);
2428 }
2429
2430 TargetLowering::MakeLibCallOptions CallOptions;
2431 return TLI
2432 .makeLibCall(DAG, LibcallImpl: LCImpl, RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N))
2433 .first;
2434}
2435
2436SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2437 EVT RetVT = N->getOperand(Num: 0).getValueType();
2438 return ExpandFloatOp_XRINT_XROUND(
2439 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LROUND_F32, Call_F64: RTLIB::LROUND_F64,
2440 Call_F80: RTLIB::LROUND_F80, Call_F128: RTLIB::LROUND_F128,
2441 Call_PPCF128: RTLIB::LROUND_PPCF128));
2442}
2443
2444SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2445 EVT RetVT = N->getOperand(Num: 0).getValueType();
2446 return ExpandFloatOp_XRINT_XROUND(
2447 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LLROUND_F32, Call_F64: RTLIB::LLROUND_F64,
2448 Call_F80: RTLIB::LLROUND_F80, Call_F128: RTLIB::LLROUND_F128,
2449 Call_PPCF128: RTLIB::LLROUND_PPCF128));
2450}
2451
2452SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2453 EVT RetVT = N->getOperand(Num: 0).getValueType();
2454 return ExpandFloatOp_XRINT_XROUND(
2455 N,
2456 LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LRINT_F32, Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
2457 Call_F128: RTLIB::LRINT_F128, Call_PPCF128: RTLIB::LRINT_PPCF128));
2458}
2459
2460SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2461 EVT RetVT = N->getOperand(Num: 0).getValueType();
2462 return ExpandFloatOp_XRINT_XROUND(
2463 N, LC: GetFPLibCall(VT: RetVT, Call_F32: RTLIB::LLRINT_F32, Call_F64: RTLIB::LLRINT_F64,
2464 Call_F80: RTLIB::LLRINT_F80, Call_F128: RTLIB::LLRINT_F128,
2465 Call_PPCF128: RTLIB::LLRINT_PPCF128));
2466}
2467
2468//===----------------------------------------------------------------------===//
2469// Float Operand Promotion
2470//===----------------------------------------------------------------------===//
2471//
2472
2473static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2474 if (OpVT == MVT::f16)
2475 return ISD::FP16_TO_FP;
2476 if (RetVT == MVT::f16)
2477 return ISD::FP_TO_FP16;
2478 if (OpVT == MVT::bf16)
2479 return ISD::BF16_TO_FP;
2480 if (RetVT == MVT::bf16)
2481 return ISD::FP_TO_BF16;
2482 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2483}
2484
2485static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2486 if (OpVT == MVT::f16)
2487 return ISD::STRICT_FP16_TO_FP;
2488 if (RetVT == MVT::f16)
2489 return ISD::STRICT_FP_TO_FP16;
2490 if (OpVT == MVT::bf16)
2491 return ISD::STRICT_BF16_TO_FP;
2492 if (RetVT == MVT::bf16)
2493 return ISD::STRICT_FP_TO_BF16;
2494 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2495}
2496
2497SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2498 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2499 SDLoc SL(N);
2500
2501 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
2502 EVT CastVT = CastVal.getValueType();
2503
2504 SDValue NewAtomic
2505 = DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl: SL, MemVT: CastVT,
2506 VTList: DAG.getVTList(VT1: CastVT, VT2: MVT::Other),
2507 Ops: { AM->getChain(), AM->getBasePtr(), CastVal },
2508 MMO: AM->getMemOperand());
2509
2510 SDValue Result = NewAtomic;
2511
2512 // Legalize the chain result by replacing uses of the old value chain with the
2513 // new one
2514 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
2515
2516 return Result;
2517}
2518
2519//===----------------------------------------------------------------------===//
2520// Half Result Soft Promotion
2521//===----------------------------------------------------------------------===//
2522
2523void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2524 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2525 N->dump(&DAG));
2526 SDValue R = SDValue();
2527
2528 // See if the target wants to custom expand this node.
2529 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2530 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2531 return;
2532 }
2533
2534 switch (N->getOpcode()) {
2535 default:
2536#ifndef NDEBUG
2537 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2538 N->dump(&DAG); dbgs() << "\n";
2539#endif
2540 report_fatal_error(reason: "Do not know how to soft promote this operator's "
2541 "result!");
2542
2543 case ISD::ARITH_FENCE:
2544 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2545 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2546 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2547 case ISD::EXTRACT_VECTOR_ELT:
2548 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2549 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2550 case ISD::STRICT_FP_ROUND:
2551 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2552
2553 // Unary FP Operations
2554 case ISD::FACOS:
2555 case ISD::FASIN:
2556 case ISD::FATAN:
2557 case ISD::FCBRT:
2558 case ISD::FCEIL:
2559 case ISD::FCOS:
2560 case ISD::FCOSH:
2561 case ISD::FEXP:
2562 case ISD::FEXP2:
2563 case ISD::FEXP10:
2564 case ISD::FFLOOR:
2565 case ISD::FLOG:
2566 case ISD::FLOG2:
2567 case ISD::FLOG10:
2568 case ISD::FNEARBYINT:
2569 case ISD::FREEZE:
2570 case ISD::FRINT:
2571 case ISD::FROUND:
2572 case ISD::FROUNDEVEN:
2573 case ISD::FSIN:
2574 case ISD::FSINH:
2575 case ISD::FSQRT:
2576 case ISD::FTRUNC:
2577 case ISD::FTAN:
2578 case ISD::FTANH:
2579 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2580 case ISD::FABS:
2581 R = SoftPromoteHalfRes_FABS(N);
2582 break;
2583 case ISD::FNEG:
2584 R = SoftPromoteHalfRes_FNEG(N);
2585 break;
2586 case ISD::AssertNoFPClass:
2587 R = SoftPromoteHalfRes_AssertNoFPClass(N);
2588 break;
2589
2590 // Binary FP Operations
2591 case ISD::FADD:
2592 case ISD::FDIV:
2593 case ISD::FMAXIMUM:
2594 case ISD::FMINIMUM:
2595 case ISD::FMAXIMUMNUM:
2596 case ISD::FMINIMUMNUM:
2597 case ISD::FMAXNUM:
2598 case ISD::FMINNUM:
2599 case ISD::FMUL:
2600 case ISD::FPOW:
2601 case ISD::FATAN2:
2602 case ISD::FREM:
2603 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2604
2605 case ISD::FMA: // FMA is same as FMAD
2606 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2607
2608 case ISD::FPOWI:
2609 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2610
2611 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2612
2613 case ISD::FMODF:
2614 case ISD::FSINCOS:
2615 case ISD::FSINCOSPI:
2616 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
2617 break;
2618
2619 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2620 case ISD::ATOMIC_LOAD:
2621 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2622 break;
2623 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2624 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2625 case ISD::STRICT_SINT_TO_FP:
2626 case ISD::STRICT_UINT_TO_FP:
2627 case ISD::SINT_TO_FP:
2628 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2629 case ISD::CONVERT_FROM_ARBITRARY_FP:
2630 R = SoftPromoteHalfRes_CONVERT_FROM_ARBITRARY_FP(N);
2631 break;
2632 case ISD::POISON:
2633 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2634 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2635 case ISD::VECREDUCE_FADD:
2636 case ISD::VECREDUCE_FMUL:
2637 case ISD::VECREDUCE_FMIN:
2638 case ISD::VECREDUCE_FMAX:
2639 case ISD::VECREDUCE_FMAXIMUM:
2640 case ISD::VECREDUCE_FMINIMUM:
2641 case ISD::VECREDUCE_FMAXIMUMNUM:
2642 case ISD::VECREDUCE_FMINIMUMNUM:
2643 R = SoftPromoteHalfRes_VECREDUCE(N);
2644 break;
2645 case ISD::VECREDUCE_SEQ_FADD:
2646 case ISD::VECREDUCE_SEQ_FMUL:
2647 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2648 break;
2649 }
2650
2651 if (R.getNode())
2652 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
2653}
2654
2655SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2656 return DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: MVT::i16,
2657 Operand: BitConvertToInteger(Op: N->getOperand(Num: 0)));
2658}
2659
2660SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2661 return BitConvertToInteger(Op: N->getOperand(Num: 0));
2662}
2663
2664SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2665 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
2666
2667 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2668 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
2669 VT: MVT::i16);
2670}
2671
2672SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2673 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2674 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
2675 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
2676 N2: N->getOperand(Num: 1));
2677}
2678
2679SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2680 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2681 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
2682 SDLoc dl(N);
2683
2684 EVT LVT = LHS.getValueType();
2685 EVT RVT = RHS.getValueType();
2686
2687 unsigned LSize = LVT.getSizeInBits();
2688 unsigned RSize = RVT.getSizeInBits();
2689
2690 // First get the sign bit of second operand.
2691 SDValue SignBit = DAG.getNode(
2692 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
2693 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
2694 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
2695 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
2696
2697 // Shift right or sign-extend it if the two operands have different types.
2698 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2699 if (SizeDiff > 0) {
2700 SignBit =
2701 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
2702 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
2703 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2704 DL: DAG.getDataLayout())));
2705 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
2706 } else if (SizeDiff < 0) {
2707 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
2708 SignBit =
2709 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
2710 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
2711 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
2712 DL: DAG.getDataLayout())));
2713 }
2714
2715 // Clear the sign bit of the first operand.
2716 SDValue Mask = DAG.getNode(
2717 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
2718 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
2719 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
2720 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
2721 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
2722
2723 // Or the value with the sign bit.
2724 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
2725}
2726
2727SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2728 EVT OVT = N->getValueType(ResNo: 0);
2729 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2730 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2731 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2732 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2733 SDNodeFlags Flags = N->getFlags();
2734 SDLoc dl(N);
2735
2736 // Promote to the larger FP type.
2737 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2738 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2739 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2740 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
2741
2742 SDValue Res;
2743 if (OVT == MVT::f16) {
2744 // If f16 fma is not natively supported, the value must be promoted to an
2745 // f64 (and not to f32!) to prevent double rounding issues.
2746 SDValue A64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op0, Flags);
2747 SDValue B64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op1, Flags);
2748 SDValue C64 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Op2, Flags);
2749
2750 // Prefer a wide FMA node if available; otherwise expand to mul+add.
2751 SDValue WideRes;
2752 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), MVT::f64)) {
2753 WideRes = DAG.getNode(Opcode: ISD::FMA, DL: dl, VT: MVT::f64, N1: A64, N2: B64, N3: C64, Flags);
2754 } else {
2755 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT: MVT::f64, N1: A64, N2: B64, Flags);
2756 WideRes = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: MVT::f64, N1: Mul, N2: C64, Flags);
2757 }
2758
2759 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: MVT::f64, RetVT: OVT), DL: dl, VT: MVT::i16,
2760 Operand: WideRes);
2761 }
2762
2763 Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2, Flags);
2764 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2765}
2766
2767SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2768 EVT OVT = N->getValueType(ResNo: 0);
2769 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2770 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2771 SDValue Op1 = N->getOperand(Num: 1);
2772 SDLoc dl(N);
2773
2774 // Promote to the larger FP type.
2775 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
2776
2777 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
2778
2779 // Convert back to FP16 as an integer.
2780 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2781}
2782
2783SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2784 EVT OVT = N->getValueType(ResNo: 0);
2785 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2786 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2787 SDLoc dl(N);
2788
2789 // Promote to the larger FP type.
2790 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2791
2792 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
2793 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
2794
2795 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2796
2797 // Convert back to FP16 as an integer.
2798 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2799}
2800
2801SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
2802 EVT OVT = N->getValueType(ResNo: 0);
2803 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2804 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2805 SDLoc dl(N);
2806
2807 // Promote to the larger FP type.
2808 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2809 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Op);
2810
2811 // Convert back to FP16 as an integer.
2812 ISD::NodeType Truncate = GetPromotionOpcode(OpVT: NVT, RetVT: OVT);
2813 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
2814 ++ResNum) {
2815 SDValue Trunc = DAG.getNode(Opcode: Truncate, DL: dl, VT: MVT::i16, Operand: Res.getValue(R: ResNum));
2816 SetSoftPromotedHalf(Op: SDValue(N, ResNum), Result: Trunc);
2817 }
2818
2819 return SDValue();
2820}
2821
2822SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2823 EVT RVT = N->getValueType(ResNo: 0);
2824 bool IsStrict = N->isStrictFPOpcode();
2825 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2826 EVT SVT = Op.getValueType();
2827
2828 // When an f32 value is known to already be an exact bf16 (Trunc == 1), simply
2829 // keep its upper 16 bits.
2830 if (RVT == MVT::bf16 && SVT == MVT::f32 &&
2831 N->getConstantOperandVal(Num: IsStrict ? 2 : 1) == 1) {
2832 SDLoc DL(N);
2833 if (IsStrict)
2834 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2835 SDValue IntVal = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Op);
2836 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i32, N1: IntVal,
2837 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL));
2838 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Shifted);
2839 }
2840
2841 // If the input type needs to be softened, do that now so that call lowering
2842 // will see the f16 type.
2843 if (getTypeAction(VT: SVT) == TargetLowering::TypeSoftenFloat) {
2844 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: RVT);
2845 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
2846
2847 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2848 Op = GetSoftenedFloat(Op);
2849 TargetLowering::MakeLibCallOptions CallOptions;
2850 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT);
2851 std::pair<SDValue, SDValue> Tmp =
2852 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
2853 if (IsStrict)
2854 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2855 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: MVT::i16, Operand: Tmp.first);
2856 }
2857
2858 if (IsStrict) {
2859 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
2860 ResultTys: {MVT::i16, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
2861 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2862 return Res;
2863 }
2864
2865 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: MVT::i16,
2866 Operand: N->getOperand(Num: 0));
2867}
2868
2869SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2870 LoadSDNode *L = cast<LoadSDNode>(Val: N);
2871
2872 // Load the value as an integer value with the same number of bits.
2873 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2874 SDValue NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(),
2875 VT: MVT::i16, dl: SDLoc(N), Chain: L->getChain(), Ptr: L->getBasePtr(),
2876 Offset: L->getOffset(), PtrInfo: L->getPointerInfo(), MemVT: MVT::i16,
2877 Alignment: L->getBaseAlign(), MMOFlags: L->getMemOperand()->getFlags(),
2878 Metadata: L->getMMOMetadataForSubAccess());
2879 // Legalize the chain result by replacing uses of the old value chain with the
2880 // new one
2881 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
2882 return NewL;
2883}
2884
2885SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
2886 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
2887
2888 // Load the value as an integer value with the same number of bits.
2889 SDValue NewL = DAG.getAtomic(
2890 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: MVT::i16, VTList: DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other),
2891 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
2892
2893 // Legalize the chain result by replacing uses of the old value chain with the
2894 // new one
2895 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
2896 return NewL;
2897}
2898
2899SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2900 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2901 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2902 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1, RHS: Op2,
2903 Flags: N->getFlags());
2904}
2905
2906SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2907 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
2908 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
2909 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
2910 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
2911 N5: N->getOperand(Num: 4));
2912}
2913
2914SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2915 EVT OVT = N->getValueType(ResNo: 0);
2916 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2917 SDLoc dl(N);
2918
2919 if (N->isStrictFPOpcode()) {
2920 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
2921 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
2922 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: NVT, RetVT: OVT), DL: dl,
2923 ResultTys: {MVT::i16, MVT::Other}, Ops: {Op.getValue(R: 1), Op});
2924 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
2925 return Op;
2926 }
2927
2928 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
2929
2930 // Round the value to the softened type.
2931 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2932}
2933
2934SDValue
2935DAGTypeLegalizer::SoftPromoteHalfRes_CONVERT_FROM_ARBITRARY_FP(SDNode *N) {
2936 EVT OVT = N->getValueType(ResNo: 0);
2937 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2938 SDLoc dl(N);
2939
2940 SDValue Res = DAG.getNode(Opcode: ISD::CONVERT_FROM_ARBITRARY_FP, DL: dl, VT: NVT,
2941 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
2942
2943 // Round the value to the softened type.
2944 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2945}
2946
2947SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2948 return DAG.getUNDEF(VT: MVT::i16);
2949}
2950
2951SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2952 EVT OVT = N->getValueType(ResNo: 0);
2953 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2954 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2955 SDLoc dl(N);
2956
2957 // Promote to the larger FP type.
2958 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
2959
2960 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
2961
2962 // Convert back to FP16 as an integer.
2963 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
2964}
2965
2966SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FABS(SDNode *N) {
2967 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2968 SDLoc dl(N);
2969
2970 // Clear the sign bit.
2971 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i16, N1: Op,
2972 N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: MVT::i16));
2973}
2974
2975SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FNEG(SDNode *N) {
2976 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2977 SDLoc dl(N);
2978
2979 // Invert the sign bit.
2980 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i16, N1: Op,
2981 N2: DAG.getConstant(Val: 0x8000, DL: dl, VT: MVT::i16));
2982}
2983
2984SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
2985 return GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2986}
2987
2988SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2989 EVT OVT = N->getValueType(ResNo: 0);
2990 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
2991 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
2992 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
2993 SDLoc dl(N);
2994
2995 // Promote to the larger FP type.
2996 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
2997 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
2998 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
2999
3000 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3001
3002 // Convert back to FP16 as an integer.
3003 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3004}
3005
3006SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3007 // Expand and soften recursively.
3008 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
3009 return SDValue();
3010}
3011
3012SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3013 // Expand and soften.
3014 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3015 return SDValue();
3016}
3017
3018//===----------------------------------------------------------------------===//
3019// Half Operand Soft Promotion
3020//===----------------------------------------------------------------------===//
3021
3022bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3023 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3024 N->dump(&DAG));
3025 SDValue Res = SDValue();
3026
3027 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
3028 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3029 return false;
3030 }
3031
3032 // Nodes that use a promotion-requiring floating point operand, but doesn't
3033 // produce a soft promotion-requiring floating point result, need to be
3034 // legalized to use the soft promoted float operand. Nodes that produce at
3035 // least one soft promotion-requiring floating point result have their
3036 // operands legalized as a part of PromoteFloatResult.
3037 switch (N->getOpcode()) {
3038 default:
3039 #ifndef NDEBUG
3040 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3041 N->dump(&DAG); dbgs() << "\n";
3042 #endif
3043 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3044 "operand!");
3045
3046 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3047 case ISD::BUILD_VECTOR:
3048 Res = SoftPromoteHalfOp_BUILD_VECTOR(N);
3049 break;
3050 case ISD::INSERT_VECTOR_ELT:
3051 Res = SoftPromoteHalfOp_INSERT_VECTOR_ELT(N, OpNo);
3052 break;
3053 case ISD::FAKE_USE:
3054 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
3055 break;
3056 case ISD::FCOPYSIGN:
3057 Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo);
3058 break;
3059 case ISD::FP_TO_SINT:
3060 case ISD::FP_TO_UINT:
3061 case ISD::STRICT_FP_TO_SINT:
3062 case ISD::STRICT_FP_TO_UINT:
3063 case ISD::LLRINT:
3064 case ISD::LLROUND:
3065 case ISD::LRINT:
3066 case ISD::LROUND:
3067 case ISD::STRICT_LLRINT:
3068 case ISD::STRICT_LLROUND:
3069 case ISD::STRICT_LRINT:
3070 case ISD::STRICT_LROUND:
3071 Res = SoftPromoteHalfOp_Op0WithStrict(N);
3072 break;
3073 case ISD::FP_TO_SINT_SAT:
3074 case ISD::FP_TO_UINT_SAT:
3075 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3076 case ISD::CONVERT_TO_ARBITRARY_FP:
3077 Res = SoftPromoteHalfOp_CONVERT_TO_ARBITRARY_FP(N);
3078 break;
3079 case ISD::STRICT_FP_EXTEND:
3080 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3081 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3082 case ISD::BR_CC:
3083 Res = SoftPromoteHalfOp_BR_CC(N);
3084 break;
3085 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3086 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3087 case ISD::ATOMIC_STORE:
3088 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3089 break;
3090 case ISD::STACKMAP:
3091 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3092 break;
3093 case ISD::PATCHPOINT:
3094 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3095 break;
3096 }
3097
3098 if (!Res.getNode())
3099 return false;
3100
3101 assert(Res.getNode() != N && "Expected a new node!");
3102
3103 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3104 "Invalid operand expansion");
3105
3106 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3107 return false;
3108}
3109
3110SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3111 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3112
3113 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3114}
3115
3116SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BUILD_VECTOR(SDNode *N) {
3117 SDLoc dl(N);
3118 EVT VT = N->getValueType(ResNo: 0);
3119
3120 SmallVector<SDValue, 8> Ops(N->getNumOperands());
3121 for (unsigned I = 0, E = N->getNumOperands(); I != E; ++I)
3122 Ops[I] = GetSoftPromotedHalf(Op: N->getOperand(Num: I));
3123
3124 EVT IVT = VT.changeVectorElementTypeToInteger();
3125 SDValue Res = DAG.getBuildVector(VT: IVT, DL: dl, Ops);
3126 return DAG.getBitcast(VT, V: Res);
3127}
3128
3129SDValue DAGTypeLegalizer::SoftPromoteHalfOp_INSERT_VECTOR_ELT(SDNode *N,
3130 unsigned OpNo) {
3131 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3132 SDValue Vec = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
3133 SDValue Elt = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3134 SDValue Res = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(N),
3135 VT: Vec.getValueType(), N1: Vec, N2: Elt, N3: N->getOperand(Num: 2));
3136 return DAG.getBitcast(VT: N->getValueType(ResNo: 0), V: Res);
3137}
3138
3139SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3140 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3141 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3142 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::Other, N1: N->getOperand(Num: 0),
3143 N2: Op);
3144}
3145
3146SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3147 unsigned OpNo) {
3148 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3149 SDValue Op1 = N->getOperand(Num: 1);
3150 EVT RVT = Op1.getValueType();
3151 SDLoc dl(N);
3152
3153 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3154
3155 Op1 = GetSoftPromotedHalf(Op: Op1);
3156 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3157
3158 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3159 N2: Op1);
3160}
3161
3162SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3163 EVT RVT = N->getValueType(ResNo: 0);
3164 bool IsStrict = N->isStrictFPOpcode();
3165 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3166 EVT SVT = Op.getValueType();
3167 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3168
3169 if (IsStrict) {
3170 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
3171 ResultTys: {RVT, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
3172 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3173 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3174 return SDValue();
3175 }
3176
3177 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3178}
3179
3180SDValue DAGTypeLegalizer::SoftPromoteHalfOp_Op0WithStrict(SDNode *N) {
3181 EVT RVT = N->getValueType(ResNo: 0);
3182 bool IsStrict = N->isStrictFPOpcode();
3183 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3184 EVT SVT = Op.getValueType();
3185 SDLoc dl(N);
3186
3187 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3188 Op = GetSoftPromotedHalf(Op);
3189
3190 if (IsStrict) {
3191 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: dl, ResultTys: {NVT, MVT::Other},
3192 Ops: {N->getOperand(Num: 0), Op});
3193 Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {RVT, MVT::Other},
3194 Ops: {Op.getValue(R: 1), Op});
3195 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3196 ReplaceValueWith(From: SDValue(N, 0), To: Op);
3197 return SDValue();
3198 }
3199
3200 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3201 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: RVT, Operand: Res);
3202}
3203
3204SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3205 EVT RVT = N->getValueType(ResNo: 0);
3206 SDValue Op = N->getOperand(Num: 0);
3207 EVT SVT = Op.getValueType();
3208 SDLoc dl(N);
3209
3210 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3211
3212 Op = GetSoftPromotedHalf(Op);
3213
3214 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3215
3216 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3217 N2: N->getOperand(Num: 1));
3218}
3219
3220// TODO: CONVERT_TO_ARBITRARY_FP also needs SoftenFloatOperand and
3221// ExpandFloatOperand handlers for targets with software float or ppcf128
3222// source types. Same gap exists for CONVERT_FROM_ARBITRARY_FP.
3223SDValue DAGTypeLegalizer::SoftPromoteHalfOp_CONVERT_TO_ARBITRARY_FP(SDNode *N) {
3224 EVT RVT = N->getValueType(ResNo: 0);
3225 SDValue Op = N->getOperand(Num: 0);
3226 EVT SVT = Op.getValueType();
3227 SDLoc dl(N);
3228
3229 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3230 Op = GetSoftPromotedHalf(Op);
3231 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3232
3233 return DAG.getNode(Opcode: ISD::CONVERT_TO_ARBITRARY_FP, DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3234 N2: N->getOperand(Num: 1), N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3));
3235}
3236
3237SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BR_CC(SDNode *N) {
3238 // ISD::BR_CC node: chain(0), condcode(1), LHS(2), RHS(3), dest(4)
3239 // The comparison operands (LHS, RHS) are soft-promoted halfs.
3240 SDValue Op0 = N->getOperand(Num: 2);
3241 SDValue Op1 = N->getOperand(Num: 3);
3242 SDLoc dl(N);
3243
3244 EVT SVT = Op0.getValueType();
3245 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3246
3247 // Get the soft-promoted i16 values
3248 Op0 = GetSoftPromotedHalf(Op: Op0);
3249 Op1 = GetSoftPromotedHalf(Op: Op1);
3250
3251 // Promote both comparison operands to the larger FP type.
3252 unsigned PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3253 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3254 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3255
3256 // Create a new BR_CC node with promoted operands
3257 return DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: N->getOperand(Num: 0),
3258 N2: N->getOperand(Num: 1), N3: Op0, N4: Op1, N5: N->getOperand(Num: 4));
3259}
3260
3261SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3262 unsigned OpNo) {
3263 assert(OpNo == 0 && "Can only soften the comparison values");
3264 SDValue Op0 = N->getOperand(Num: 0);
3265 SDValue Op1 = N->getOperand(Num: 1);
3266 SDLoc dl(N);
3267
3268 EVT SVT = Op0.getValueType();
3269 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3270
3271 Op0 = GetSoftPromotedHalf(Op: Op0);
3272 Op1 = GetSoftPromotedHalf(Op: Op1);
3273
3274 // Promote to the larger FP type.
3275 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3276 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3277 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3278
3279 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3280 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3281}
3282
3283SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3284 SDValue Op0 = N->getOperand(Num: 0);
3285 SDValue Op1 = N->getOperand(Num: 1);
3286 ISD::CondCode CCCode = N->getOperandCondCode(Num: 2);
3287 SDLoc dl(N);
3288
3289 EVT SVT = Op0.getValueType();
3290 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3291
3292 Op0 = GetSoftPromotedHalf(Op: Op0);
3293 Op1 = GetSoftPromotedHalf(Op: Op1);
3294
3295 // Promote to the larger FP type.
3296 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3297 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3298 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3299
3300 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3301}
3302
3303SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3304 assert(OpNo == 1 && "Can only soften the stored value!");
3305 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3306 SDValue Val = ST->getValue();
3307 SDLoc dl(N);
3308
3309 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3310 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3311 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3312 MMO: ST->getMemOperand());
3313}
3314
3315SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3316 unsigned OpNo) {
3317 assert(OpNo == 1 && "Can only soften the stored value!");
3318 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
3319 SDValue Val = ST->getVal();
3320 SDLoc dl(N);
3321
3322 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3323 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: Promoted.getValueType(),
3324 Chain: ST->getChain(), Ptr: Promoted, Val: ST->getBasePtr(),
3325 MMO: ST->getMemOperand());
3326}
3327
3328SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3329 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3330 SmallVector<SDValue> NewOps(N->ops());
3331 SDValue Op = N->getOperand(Num: OpNo);
3332 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3333 SDValue NewNode =
3334 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3335
3336 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3337 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3338
3339 return SDValue(); // Signal that we replaced the node ourselves.
3340}
3341
3342SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3343 unsigned OpNo) {
3344 assert(OpNo >= 7);
3345 SmallVector<SDValue> NewOps(N->ops());
3346 SDValue Op = N->getOperand(Num: OpNo);
3347 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3348 SDValue NewNode =
3349 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3350
3351 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3352 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3353
3354 return SDValue(); // Signal that we replaced the node ourselves.
3355}
3356