1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error(reason: "Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 case ISD::EXTRACT_VECTOR_ELT:
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::STRICT_FMINNUM:
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::STRICT_FMAXNUM:
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break;
78 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break;
79 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break;
80 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break;
81 case ISD::STRICT_FADD:
82 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
83 case ISD::STRICT_FACOS:
84 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break;
85 case ISD::STRICT_FASIN:
86 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break;
87 case ISD::STRICT_FATAN:
88 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break;
89 case ISD::STRICT_FATAN2:
90 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break;
91 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
92 case ISD::STRICT_FCEIL:
93 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
94 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
95 case ISD::STRICT_FCOS:
96 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
97 case ISD::STRICT_FCOSH:
98 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break;
99 case ISD::STRICT_FDIV:
100 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
101 case ISD::STRICT_FEXP:
102 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
103 case ISD::STRICT_FEXP2:
104 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
105 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
106 case ISD::STRICT_FFLOOR:
107 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
108 case ISD::STRICT_FLOG:
109 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
110 case ISD::STRICT_FLOG2:
111 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
112 case ISD::STRICT_FLOG10:
113 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
114 case ISD::STRICT_FMA:
115 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
116 case ISD::STRICT_FMUL:
117 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
118 case ISD::STRICT_FNEARBYINT:
119 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
120 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
121 case ISD::STRICT_FP_EXTEND:
122 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
123 case ISD::STRICT_FP_ROUND:
124 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
125 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
126 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
127 case ISD::STRICT_FPOW:
128 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
129 case ISD::STRICT_FPOWI:
130 case ISD::FPOWI:
131 case ISD::FLDEXP:
132 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
133 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
134 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break;
135 case ISD::FMODF: R = SoftenFloatRes_FMODF(N); break;
136 case ISD::STRICT_FREM:
137 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
138 case ISD::STRICT_FRINT:
139 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
140 case ISD::STRICT_FROUND:
141 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
142 case ISD::STRICT_FROUNDEVEN:
143 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
144 case ISD::STRICT_FSIN:
145 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
146 case ISD::STRICT_FSINH:
147 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break;
148 case ISD::STRICT_FSQRT:
149 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
150 case ISD::STRICT_FSUB:
151 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
152 case ISD::STRICT_FTAN:
153 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
154 case ISD::STRICT_FTANH:
155 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break;
156 case ISD::STRICT_FTRUNC:
157 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
158 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
159 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
160 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
161 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
162 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
163 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
164 case ISD::STRICT_SINT_TO_FP:
165 case ISD::STRICT_UINT_TO_FP:
166 case ISD::SINT_TO_FP:
167 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
168 case ISD::POISON:
169 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
170 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
171 case ISD::AssertNoFPClass: R = GetSoftenedFloat(Op: N->getOperand(Num: 0)); break;
172 case ISD::VECREDUCE_FADD:
173 case ISD::VECREDUCE_FMUL:
174 case ISD::VECREDUCE_FMIN:
175 case ISD::VECREDUCE_FMAX:
176 case ISD::VECREDUCE_FMAXIMUM:
177 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
178 case ISD::VECREDUCE_SEQ_FADD:
179 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
180 // clang-format on
181 }
182
183 // If R is null, the sub-method took care of registering the result.
184 if (R.getNode()) {
185 assert(R.getNode() != N);
186 SetSoftenedFloat(Op: SDValue(N, ResNo), Result: R);
187 }
188}
189
190SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
191 bool IsStrict = N->isStrictFPOpcode();
192 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
193 unsigned Offset = IsStrict ? 1 : 0;
194 assert(N->getNumOperands() == (1 + Offset) &&
195 "Unexpected number of operands!");
196 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
197 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
198 TargetLowering::MakeLibCallOptions CallOptions;
199 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
200 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
202 CallOptions, dl: SDLoc(N),
203 Chain);
204 if (IsStrict)
205 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
206 return Tmp.first;
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
210 bool IsStrict = N->isStrictFPOpcode();
211 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
212 unsigned Offset = IsStrict ? 1 : 0;
213 assert(N->getNumOperands() == (2 + Offset) &&
214 "Unexpected number of operands!");
215 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
216 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)) };
217 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
218 TargetLowering::MakeLibCallOptions CallOptions;
219 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
220 N->getOperand(Num: 1 + Offset).getValueType() };
221 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
222 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
223 CallOptions, dl: SDLoc(N),
224 Chain);
225 if (IsStrict)
226 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
227 return Tmp.first;
228}
229
230SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
231 return BitConvertToInteger(Op: N->getOperand(Num: 0));
232}
233
234SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
235 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
236 return DAG.getNode(Opcode: ISD::FREEZE, DL: SDLoc(N), VT: Ty,
237 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
238}
239
240SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
241 EVT Ty = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
242 SDValue NewFence = DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: Ty,
243 Operand: GetSoftenedFloat(Op: N->getOperand(Num: 0)));
244 return NewFence;
245}
246
247SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
248 unsigned ResNo) {
249 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
250 return BitConvertToInteger(Op);
251}
252
253SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
254 // Convert the inputs to integers, and build a new pair out of them.
255 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N),
256 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
257 VT: N->getValueType(ResNo: 0)),
258 N1: BitConvertToInteger(Op: N->getOperand(Num: 0)),
259 N2: BitConvertToInteger(Op: N->getOperand(Num: 1)));
260}
261
262SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
263 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
264 // In ppcf128, the high 64 bits are always first in memory regardless
265 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
266 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
267 // way. However, APInt's are serialized in an Endian-sensitive fashion,
268 // so on big-Endian targets, the two doubles are output in the wrong
269 // order. Fix this by manually flipping the order of the high 64 bits
270 // and the low 64 bits here.
271 if (DAG.getDataLayout().isBigEndian() &&
272 CN->getValueType(ResNo: 0).getSimpleVT() == llvm::MVT::ppcf128) {
273 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
274 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
275 APInt Val(128, words);
276 return DAG.getConstant(Val, DL: SDLoc(CN),
277 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
278 VT: CN->getValueType(ResNo: 0)));
279 } else {
280 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
281 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
282 VT: CN->getValueType(ResNo: 0)));
283 }
284}
285
286SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
287 SDValue Src = N->getOperand(Num: 0);
288 assert(Src.getValueType() == MVT::ppcf128 &&
289 "In floats only ppcf128 can be extracted by element!");
290 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(N),
291 VT: N->getValueType(ResNo: 0).changeTypeToInteger(),
292 N1: DAG.getBitcast(VT: MVT::i128, V: Src), N2: N->getOperand(Num: 1));
293}
294
295SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
296 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
297 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
298 VT: NewOp.getValueType().getVectorElementType(),
299 N1: NewOp, N2: N->getOperand(Num: 1));
300}
301
302SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
303 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
304 unsigned Size = NVT.getSizeInBits();
305
306 // Mask = ~(1 << (Size-1))
307 APInt API = APInt::getAllOnes(numBits: Size);
308 API.clearBit(BitPosition: Size - 1);
309 SDValue Mask = DAG.getConstant(Val: API, DL: SDLoc(N), VT: NVT);
310 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0));
311 return DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: NVT, N1: Op, N2: Mask);
312}
313
314SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
315 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
316 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
317 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
318 Call_F32: RTLIB::FMIN_F32,
319 Call_F64: RTLIB::FMIN_F64,
320 Call_F80: RTLIB::FMIN_F80,
321 Call_F128: RTLIB::FMIN_F128,
322 Call_PPCF128: RTLIB::FMIN_PPCF128));
323}
324
325SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
326 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(Node: N, DAG))
327 return SoftenFloatRes_SELECT_CC(N: SelCC.getNode());
328 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
329 Call_F32: RTLIB::FMAX_F32,
330 Call_F64: RTLIB::FMAX_F64,
331 Call_F80: RTLIB::FMAX_F80,
332 Call_F128: RTLIB::FMAX_F128,
333 Call_PPCF128: RTLIB::FMAX_PPCF128));
334}
335
336SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) {
337 return SoftenFloatRes_Binary(
338 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_NUM_F32,
339 Call_F64: RTLIB::FMINIMUM_NUM_F64, Call_F80: RTLIB::FMINIMUM_NUM_F80,
340 Call_F128: RTLIB::FMINIMUM_NUM_F128, Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128));
341}
342
343SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) {
344 return SoftenFloatRes_Binary(
345 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_NUM_F32,
346 Call_F64: RTLIB::FMAXIMUM_NUM_F64, Call_F80: RTLIB::FMAXIMUM_NUM_F80,
347 Call_F128: RTLIB::FMAXIMUM_NUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128));
348}
349
350SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) {
351 return SoftenFloatRes_Binary(
352 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_F32,
353 Call_F64: RTLIB::FMINIMUM_F64, Call_F80: RTLIB::FMINIMUM_F80,
354 Call_F128: RTLIB::FMINIMUM_F128, Call_PPCF128: RTLIB::FMINIMUM_PPCF128));
355}
356
357SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) {
358 return SoftenFloatRes_Binary(
359 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_F32,
360 Call_F64: RTLIB::FMAXIMUM_F64, Call_F80: RTLIB::FMAXIMUM_F80,
361 Call_F128: RTLIB::FMAXIMUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_PPCF128));
362}
363
364SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
365 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
366 Call_F32: RTLIB::ADD_F32,
367 Call_F64: RTLIB::ADD_F64,
368 Call_F80: RTLIB::ADD_F80,
369 Call_F128: RTLIB::ADD_F128,
370 Call_PPCF128: RTLIB::ADD_PPCF128));
371}
372
373SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) {
374 return SoftenFloatRes_Unary(
375 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64,
376 Call_F80: RTLIB::ACOS_F80, Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128));
377}
378
379SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) {
380 return SoftenFloatRes_Unary(
381 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64,
382 Call_F80: RTLIB::ASIN_F80, Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128));
383}
384
385SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) {
386 return SoftenFloatRes_Unary(
387 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64,
388 Call_F80: RTLIB::ATAN_F80, Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128));
389}
390
391SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) {
392 return SoftenFloatRes_Binary(
393 N,
394 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN2_F32, Call_F64: RTLIB::ATAN2_F64,
395 Call_F80: RTLIB::ATAN2_F80, Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128));
396}
397
398SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
399 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
400 Call_F32: RTLIB::CBRT_F32,
401 Call_F64: RTLIB::CBRT_F64,
402 Call_F80: RTLIB::CBRT_F80,
403 Call_F128: RTLIB::CBRT_F128,
404 Call_PPCF128: RTLIB::CBRT_PPCF128));
405}
406
407SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
408 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
409 Call_F32: RTLIB::CEIL_F32,
410 Call_F64: RTLIB::CEIL_F64,
411 Call_F80: RTLIB::CEIL_F80,
412 Call_F128: RTLIB::CEIL_F128,
413 Call_PPCF128: RTLIB::CEIL_PPCF128));
414}
415
416SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
417 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 0));
418 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
419 SDLoc dl(N);
420
421 EVT LVT = LHS.getValueType();
422 EVT RVT = RHS.getValueType();
423
424 unsigned LSize = LVT.getSizeInBits();
425 unsigned RSize = RVT.getSizeInBits();
426
427 // First get the sign bit of second operand.
428 SDValue SignBit = DAG.getNode(
429 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
430 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
431 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
432 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
433
434 // Shift right or sign-extend it if the two operands have different types.
435 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
436 if (SizeDiff > 0) {
437 SignBit =
438 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
439 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
440 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
441 DL: DAG.getDataLayout())));
442 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
443 } else if (SizeDiff < 0) {
444 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
445 SignBit =
446 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
447 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
448 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
449 DL: DAG.getDataLayout())));
450 }
451
452 // Clear the sign bit of the first operand.
453 SDValue Mask = DAG.getNode(
454 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
455 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
456 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
457 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
458 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
459
460 // Or the value with the sign bit.
461 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
462}
463
464SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
465 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
466 Call_F32: RTLIB::COS_F32,
467 Call_F64: RTLIB::COS_F64,
468 Call_F80: RTLIB::COS_F80,
469 Call_F128: RTLIB::COS_F128,
470 Call_PPCF128: RTLIB::COS_PPCF128));
471}
472
473SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) {
474 return SoftenFloatRes_Unary(
475 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64,
476 Call_F80: RTLIB::COSH_F80, Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128));
477}
478
479SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
480 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
481 Call_F32: RTLIB::DIV_F32,
482 Call_F64: RTLIB::DIV_F64,
483 Call_F80: RTLIB::DIV_F80,
484 Call_F128: RTLIB::DIV_F128,
485 Call_PPCF128: RTLIB::DIV_PPCF128));
486}
487
488SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
489 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
490 Call_F32: RTLIB::EXP_F32,
491 Call_F64: RTLIB::EXP_F64,
492 Call_F80: RTLIB::EXP_F80,
493 Call_F128: RTLIB::EXP_F128,
494 Call_PPCF128: RTLIB::EXP_PPCF128));
495}
496
497SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
498 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
499 Call_F32: RTLIB::EXP2_F32,
500 Call_F64: RTLIB::EXP2_F64,
501 Call_F80: RTLIB::EXP2_F80,
502 Call_F128: RTLIB::EXP2_F128,
503 Call_PPCF128: RTLIB::EXP2_PPCF128));
504}
505
506SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
507 return SoftenFloatRes_Unary(
508 N,
509 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64,
510 Call_F80: RTLIB::EXP10_F80, Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128));
511}
512
513SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
514 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
515 Call_F32: RTLIB::FLOOR_F32,
516 Call_F64: RTLIB::FLOOR_F64,
517 Call_F80: RTLIB::FLOOR_F80,
518 Call_F128: RTLIB::FLOOR_F128,
519 Call_PPCF128: RTLIB::FLOOR_PPCF128));
520}
521
522SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
523 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
524 Call_F32: RTLIB::LOG_F32,
525 Call_F64: RTLIB::LOG_F64,
526 Call_F80: RTLIB::LOG_F80,
527 Call_F128: RTLIB::LOG_F128,
528 Call_PPCF128: RTLIB::LOG_PPCF128));
529}
530
531SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
532 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
533 Call_F32: RTLIB::LOG2_F32,
534 Call_F64: RTLIB::LOG2_F64,
535 Call_F80: RTLIB::LOG2_F80,
536 Call_F128: RTLIB::LOG2_F128,
537 Call_PPCF128: RTLIB::LOG2_PPCF128));
538}
539
540SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
541 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
542 Call_F32: RTLIB::LOG10_F32,
543 Call_F64: RTLIB::LOG10_F64,
544 Call_F80: RTLIB::LOG10_F80,
545 Call_F128: RTLIB::LOG10_F128,
546 Call_PPCF128: RTLIB::LOG10_PPCF128));
547}
548
549SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
550 bool IsStrict = N->isStrictFPOpcode();
551 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
552 unsigned Offset = IsStrict ? 1 : 0;
553 SDValue Ops[3] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
554 GetSoftenedFloat(Op: N->getOperand(Num: 1 + Offset)),
555 GetSoftenedFloat(Op: N->getOperand(Num: 2 + Offset)) };
556 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
557 TargetLowering::MakeLibCallOptions CallOptions;
558 EVT OpsVT[3] = { N->getOperand(Num: 0 + Offset).getValueType(),
559 N->getOperand(Num: 1 + Offset).getValueType(),
560 N->getOperand(Num: 2 + Offset).getValueType() };
561 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
562 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
563 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
564 Call_F32: RTLIB::FMA_F32,
565 Call_F64: RTLIB::FMA_F64,
566 Call_F80: RTLIB::FMA_F80,
567 Call_F128: RTLIB::FMA_F128,
568 Call_PPCF128: RTLIB::FMA_PPCF128),
569 RetVT: NVT, Ops, CallOptions, dl: SDLoc(N), Chain);
570 if (IsStrict)
571 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
572 return Tmp.first;
573}
574
575SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
576 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
577 Call_F32: RTLIB::MUL_F32,
578 Call_F64: RTLIB::MUL_F64,
579 Call_F80: RTLIB::MUL_F80,
580 Call_F128: RTLIB::MUL_F128,
581 Call_PPCF128: RTLIB::MUL_PPCF128));
582}
583
584SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
585 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
586 Call_F32: RTLIB::NEARBYINT_F32,
587 Call_F64: RTLIB::NEARBYINT_F64,
588 Call_F80: RTLIB::NEARBYINT_F80,
589 Call_F128: RTLIB::NEARBYINT_F128,
590 Call_PPCF128: RTLIB::NEARBYINT_PPCF128));
591}
592
593SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
594 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
595 SDLoc dl(N);
596
597 // Expand Y = FNEG(X) -> Y = X ^ sign mask
598 APInt SignMask = APInt::getSignMask(BitWidth: NVT.getSizeInBits());
599 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: NVT, N1: GetSoftenedFloat(Op: N->getOperand(Num: 0)),
600 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: NVT));
601}
602
603SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
604 bool IsStrict = N->isStrictFPOpcode();
605 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
606 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
607
608 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
609
610 if (getTypeAction(VT: Op.getValueType()) == TargetLowering::TypePromoteFloat) {
611 Op = GetPromotedFloat(Op);
612 // If the promotion did the FP_EXTEND to the destination type for us,
613 // there's nothing left to do here.
614 if (Op.getValueType() == N->getValueType(ResNo: 0)) {
615 if (IsStrict)
616 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
617 return BitConvertToInteger(Op);
618 }
619 }
620
621 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
622 // -> f32, so proceed in two stages. Also, it's entirely possible for both
623 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
624 // than FP16_TO_FP.
625 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
626 N->getValueType(ResNo: 0) != MVT::f32) {
627 if (IsStrict) {
628 Op = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N),
629 ResultTys: { MVT::f32, MVT::Other }, Ops: { Chain, Op });
630 Chain = Op.getValue(R: 1);
631 } else {
632 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT: MVT::f32, Operand: Op);
633 }
634 }
635
636 if (Op.getValueType() == MVT::bf16) {
637 // FIXME: Need ReplaceValueWith on chain in strict case
638 return SoftenFloatRes_BF16_TO_FP(N);
639 }
640
641 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
642 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
643 TargetLowering::MakeLibCallOptions CallOptions;
644 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
645 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
646 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
647 CallOptions, dl: SDLoc(N),
648 Chain);
649 if (IsStrict)
650 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
651 return Tmp.first;
652}
653
654// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
655// nodes?
656SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
657 EVT MidVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
658 SDValue Op = N->getOperand(Num: 0);
659 TargetLowering::MakeLibCallOptions CallOptions;
660 EVT OpsVT[1] = { N->getOperand(Num: 0).getValueType() };
661 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
662 SDValue Res32 = TLI.makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MidVT, Ops: Op,
663 CallOptions, dl: SDLoc(N)).first;
664 if (N->getValueType(ResNo: 0) == MVT::f32)
665 return Res32;
666
667 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
668 RTLIB::Libcall LC = RTLIB::getFPEXT(OpVT: MVT::f32, RetVT: N->getValueType(ResNo: 0));
669 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
670 return TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Res32, CallOptions, dl: SDLoc(N)).first;
671}
672
673// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
674// nodes?
675SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
676 assert(N->getValueType(0) == MVT::f32 &&
677 "Can only soften BF16_TO_FP with f32 result");
678 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: MVT::f32);
679 SDValue Op = N->getOperand(Num: 0);
680 SDLoc DL(N);
681 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: NVT,
682 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op));
683 SDValue Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: NVT, N1: Op,
684 N2: DAG.getShiftAmountConstant(Val: 16, VT: NVT, DL));
685 return Res;
686}
687
688SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
689 bool IsStrict = N->isStrictFPOpcode();
690 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
691 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
692 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
693 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: N->getValueType(ResNo: 0));
694 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
695 TargetLowering::MakeLibCallOptions CallOptions;
696 EVT OpVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
697 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
698 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
699 CallOptions, dl: SDLoc(N),
700 Chain);
701 if (IsStrict)
702 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
703 return Tmp.first;
704}
705
706SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
707 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
708 Call_F32: RTLIB::POW_F32,
709 Call_F64: RTLIB::POW_F64,
710 Call_F80: RTLIB::POW_F80,
711 Call_F128: RTLIB::POW_F128,
712 Call_PPCF128: RTLIB::POW_PPCF128));
713}
714
715SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
716 bool IsStrict = N->isStrictFPOpcode();
717 unsigned Offset = IsStrict ? 1 : 0;
718 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
719 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
720 "Unsupported power type!");
721 bool IsPowI =
722 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
723
724 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0))
725 : RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0));
726 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
727 if (!TLI.getLibcallName(Call: LC)) {
728 // Some targets don't have a powi libcall; use pow instead.
729 // FIXME: Implement this if some target needs it.
730 DAG.getContext()->emitError(ErrorStr: "Don't know how to soften fpowi to fpow");
731 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
732 }
733
734 if (DAG.getLibInfo().getIntSize() !=
735 N->getOperand(Num: 1 + Offset).getValueType().getSizeInBits()) {
736 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
737 // would use the wrong type for the argument.
738 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
739 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
740 }
741
742 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
743 SDValue Ops[2] = { GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset)),
744 N->getOperand(Num: 1 + Offset) };
745 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
746 TargetLowering::MakeLibCallOptions CallOptions;
747 EVT OpsVT[2] = { N->getOperand(Num: 0 + Offset).getValueType(),
748 N->getOperand(Num: 1 + Offset).getValueType() };
749 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT: N->getValueType(ResNo: 0), Value: true);
750 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops,
751 CallOptions, dl: SDLoc(N),
752 Chain);
753 if (IsStrict)
754 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
755 return Tmp.first;
756}
757
758SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
759 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
760 EVT VT0 = N->getValueType(ResNo: 0);
761 EVT VT1 = N->getValueType(ResNo: 1);
762 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: VT0);
763
764 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
765 // If the exponent does not match with sizeof(int) a libcall would use the
766 // wrong type for the argument.
767 // TODO: Should be able to handle mismatches.
768 DAG.getContext()->emitError(ErrorStr: "ffrexp exponent does not match sizeof(int)");
769 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
770 }
771
772 EVT NVT0 = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT0);
773 SDValue StackSlot = DAG.CreateStackTemporary(VT: VT1);
774
775 SDLoc DL(N);
776
777 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
778 TargetLowering::MakeLibCallOptions CallOptions;
779 SDValue Ops[2] = {GetSoftenedFloat(Op: N->getOperand(Num: 0)), StackSlot};
780 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
781 Type *CallOpsTypeOverrides[2] = {nullptr, PointerTy};
782
783 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
784 // but we only really need to handle the 0th one for softening anyway.
785 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT0, Value: true)
786 .setOpsTypeOverrides(CallOpsTypeOverrides);
787
788 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, RetVT: NVT0, Ops, CallOptions, dl: DL,
789 /*Chain=*/SDValue());
790 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
791 auto PtrInfo =
792 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
793
794 SDValue LoadExp = DAG.getLoad(VT: VT1, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
795
796 ReplaceValueWith(From: SDValue(N, 1), To: LoadExp);
797 return ReturnVal;
798}
799
800SDValue DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
801 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
802 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
803 EVT VT = N->getValueType(ResNo: 0);
804
805 assert(VT == N->getValueType(1) &&
806 "expected both return values to have the same type");
807
808 if (!TLI.getLibcallName(Call: LC))
809 return SDValue();
810
811 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
812
813 SDLoc DL(N);
814
815 SmallVector<SDValue, 3> Ops = {GetSoftenedFloat(Op: N->getOperand(Num: 0))};
816 SmallVector<EVT, 3> OpsVT = {VT};
817
818 std::array<SDValue, 2> StackSlots;
819 SmallVector<Type *, 3> CallOpsTypeOverrides = {nullptr};
820 auto PointerTy = PointerType::getUnqual(C&: *DAG.getContext());
821 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ++ResNum) {
822 if (ResNum == CallRetResNo)
823 continue;
824 SDValue StackSlot = DAG.CreateStackTemporary(VT: NVT);
825 Ops.push_back(Elt: StackSlot);
826 OpsVT.push_back(Elt: StackSlot.getValueType());
827 StackSlots[ResNum] = StackSlot;
828 CallOpsTypeOverrides.push_back(Elt: PointerTy);
829 }
830
831 TargetLowering::MakeLibCallOptions CallOptions;
832 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
833 // but since both returns have the same type it should be okay.
834 CallOptions.setTypeListBeforeSoften(OpsVT: {OpsVT}, RetVT: VT, Value: true)
835 .setOpsTypeOverrides(CallOpsTypeOverrides);
836
837 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops, CallOptions, dl: DL,
838 /*Chain=*/SDValue());
839
840 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) {
841 int FrameIdx = cast<FrameIndexSDNode>(Val&: StackSlot)->getIndex();
842 auto PtrInfo =
843 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
844 return DAG.getLoad(VT: NVT, dl: DL, Chain, Ptr: StackSlot, PtrInfo);
845 };
846
847 for (auto [ResNum, SlackSlot] : enumerate(First&: StackSlots)) {
848 if (CallRetResNo == ResNum) {
849 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: ReturnVal);
850 continue;
851 }
852 SetSoftenedFloat(Op: SDValue(N, ResNum), Result: CreateStackLoad(SlackSlot));
853 }
854
855 return SDValue();
856}
857
858SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) {
859 return SoftenFloatRes_UnaryWithTwoFPResults(
860 N, LC: RTLIB::getSINCOS(RetVT: N->getValueType(ResNo: 0)));
861}
862
863SDValue DAGTypeLegalizer::SoftenFloatRes_FMODF(SDNode *N) {
864 return SoftenFloatRes_UnaryWithTwoFPResults(
865 N, LC: RTLIB::getMODF(RetVT: N->getValueType(ResNo: 0)), /*CallRetResNo=*/0);
866}
867
868SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
869 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
870 Call_F32: RTLIB::REM_F32,
871 Call_F64: RTLIB::REM_F64,
872 Call_F80: RTLIB::REM_F80,
873 Call_F128: RTLIB::REM_F128,
874 Call_PPCF128: RTLIB::REM_PPCF128));
875}
876
877SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
878 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
879 Call_F32: RTLIB::RINT_F32,
880 Call_F64: RTLIB::RINT_F64,
881 Call_F80: RTLIB::RINT_F80,
882 Call_F128: RTLIB::RINT_F128,
883 Call_PPCF128: RTLIB::RINT_PPCF128));
884}
885
886SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
887 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
888 Call_F32: RTLIB::ROUND_F32,
889 Call_F64: RTLIB::ROUND_F64,
890 Call_F80: RTLIB::ROUND_F80,
891 Call_F128: RTLIB::ROUND_F128,
892 Call_PPCF128: RTLIB::ROUND_PPCF128));
893}
894
895SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
896 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
897 Call_F32: RTLIB::ROUNDEVEN_F32,
898 Call_F64: RTLIB::ROUNDEVEN_F64,
899 Call_F80: RTLIB::ROUNDEVEN_F80,
900 Call_F128: RTLIB::ROUNDEVEN_F128,
901 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128));
902}
903
904SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
905 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
906 Call_F32: RTLIB::SIN_F32,
907 Call_F64: RTLIB::SIN_F64,
908 Call_F80: RTLIB::SIN_F80,
909 Call_F128: RTLIB::SIN_F128,
910 Call_PPCF128: RTLIB::SIN_PPCF128));
911}
912
913SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) {
914 return SoftenFloatRes_Unary(
915 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64,
916 Call_F80: RTLIB::SINH_F80, Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128));
917}
918
919SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
920 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
921 Call_F32: RTLIB::SQRT_F32,
922 Call_F64: RTLIB::SQRT_F64,
923 Call_F80: RTLIB::SQRT_F80,
924 Call_F128: RTLIB::SQRT_F128,
925 Call_PPCF128: RTLIB::SQRT_PPCF128));
926}
927
928SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
929 return SoftenFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
930 Call_F32: RTLIB::SUB_F32,
931 Call_F64: RTLIB::SUB_F64,
932 Call_F80: RTLIB::SUB_F80,
933 Call_F128: RTLIB::SUB_F128,
934 Call_PPCF128: RTLIB::SUB_PPCF128));
935}
936
937SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
938 return SoftenFloatRes_Unary(
939 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64,
940 Call_F80: RTLIB::TAN_F80, Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128));
941}
942
943SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) {
944 return SoftenFloatRes_Unary(
945 N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64,
946 Call_F80: RTLIB::TANH_F80, Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128));
947}
948
949SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
950 return SoftenFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
951 Call_F32: RTLIB::TRUNC_F32,
952 Call_F64: RTLIB::TRUNC_F64,
953 Call_F80: RTLIB::TRUNC_F80,
954 Call_F128: RTLIB::TRUNC_F128,
955 Call_PPCF128: RTLIB::TRUNC_PPCF128));
956}
957
958SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
959 LoadSDNode *L = cast<LoadSDNode>(Val: N);
960 EVT VT = N->getValueType(ResNo: 0);
961 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
962 SDLoc dl(N);
963
964 auto MMOFlags =
965 L->getMemOperand()->getFlags() &
966 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
967 SDValue NewL;
968 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
969 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: NVT, dl,
970 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
971 PtrInfo: L->getPointerInfo(), MemVT: NVT, Alignment: L->getBaseAlign(), MMOFlags,
972 AAInfo: L->getAAInfo());
973 // Legalized the chain result - switch anything that used the old chain to
974 // use the new one.
975 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
976 return NewL;
977 }
978
979 // Do a non-extending load followed by FP_EXTEND.
980 NewL = DAG.getLoad(AM: L->getAddressingMode(), ExtType: ISD::NON_EXTLOAD, VT: L->getMemoryVT(),
981 dl, Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
982 PtrInfo: L->getPointerInfo(), MemVT: L->getMemoryVT(), Alignment: L->getBaseAlign(),
983 MMOFlags, AAInfo: L->getAAInfo());
984 // Legalized the chain result - switch anything that used the old chain to
985 // use the new one.
986 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
987 auto ExtendNode = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT, Operand: NewL);
988 return BitConvertToInteger(Op: ExtendNode);
989}
990
991SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
992 AtomicSDNode *L = cast<AtomicSDNode>(Val: N);
993 EVT VT = N->getValueType(ResNo: 0);
994 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
995 SDLoc dl(N);
996
997 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
998 SDValue NewL =
999 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
1000 Ops: {L->getChain(), L->getBasePtr()}, MMO: L->getMemOperand());
1001
1002 // Legalized the chain result - switch anything that used the old chain to
1003 // use the new one.
1004 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
1005 return NewL;
1006 }
1007
1008 report_fatal_error(reason: "softening fp extending atomic load not handled");
1009}
1010
1011SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
1012 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 1));
1013 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
1014 return DAG.getSelect(DL: SDLoc(N),
1015 VT: LHS.getValueType(), Cond: N->getOperand(Num: 0), LHS, RHS);
1016}
1017
1018SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
1019 SDValue LHS = GetSoftenedFloat(Op: N->getOperand(Num: 2));
1020 SDValue RHS = GetSoftenedFloat(Op: N->getOperand(Num: 3));
1021 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
1022 VT: LHS.getValueType(), N1: N->getOperand(Num: 0),
1023 N2: N->getOperand(Num: 1), N3: LHS, N4: RHS, N5: N->getOperand(Num: 4));
1024}
1025
1026SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
1027 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
1028 VT: N->getValueType(ResNo: 0)));
1029}
1030
1031SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
1032 SDValue Chain = N->getOperand(Num: 0); // Get the chain.
1033 SDValue Ptr = N->getOperand(Num: 1); // Get the pointer.
1034 EVT VT = N->getValueType(ResNo: 0);
1035 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
1036 SDLoc dl(N);
1037
1038 SDValue NewVAARG;
1039 NewVAARG = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: N->getOperand(Num: 2),
1040 Align: N->getConstantOperandVal(Num: 3));
1041
1042 // Legalized the chain result - switch anything that used the old chain to
1043 // use the new one.
1044 if (N != NewVAARG.getValue(R: 1).getNode())
1045 ReplaceValueWith(From: SDValue(N, 1), To: NewVAARG.getValue(R: 1));
1046 return NewVAARG;
1047}
1048
1049SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
1050 bool IsStrict = N->isStrictFPOpcode();
1051 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
1052 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1053 EVT SVT = N->getOperand(Num: IsStrict ? 1 : 0).getValueType();
1054 EVT RVT = N->getValueType(ResNo: 0);
1055 EVT NVT = EVT();
1056 SDLoc dl(N);
1057
1058 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
1059 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
1060 // match. Look for an appropriate libcall.
1061 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1062 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
1063 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
1064 NVT = (MVT::SimpleValueType)t;
1065 // The source needs to big enough to hold the operand.
1066 if (NVT.bitsGE(VT: SVT))
1067 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT):RTLIB::getUINTTOFP (OpVT: NVT, RetVT: RVT);
1068 }
1069 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1070
1071 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1072 // Sign/zero extend the argument if the libcall takes a larger type.
1073 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
1074 VT: NVT, Operand: N->getOperand(Num: IsStrict ? 1 : 0));
1075 TargetLowering::MakeLibCallOptions CallOptions;
1076 CallOptions.setIsSigned(Signed);
1077 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
1078 std::pair<SDValue, SDValue> Tmp =
1079 TLI.makeLibCall(DAG, LC, RetVT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: RVT),
1080 Ops: Op, CallOptions, dl, Chain);
1081
1082 if (IsStrict)
1083 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1084 return Tmp.first;
1085}
1086
1087SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
1088 // Expand and soften recursively.
1089 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
1090 return SDValue();
1091}
1092
1093SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
1094 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
1095 return SDValue();
1096}
1097
1098//===----------------------------------------------------------------------===//
1099// Convert Float Operand to Integer
1100//===----------------------------------------------------------------------===//
1101
1102bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
1103 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
1104 SDValue Res = SDValue();
1105
1106 switch (N->getOpcode()) {
1107 default:
1108#ifndef NDEBUG
1109 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
1110 N->dump(&DAG); dbgs() << "\n";
1111#endif
1112 report_fatal_error(reason: "Do not know how to soften this operator's operand!");
1113
1114 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
1115 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
1116 case ISD::STRICT_FP_TO_FP16:
1117 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
1118 case ISD::FP_TO_BF16:
1119 case ISD::STRICT_FP_TO_BF16:
1120 case ISD::STRICT_FP_ROUND:
1121 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
1122 case ISD::STRICT_FP_TO_SINT:
1123 case ISD::STRICT_FP_TO_UINT:
1124 case ISD::FP_TO_SINT:
1125 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
1126 case ISD::FP_TO_SINT_SAT:
1127 case ISD::FP_TO_UINT_SAT:
1128 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
1129 case ISD::STRICT_LROUND:
1130 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
1131 case ISD::STRICT_LLROUND:
1132 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
1133 case ISD::STRICT_LRINT:
1134 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
1135 case ISD::STRICT_LLRINT:
1136 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
1137 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
1138 case ISD::STRICT_FSETCC:
1139 case ISD::STRICT_FSETCCS:
1140 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
1141 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
1142 case ISD::ATOMIC_STORE:
1143 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
1144 break;
1145 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
1146 case ISD::FAKE_USE:
1147 Res = SoftenFloatOp_FAKE_USE(N);
1148 break;
1149 }
1150
1151 // If the result is null, the sub-method took care of registering results etc.
1152 if (!Res.getNode()) return false;
1153
1154 // If the result is N, the sub-method updated N in place. Tell the legalizer
1155 // core about this to re-analyze.
1156 if (Res.getNode() == N)
1157 return true;
1158
1159 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1160 "Invalid operand softening");
1161
1162 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1163 return false;
1164}
1165
1166SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1167 SDValue Op0 = GetSoftenedFloat(Op: N->getOperand(Num: 0));
1168
1169 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
1170}
1171
1172SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1173 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1174 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1175 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1176 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1177 N->getOpcode() == ISD::FP_TO_BF16 ||
1178 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1179 N->getOpcode() == ISD::STRICT_FP_ROUND);
1180
1181 bool IsStrict = N->isStrictFPOpcode();
1182 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1183 EVT SVT = Op.getValueType();
1184 EVT RVT = N->getValueType(ResNo: 0);
1185 EVT FloatRVT = RVT;
1186 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1187 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1188 FloatRVT = MVT::f16;
1189 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1190 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1191 FloatRVT = MVT::bf16;
1192
1193 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: FloatRVT);
1194 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1195
1196 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1197 Op = GetSoftenedFloat(Op);
1198 TargetLowering::MakeLibCallOptions CallOptions;
1199 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
1200 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op,
1201 CallOptions, dl: SDLoc(N),
1202 Chain);
1203 if (IsStrict) {
1204 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1205 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1206 return SDValue();
1207 }
1208 return Tmp.first;
1209}
1210
1211SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1212 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
1213 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
1214
1215 EVT VT = NewLHS.getValueType();
1216 NewLHS = GetSoftenedFloat(Op: NewLHS);
1217 NewRHS = GetSoftenedFloat(Op: NewRHS);
1218 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1219 OldLHS: N->getOperand(Num: 2), OldRHS: N->getOperand(Num: 3));
1220
1221 // If softenSetCCOperands returned a scalar, we need to compare the result
1222 // against zero to select between true and false values.
1223 if (!NewRHS.getNode()) {
1224 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1225 CCCode = ISD::SETNE;
1226 }
1227
1228 // Update N to have the operands specified.
1229 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
1230 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
1231 Op5: N->getOperand(Num: 4)),
1232 0);
1233}
1234
1235// Even if the result type is legal, no libcall may exactly match. (e.g. We
1236// don't have FP-i8 conversions) This helper method looks for an appropriate
1237// promoted libcall.
1238static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1239 bool Signed) {
1240 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1241 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1242 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1243 ++IntVT) {
1244 Promoted = (MVT::SimpleValueType)IntVT;
1245 // The type needs to big enough to hold the result.
1246 if (Promoted.bitsGE(VT: RetVT))
1247 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SrcVT, RetVT: Promoted)
1248 : RTLIB::getFPTOUINT(OpVT: SrcVT, RetVT: Promoted);
1249 }
1250 return LC;
1251}
1252
1253SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1254 bool IsStrict = N->isStrictFPOpcode();
1255 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1256 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1257
1258 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
1259 EVT SVT = Op.getValueType();
1260 EVT RVT = N->getValueType(ResNo: 0);
1261 EVT NVT = EVT();
1262 SDLoc dl(N);
1263
1264 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1265 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1266 // match, eg. we don't have fp -> i8 conversions.
1267 // Look for an appropriate libcall.
1268 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: SVT, RetVT: RVT, Promoted&: NVT, Signed);
1269 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1270 "Unsupported FP_TO_XINT!");
1271
1272 Op = GetSoftenedFloat(Op);
1273 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1274 TargetLowering::MakeLibCallOptions CallOptions;
1275 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
1276 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1277 CallOptions, dl, Chain);
1278
1279 // Truncate the result if the libcall returns a larger type.
1280 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first);
1281
1282 if (!IsStrict)
1283 return Res;
1284
1285 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1286 ReplaceValueWith(From: SDValue(N, 0), To: Res);
1287 return SDValue();
1288}
1289
1290SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1291 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1292 return Res;
1293}
1294
1295SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1296 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
1297 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
1298
1299 EVT VT = NewLHS.getValueType();
1300 NewLHS = GetSoftenedFloat(Op: NewLHS);
1301 NewRHS = GetSoftenedFloat(Op: NewRHS);
1302 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N),
1303 OldLHS: N->getOperand(Num: 0), OldRHS: N->getOperand(Num: 1));
1304
1305 // If softenSetCCOperands returned a scalar, we need to compare the result
1306 // against zero to select between true and false values.
1307 if (!NewRHS.getNode()) {
1308 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
1309 CCCode = ISD::SETNE;
1310 }
1311
1312 // Update N to have the operands specified.
1313 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1314 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
1315 Op5: DAG.getCondCode(Cond: CCCode)),
1316 0);
1317}
1318
1319SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1320 bool IsStrict = N->isStrictFPOpcode();
1321 SDValue Op0 = N->getOperand(Num: IsStrict ? 1 : 0);
1322 SDValue Op1 = N->getOperand(Num: IsStrict ? 2 : 1);
1323 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1324 ISD::CondCode CCCode =
1325 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
1326
1327 EVT VT = Op0.getValueType();
1328 SDValue NewLHS = GetSoftenedFloat(Op: Op0);
1329 SDValue NewRHS = GetSoftenedFloat(Op: Op1);
1330 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: SDLoc(N), OldLHS: Op0, OldRHS: Op1,
1331 Chain, IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
1332
1333 // Update N to have the operands specified.
1334 if (NewRHS.getNode()) {
1335 if (IsStrict)
1336 NewLHS = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: NewLHS,
1337 N2: NewRHS, N3: DAG.getCondCode(Cond: CCCode));
1338 else
1339 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
1340 Op3: DAG.getCondCode(Cond: CCCode)), 0);
1341 }
1342
1343 // Otherwise, softenSetCCOperands returned a scalar, use it.
1344 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1345 "Unexpected setcc expansion!");
1346
1347 if (IsStrict) {
1348 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
1349 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1350 return SDValue();
1351 }
1352 return NewLHS;
1353}
1354
1355SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1356 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1357 assert(OpNo == 1 && "Can only soften the stored value!");
1358 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
1359 SDValue Val = ST->getValue();
1360 SDLoc dl(N);
1361
1362 if (ST->isTruncatingStore())
1363 // Do an FP_ROUND followed by a non-truncating store.
1364 Val = BitConvertToInteger(
1365 Op: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: ST->getMemoryVT(), N1: Val,
1366 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
1367 else
1368 Val = GetSoftenedFloat(Op: Val);
1369
1370 return DAG.getStore(Chain: ST->getChain(), dl, Val, Ptr: ST->getBasePtr(),
1371 MMO: ST->getMemOperand());
1372}
1373
1374SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1375 assert(OpNo == 1 && "Can only soften the stored value!");
1376 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
1377 SDValue Val = ST->getVal();
1378 EVT VT = Val.getValueType();
1379 SDLoc dl(N);
1380
1381 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1382
1383 SDValue NewVal = GetSoftenedFloat(Op: Val);
1384 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: VT, Chain: ST->getChain(), Ptr: NewVal,
1385 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
1386}
1387
1388SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1389 SDValue LHS = N->getOperand(Num: 0);
1390 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
1391 SDLoc dl(N);
1392
1393 EVT LVT = LHS.getValueType();
1394 EVT ILVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LVT.getSizeInBits());
1395 EVT RVT = RHS.getValueType();
1396
1397 unsigned LSize = LVT.getSizeInBits();
1398 unsigned RSize = RVT.getSizeInBits();
1399
1400 // Shift right or sign-extend it if the two operands have different types.
1401 int SizeDiff = RSize - LSize;
1402 if (SizeDiff > 0) {
1403 RHS =
1404 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: RHS,
1405 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
1406 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1407 DL: DAG.getDataLayout())));
1408 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ILVT, Operand: RHS);
1409 } else if (SizeDiff < 0) {
1410 RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: RHS);
1411 RHS =
1412 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: ILVT, N1: RHS,
1413 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
1414 VT: TLI.getShiftAmountTy(LHSTy: RHS.getValueType(),
1415 DL: DAG.getDataLayout())));
1416 }
1417
1418 RHS = DAG.getBitcast(VT: LVT, V: RHS);
1419 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT: LVT, N1: LHS, N2: RHS);
1420}
1421
1422SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1423 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1424 bool IsStrict = N->isStrictFPOpcode();
1425 unsigned Offset = IsStrict ? 1 : 0;
1426 SDValue Op = GetSoftenedFloat(Op: N->getOperand(Num: 0 + Offset));
1427 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1428 TargetLowering::MakeLibCallOptions CallOptions;
1429 EVT OpVT = N->getOperand(Num: 0 + Offset).getValueType();
1430 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0), Value: true);
1431 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op,
1432 CallOptions, dl: SDLoc(N),
1433 Chain);
1434 if (IsStrict) {
1435 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1436 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
1437 return SDValue();
1438 }
1439
1440 return Tmp.first;
1441}
1442
1443SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1444 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1445 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1446 Call_F32: RTLIB::LROUND_F32,
1447 Call_F64: RTLIB::LROUND_F64,
1448 Call_F80: RTLIB::LROUND_F80,
1449 Call_F128: RTLIB::LROUND_F128,
1450 Call_PPCF128: RTLIB::LROUND_PPCF128));
1451}
1452
1453SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1454 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1455 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1456 Call_F32: RTLIB::LLROUND_F32,
1457 Call_F64: RTLIB::LLROUND_F64,
1458 Call_F80: RTLIB::LLROUND_F80,
1459 Call_F128: RTLIB::LLROUND_F128,
1460 Call_PPCF128: RTLIB::LLROUND_PPCF128));
1461}
1462
1463SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1464 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1465 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1466 Call_F32: RTLIB::LRINT_F32,
1467 Call_F64: RTLIB::LRINT_F64,
1468 Call_F80: RTLIB::LRINT_F80,
1469 Call_F128: RTLIB::LRINT_F128,
1470 Call_PPCF128: RTLIB::LRINT_PPCF128));
1471}
1472
1473SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1474 EVT OpVT = N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType();
1475 return SoftenFloatOp_Unary(N, LC: GetFPLibCall(VT: OpVT,
1476 Call_F32: RTLIB::LLRINT_F32,
1477 Call_F64: RTLIB::LLRINT_F64,
1478 Call_F80: RTLIB::LLRINT_F80,
1479 Call_F128: RTLIB::LLRINT_F128,
1480 Call_PPCF128: RTLIB::LLRINT_PPCF128));
1481}
1482
1483SDValue DAGTypeLegalizer::SoftenFloatOp_FAKE_USE(SDNode *N) {
1484 SDValue Op1 = BitConvertToInteger(Op: N->getOperand(Num: 1));
1485 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
1486 N1: N->getOperand(Num: 0), N2: Op1);
1487}
1488
1489//===----------------------------------------------------------------------===//
1490// Float Result Expansion
1491//===----------------------------------------------------------------------===//
1492
1493/// ExpandFloatResult - This method is called when the specified result of the
1494/// specified node is found to need expansion. At this point, the node may also
1495/// have invalid operands or may have other results that need promotion, we just
1496/// know that (at least) one result needs expansion.
1497void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1498 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1499 SDValue Lo, Hi;
1500 Lo = Hi = SDValue();
1501
1502 // See if the target wants to custom expand this node.
1503 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true))
1504 return;
1505
1506 switch (N->getOpcode()) {
1507 default:
1508#ifndef NDEBUG
1509 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1510 N->dump(&DAG); dbgs() << "\n";
1511#endif
1512 report_fatal_error(reason: "Do not know how to expand the result of this "
1513 "operator!");
1514 // clang-format off
1515 case ISD::POISON:
1516 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1517 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1518 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1519
1520 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1521 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1522 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1523 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1524 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1525 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1526
1527 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1528 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1529 case ISD::STRICT_FMINNUM:
1530 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1531 case ISD::STRICT_FMAXNUM:
1532 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1533 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break;
1534 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break;
1535 case ISD::STRICT_FADD:
1536 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1537 case ISD::STRICT_FACOS:
1538 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break;
1539 case ISD::STRICT_FASIN:
1540 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break;
1541 case ISD::STRICT_FATAN:
1542 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break;
1543 case ISD::STRICT_FATAN2:
1544 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break;
1545 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1546 case ISD::STRICT_FCEIL:
1547 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1548 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1549 case ISD::STRICT_FCOS:
1550 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1551 case ISD::STRICT_FCOSH:
1552 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break;
1553 case ISD::STRICT_FDIV:
1554 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1555 case ISD::STRICT_FEXP:
1556 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1557 case ISD::STRICT_FEXP2:
1558 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1559 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1560 case ISD::STRICT_FFLOOR:
1561 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1562 case ISD::STRICT_FLOG:
1563 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1564 case ISD::STRICT_FLOG2:
1565 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1566 case ISD::STRICT_FLOG10:
1567 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1568 case ISD::STRICT_FMA:
1569 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1570 case ISD::STRICT_FMUL:
1571 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1572 case ISD::STRICT_FNEARBYINT:
1573 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1574 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1575 case ISD::STRICT_FP_EXTEND:
1576 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1577 case ISD::STRICT_FPOW:
1578 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1579 case ISD::STRICT_FPOWI:
1580 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1581 case ISD::FLDEXP:
1582 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1583 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1584 case ISD::STRICT_FRINT:
1585 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1586 case ISD::STRICT_FROUND:
1587 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1588 case ISD::STRICT_FROUNDEVEN:
1589 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1590 case ISD::STRICT_FSIN:
1591 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1592 case ISD::STRICT_FSINH:
1593 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break;
1594 case ISD::STRICT_FSQRT:
1595 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1596 case ISD::STRICT_FSUB:
1597 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1598 case ISD::STRICT_FTAN:
1599 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1600 case ISD::STRICT_FTANH:
1601 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break;
1602 case ISD::STRICT_FTRUNC:
1603 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1604 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1605 case ISD::STRICT_SINT_TO_FP:
1606 case ISD::STRICT_UINT_TO_FP:
1607 case ISD::SINT_TO_FP:
1608 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1609 case ISD::STRICT_FREM:
1610 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1611 case ISD::FMODF: ExpandFloatRes_FMODF(N); break;
1612 case ISD::FSINCOS: ExpandFloatRes_FSINCOS(N); break;
1613 case ISD::FSINCOSPI: ExpandFloatRes_FSINCOSPI(N); break;
1614 // clang-format on
1615 }
1616
1617 // If Lo/Hi is null, the sub-method took care of registering results etc.
1618 if (Lo.getNode())
1619 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1620}
1621
1622void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1623 SDValue &Hi) {
1624 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1625 assert(NVT.getSizeInBits() == 64 &&
1626 "Do not know how to expand this float constant!");
1627 APInt C = cast<ConstantFPSDNode>(Val: N)->getValueAPF().bitcastToAPInt();
1628 SDLoc dl(N);
1629 const fltSemantics &Sem = NVT.getFltSemantics();
1630 Lo = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 64)), DL: dl, VT: NVT);
1631 Hi = DAG.getConstantFP(Val: APFloat(Sem, C.extractBits(numBits: 64, bitPosition: 0)), DL: dl, VT: NVT);
1632}
1633
1634void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1635 SDValue &Lo, SDValue &Hi) {
1636 bool IsStrict = N->isStrictFPOpcode();
1637 unsigned Offset = IsStrict ? 1 : 0;
1638 SDValue Op = N->getOperand(Num: 0 + Offset);
1639 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1640 TargetLowering::MakeLibCallOptions CallOptions;
1641 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1642 Ops: Op, CallOptions, dl: SDLoc(N),
1643 Chain);
1644 if (IsStrict)
1645 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1646 GetPairElements(Pair: Tmp.first, Lo, Hi);
1647}
1648
1649void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1650 SDValue &Lo, SDValue &Hi) {
1651 bool IsStrict = N->isStrictFPOpcode();
1652 unsigned Offset = IsStrict ? 1 : 0;
1653 SDValue Ops[] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset) };
1654 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1655 TargetLowering::MakeLibCallOptions CallOptions;
1656 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0),
1657 Ops, CallOptions, dl: SDLoc(N),
1658 Chain);
1659 if (IsStrict)
1660 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1661 GetPairElements(Pair: Tmp.first, Lo, Hi);
1662}
1663
1664void DAGTypeLegalizer::ExpandFloatRes_FMODF(SDNode *N) {
1665 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getMODF(RetVT: N->getValueType(ResNo: 0)),
1666 /*CallRetResNo=*/0);
1667}
1668
1669void DAGTypeLegalizer::ExpandFloatRes_FSINCOS(SDNode *N) {
1670 ExpandFloatRes_UnaryWithTwoFPResults(N, LC: RTLIB::getSINCOS(RetVT: N->getValueType(ResNo: 0)));
1671}
1672
1673void DAGTypeLegalizer::ExpandFloatRes_FSINCOSPI(SDNode *N) {
1674 ExpandFloatRes_UnaryWithTwoFPResults(N,
1675 LC: RTLIB::getSINCOSPI(RetVT: N->getValueType(ResNo: 0)));
1676}
1677
1678void DAGTypeLegalizer::ExpandFloatRes_UnaryWithTwoFPResults(
1679 SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo) {
1680 assert(!N->isStrictFPOpcode() && "strictfp not implemented");
1681 SmallVector<SDValue> Results;
1682 DAG.expandMultipleResultFPLibCall(LC, Node: N, Results, CallRetResNo);
1683 for (auto [ResNo, Res] : enumerate(First&: Results)) {
1684 SDValue Lo, Hi;
1685 GetPairElements(Pair: Res, Lo, Hi);
1686 SetExpandedFloat(Op: SDValue(N, ResNo), Lo, Hi);
1687 }
1688}
1689
1690void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1691 SDValue &Hi) {
1692 assert(N->getValueType(0) == MVT::ppcf128 &&
1693 "Logic only correct for ppcf128!");
1694 SDLoc dl(N);
1695 SDValue Tmp;
1696 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi&: Tmp);
1697 Hi = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: Tmp.getValueType(), Operand: Tmp);
1698 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1699 Lo = DAG.getSelectCC(DL: dl, LHS: Tmp, RHS: Hi, True: Lo,
1700 False: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo),
1701 Cond: ISD::SETEQ);
1702}
1703
1704void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1705 SDValue &Hi) {
1706 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1707 Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
1708 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
1709 Call_PPCF128: RTLIB::FMIN_PPCF128), Lo, Hi);
1710}
1711
1712void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1713 SDValue &Hi) {
1714 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1715 Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
1716 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
1717 Call_PPCF128: RTLIB::FMAX_PPCF128), Lo, Hi);
1718}
1719
1720void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo,
1721 SDValue &Hi) {
1722 ExpandFloatRes_Binary(
1723 N,
1724 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMINIMUM_NUM_F32,
1725 Call_F64: RTLIB::FMINIMUM_NUM_F64, Call_F80: RTLIB::FMINIMUM_NUM_F80,
1726 Call_F128: RTLIB::FMINIMUM_NUM_F128, Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128),
1727 Lo, Hi);
1728}
1729
1730void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo,
1731 SDValue &Hi) {
1732 ExpandFloatRes_Binary(
1733 N,
1734 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::FMAXIMUM_NUM_F32,
1735 Call_F64: RTLIB::FMAXIMUM_NUM_F64, Call_F80: RTLIB::FMAXIMUM_NUM_F80,
1736 Call_F128: RTLIB::FMAXIMUM_NUM_F128, Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128),
1737 Lo, Hi);
1738}
1739
1740void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1741 SDValue &Hi) {
1742 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1743 Call_F32: RTLIB::ADD_F32, Call_F64: RTLIB::ADD_F64,
1744 Call_F80: RTLIB::ADD_F80, Call_F128: RTLIB::ADD_F128,
1745 Call_PPCF128: RTLIB::ADD_PPCF128), Lo, Hi);
1746}
1747
1748void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo,
1749 SDValue &Hi) {
1750 ExpandFloatRes_Unary(N,
1751 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ACOS_F32,
1752 Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
1753 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128),
1754 Lo, Hi);
1755}
1756
1757void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo,
1758 SDValue &Hi) {
1759 ExpandFloatRes_Unary(N,
1760 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ASIN_F32,
1761 Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
1762 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128),
1763 Lo, Hi);
1764}
1765
1766void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo,
1767 SDValue &Hi) {
1768 ExpandFloatRes_Unary(N,
1769 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN_F32,
1770 Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
1771 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128),
1772 Lo, Hi);
1773}
1774
1775void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo,
1776 SDValue &Hi) {
1777 ExpandFloatRes_Binary(N,
1778 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::ATAN2_F32,
1779 Call_F64: RTLIB::ATAN2_F64, Call_F80: RTLIB::ATAN2_F80,
1780 Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128),
1781 Lo, Hi);
1782}
1783
1784void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1785 SDValue &Hi) {
1786 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::CBRT_F32,
1787 Call_F64: RTLIB::CBRT_F64, Call_F80: RTLIB::CBRT_F80,
1788 Call_F128: RTLIB::CBRT_F128,
1789 Call_PPCF128: RTLIB::CBRT_PPCF128), Lo, Hi);
1790}
1791
1792void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1793 SDValue &Lo, SDValue &Hi) {
1794 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1795 Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
1796 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
1797 Call_PPCF128: RTLIB::CEIL_PPCF128), Lo, Hi);
1798}
1799
1800void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1801 SDValue &Lo, SDValue &Hi) {
1802 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1803 Call_F32: RTLIB::COPYSIGN_F32,
1804 Call_F64: RTLIB::COPYSIGN_F64,
1805 Call_F80: RTLIB::COPYSIGN_F80,
1806 Call_F128: RTLIB::COPYSIGN_F128,
1807 Call_PPCF128: RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1808}
1809
1810void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1811 SDValue &Lo, SDValue &Hi) {
1812 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1813 Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
1814 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
1815 Call_PPCF128: RTLIB::COS_PPCF128), Lo, Hi);
1816}
1817
1818void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo,
1819 SDValue &Hi) {
1820 ExpandFloatRes_Unary(N,
1821 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::COSH_F32,
1822 Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
1823 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128),
1824 Lo, Hi);
1825}
1826
1827void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1828 SDValue &Hi) {
1829 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1830 Call_F32: RTLIB::DIV_F32,
1831 Call_F64: RTLIB::DIV_F64,
1832 Call_F80: RTLIB::DIV_F80,
1833 Call_F128: RTLIB::DIV_F128,
1834 Call_PPCF128: RTLIB::DIV_PPCF128), Lo, Hi);
1835}
1836
1837void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1838 SDValue &Lo, SDValue &Hi) {
1839 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1840 Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64,
1841 Call_F80: RTLIB::EXP_F80, Call_F128: RTLIB::EXP_F128,
1842 Call_PPCF128: RTLIB::EXP_PPCF128), Lo, Hi);
1843}
1844
1845void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1846 SDValue &Lo, SDValue &Hi) {
1847 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1848 Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64,
1849 Call_F80: RTLIB::EXP2_F80, Call_F128: RTLIB::EXP2_F128,
1850 Call_PPCF128: RTLIB::EXP2_PPCF128), Lo, Hi);
1851}
1852
1853void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1854 SDValue &Hi) {
1855 ExpandFloatRes_Unary(N,
1856 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::EXP10_F32,
1857 Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
1858 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128),
1859 Lo, Hi);
1860}
1861
1862void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1863 SDValue &Lo, SDValue &Hi) {
1864 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1865 Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
1866 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
1867 Call_PPCF128: RTLIB::FLOOR_PPCF128), Lo, Hi);
1868}
1869
1870void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1871 SDValue &Lo, SDValue &Hi) {
1872 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1873 Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64,
1874 Call_F80: RTLIB::LOG_F80, Call_F128: RTLIB::LOG_F128,
1875 Call_PPCF128: RTLIB::LOG_PPCF128), Lo, Hi);
1876}
1877
1878void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1879 SDValue &Lo, SDValue &Hi) {
1880 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1881 Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64,
1882 Call_F80: RTLIB::LOG2_F80, Call_F128: RTLIB::LOG2_F128,
1883 Call_PPCF128: RTLIB::LOG2_PPCF128), Lo, Hi);
1884}
1885
1886void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1887 SDValue &Lo, SDValue &Hi) {
1888 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1889 Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64,
1890 Call_F80: RTLIB::LOG10_F80, Call_F128: RTLIB::LOG10_F128,
1891 Call_PPCF128: RTLIB::LOG10_PPCF128), Lo, Hi);
1892}
1893
1894void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1895 SDValue &Hi) {
1896 bool IsStrict = N->isStrictFPOpcode();
1897 unsigned Offset = IsStrict ? 1 : 0;
1898 SDValue Ops[3] = { N->getOperand(Num: 0 + Offset), N->getOperand(Num: 1 + Offset),
1899 N->getOperand(Num: 2 + Offset) };
1900 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
1901 TargetLowering::MakeLibCallOptions CallOptions;
1902 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1903 Call_F32: RTLIB::FMA_F32,
1904 Call_F64: RTLIB::FMA_F64,
1905 Call_F80: RTLIB::FMA_F80,
1906 Call_F128: RTLIB::FMA_F128,
1907 Call_PPCF128: RTLIB::FMA_PPCF128),
1908 RetVT: N->getValueType(ResNo: 0), Ops, CallOptions,
1909 dl: SDLoc(N), Chain);
1910 if (IsStrict)
1911 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
1912 GetPairElements(Pair: Tmp.first, Lo, Hi);
1913}
1914
1915void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1916 SDValue &Hi) {
1917 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1918 Call_F32: RTLIB::MUL_F32,
1919 Call_F64: RTLIB::MUL_F64,
1920 Call_F80: RTLIB::MUL_F80,
1921 Call_F128: RTLIB::MUL_F128,
1922 Call_PPCF128: RTLIB::MUL_PPCF128), Lo, Hi);
1923}
1924
1925void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1926 SDValue &Lo, SDValue &Hi) {
1927 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1928 Call_F32: RTLIB::NEARBYINT_F32,
1929 Call_F64: RTLIB::NEARBYINT_F64,
1930 Call_F80: RTLIB::NEARBYINT_F80,
1931 Call_F128: RTLIB::NEARBYINT_F128,
1932 Call_PPCF128: RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1933}
1934
1935void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1936 SDValue &Hi) {
1937 SDLoc dl(N);
1938 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1939 Lo = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1940 Hi = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1941}
1942
1943void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1944 SDValue &Hi) {
1945 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
1946 SDLoc dl(N);
1947 bool IsStrict = N->isStrictFPOpcode();
1948
1949 SDValue Chain;
1950 if (IsStrict) {
1951 // If the expanded type is the same as the input type, just bypass the node.
1952 if (NVT == N->getOperand(Num: 1).getValueType()) {
1953 Hi = N->getOperand(Num: 1);
1954 Chain = N->getOperand(Num: 0);
1955 } else {
1956 // Other we need to extend.
1957 Hi = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: { NVT, MVT::Other },
1958 Ops: { N->getOperand(Num: 0), N->getOperand(Num: 1) });
1959 Chain = Hi.getValue(R: 1);
1960 }
1961 } else {
1962 Hi = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
1963 }
1964
1965 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
1966
1967 if (IsStrict)
1968 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
1969}
1970
1971void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1972 SDValue &Lo, SDValue &Hi) {
1973 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
1974 Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64,
1975 Call_F80: RTLIB::POW_F80, Call_F128: RTLIB::POW_F128,
1976 Call_PPCF128: RTLIB::POW_PPCF128), Lo, Hi);
1977}
1978
1979void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1980 SDValue &Lo, SDValue &Hi) {
1981 ExpandFloatRes_Binary(N, LC: RTLIB::getPOWI(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
1982}
1983
1984void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1985 SDValue &Hi) {
1986 ExpandFloatRes_Binary(N, LC: RTLIB::getLDEXP(RetVT: N->getValueType(ResNo: 0)), Lo, Hi);
1987}
1988
1989void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1990 SDValue &Lo, SDValue &Hi) {
1991 assert(N->getValueType(0) == MVT::ppcf128 &&
1992 "Logic only correct for ppcf128!");
1993
1994 SDLoc dl(N);
1995 GetExpandedFloat(Op: N->getOperand(Num: 0), Lo, Hi);
1996 Lo = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Lo.getValueType(), Operand: Lo);
1997 Hi = DAG.getNode(Opcode: ISD::FREEZE, DL: dl, VT: Hi.getValueType(), Operand: Hi);
1998}
1999
2000void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
2001 SDValue &Lo, SDValue &Hi) {
2002 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2003 Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
2004 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
2005 Call_PPCF128: RTLIB::REM_PPCF128), Lo, Hi);
2006}
2007
2008void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
2009 SDValue &Lo, SDValue &Hi) {
2010 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2011 Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
2012 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
2013 Call_PPCF128: RTLIB::RINT_PPCF128), Lo, Hi);
2014}
2015
2016void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
2017 SDValue &Lo, SDValue &Hi) {
2018 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2019 Call_F32: RTLIB::ROUND_F32,
2020 Call_F64: RTLIB::ROUND_F64,
2021 Call_F80: RTLIB::ROUND_F80,
2022 Call_F128: RTLIB::ROUND_F128,
2023 Call_PPCF128: RTLIB::ROUND_PPCF128), Lo, Hi);
2024}
2025
2026void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
2027 SDValue &Lo, SDValue &Hi) {
2028 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2029 Call_F32: RTLIB::ROUNDEVEN_F32,
2030 Call_F64: RTLIB::ROUNDEVEN_F64,
2031 Call_F80: RTLIB::ROUNDEVEN_F80,
2032 Call_F128: RTLIB::ROUNDEVEN_F128,
2033 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
2034}
2035
2036void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
2037 SDValue &Lo, SDValue &Hi) {
2038 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2039 Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
2040 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
2041 Call_PPCF128: RTLIB::SIN_PPCF128), Lo, Hi);
2042}
2043
2044void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo,
2045 SDValue &Hi) {
2046 ExpandFloatRes_Unary(N,
2047 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::SINH_F32,
2048 Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
2049 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128),
2050 Lo, Hi);
2051}
2052
2053void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
2054 SDValue &Lo, SDValue &Hi) {
2055 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2056 Call_F32: RTLIB::SQRT_F32, Call_F64: RTLIB::SQRT_F64,
2057 Call_F80: RTLIB::SQRT_F80, Call_F128: RTLIB::SQRT_F128,
2058 Call_PPCF128: RTLIB::SQRT_PPCF128), Lo, Hi);
2059}
2060
2061void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
2062 SDValue &Hi) {
2063 ExpandFloatRes_Binary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2064 Call_F32: RTLIB::SUB_F32,
2065 Call_F64: RTLIB::SUB_F64,
2066 Call_F80: RTLIB::SUB_F80,
2067 Call_F128: RTLIB::SUB_F128,
2068 Call_PPCF128: RTLIB::SUB_PPCF128), Lo, Hi);
2069}
2070
2071void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
2072 SDValue &Hi) {
2073 ExpandFloatRes_Unary(N,
2074 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TAN_F32,
2075 Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
2076 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128),
2077 Lo, Hi);
2078}
2079
2080void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo,
2081 SDValue &Hi) {
2082 ExpandFloatRes_Unary(N,
2083 LC: GetFPLibCall(VT: N->getValueType(ResNo: 0), Call_F32: RTLIB::TANH_F32,
2084 Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
2085 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128),
2086 Lo, Hi);
2087}
2088
2089void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
2090 SDValue &Lo, SDValue &Hi) {
2091 ExpandFloatRes_Unary(N, LC: GetFPLibCall(VT: N->getValueType(ResNo: 0),
2092 Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
2093 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
2094 Call_PPCF128: RTLIB::TRUNC_PPCF128), Lo, Hi);
2095}
2096
2097void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
2098 SDValue &Hi) {
2099 if (ISD::isNormalLoad(N)) {
2100 ExpandRes_NormalLoad(N, Lo, Hi);
2101 return;
2102 }
2103
2104 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2105 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
2106 SDValue Chain = LD->getChain();
2107 SDValue Ptr = LD->getBasePtr();
2108 SDLoc dl(N);
2109
2110 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: LD->getValueType(ResNo: 0));
2111 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2112 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2113
2114 Hi = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT: NVT, Chain, Ptr,
2115 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
2116
2117 // Remember the chain.
2118 Chain = Hi.getValue(R: 1);
2119
2120 // The low part is zero.
2121 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2122
2123 // Modified the chain - switch anything that used the old chain to use the
2124 // new one.
2125 ReplaceValueWith(From: SDValue(LD, 1), To: Chain);
2126}
2127
2128void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
2129 SDValue &Hi) {
2130 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
2131 EVT VT = N->getValueType(ResNo: 0);
2132 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2133 bool Strict = N->isStrictFPOpcode();
2134 SDValue Src = N->getOperand(Num: Strict ? 1 : 0);
2135 EVT SrcVT = Src.getValueType();
2136 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2137 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2138 SDLoc dl(N);
2139 SDValue Chain = Strict ? N->getOperand(Num: 0) : DAG.getEntryNode();
2140
2141 // TODO: Any other flags to propagate?
2142 SDNodeFlags Flags;
2143 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
2144
2145 // First do an SINT_TO_FP, whether the original was signed or unsigned.
2146 // When promoting partial word types to i32 we must honor the signedness,
2147 // though.
2148 if (SrcVT.bitsLE(VT: MVT::i32)) {
2149 // The integer can be represented exactly in an f64.
2150 Lo = DAG.getConstantFP(Val: APFloat::getZero(Sem: NVT.getFltSemantics()), DL: dl, VT: NVT);
2151 if (Strict) {
2152 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
2153 Ops: {Chain, Src}, Flags);
2154 Chain = Hi.getValue(R: 1);
2155 } else
2156 Hi = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Src);
2157 } else {
2158 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2159 if (SrcVT.bitsLE(VT: MVT::i64)) {
2160 Src = DAG.getNode(Opcode: isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
2161 VT: MVT::i64, Operand: Src);
2162 LC = RTLIB::SINTTOFP_I64_PPCF128;
2163 } else if (SrcVT.bitsLE(VT: MVT::i128)) {
2164 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i128, Operand: Src);
2165 LC = RTLIB::SINTTOFP_I128_PPCF128;
2166 }
2167 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
2168
2169 TargetLowering::MakeLibCallOptions CallOptions;
2170 CallOptions.setIsSigned(true);
2171 std::pair<SDValue, SDValue> Tmp =
2172 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Src, CallOptions, dl, Chain);
2173 if (Strict)
2174 Chain = Tmp.second;
2175 GetPairElements(Pair: Tmp.first, Lo, Hi);
2176 }
2177
2178 // No need to complement for unsigned 32-bit integers
2179 if (isSigned || SrcVT.bitsLE(VT: MVT::i32)) {
2180 if (Strict)
2181 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2182
2183 return;
2184 }
2185
2186 // Unsigned - fix up the SINT_TO_FP value just calculated.
2187 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
2188 // keep semantics correctness if the integer is not exactly representable
2189 // here. See ExpandLegalINT_TO_FP.
2190 Hi = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: Lo, N2: Hi);
2191 SrcVT = Src.getValueType();
2192
2193 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
2194 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
2195 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
2196 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
2197 ArrayRef<uint64_t> Parts;
2198
2199 switch (SrcVT.getSimpleVT().SimpleTy) {
2200 default:
2201 llvm_unreachable("Unsupported UINT_TO_FP!");
2202 case MVT::i32:
2203 Parts = TwoE32;
2204 break;
2205 case MVT::i64:
2206 Parts = TwoE64;
2207 break;
2208 case MVT::i128:
2209 Parts = TwoE128;
2210 break;
2211 }
2212
2213 // TODO: Are there other fast-math-flags to propagate to this FADD?
2214 SDValue NewLo = DAG.getConstantFP(
2215 Val: APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), DL: dl, VT: MVT::ppcf128);
2216 if (Strict) {
2217 Lo = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other),
2218 Ops: {Chain, Hi, NewLo}, Flags);
2219 Chain = Lo.getValue(R: 1);
2220 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2221 } else
2222 Lo = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Hi, N2: NewLo);
2223 Lo = DAG.getSelectCC(DL: dl, LHS: Src, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT),
2224 True: Lo, False: Hi, Cond: ISD::SETLT);
2225 GetPairElements(Pair: Lo, Lo, Hi);
2226}
2227
2228
2229//===----------------------------------------------------------------------===//
2230// Float Operand Expansion
2231//===----------------------------------------------------------------------===//
2232
2233/// ExpandFloatOperand - This method is called when the specified operand of the
2234/// specified node is found to need expansion. At this point, all of the result
2235/// types of the node are known to be legal, but other operands of the node may
2236/// need promotion or expansion as well as the specified one.
2237bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
2238 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
2239 SDValue Res = SDValue();
2240
2241 // See if the target wants to custom expand this node.
2242 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false))
2243 return false;
2244
2245 switch (N->getOpcode()) {
2246 default:
2247#ifndef NDEBUG
2248 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
2249 N->dump(&DAG); dbgs() << "\n";
2250#endif
2251 report_fatal_error(reason: "Do not know how to expand this operator's operand!");
2252
2253 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2254 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2255 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2256
2257 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
2258 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
2259 case ISD::STRICT_FP_ROUND:
2260 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
2261 case ISD::STRICT_FP_TO_SINT:
2262 case ISD::STRICT_FP_TO_UINT:
2263 case ISD::FP_TO_SINT:
2264 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
2265 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
2266 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
2267 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
2268 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
2269 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
2270 case ISD::STRICT_FSETCC:
2271 case ISD::STRICT_FSETCCS:
2272 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
2273 case ISD::STORE: Res = ExpandFloatOp_STORE(N: cast<StoreSDNode>(Val: N),
2274 OpNo); break;
2275 }
2276
2277 // If the result is null, the sub-method took care of registering results etc.
2278 if (!Res.getNode()) return false;
2279
2280 // If the result is N, the sub-method updated N in place. Tell the legalizer
2281 // core about this.
2282 if (Res.getNode() == N)
2283 return true;
2284
2285 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2286 "Invalid operand expansion");
2287
2288 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2289 return false;
2290}
2291
2292/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
2293/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2294void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
2295 SDValue &NewRHS,
2296 ISD::CondCode &CCCode,
2297 const SDLoc &dl, SDValue &Chain,
2298 bool IsSignaling) {
2299 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2300 GetExpandedFloat(Op: NewLHS, Lo&: LHSLo, Hi&: LHSHi);
2301 GetExpandedFloat(Op: NewRHS, Lo&: RHSLo, Hi&: RHSHi);
2302
2303 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2304
2305 // FIXME: This generated code sucks. We want to generate
2306 // FCMPU crN, hi1, hi2
2307 // BNE crN, L:
2308 // FCMPU crN, lo1, lo2
2309 // The following can be improved, but not that much.
2310 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2311 Tmp1 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2312 RHS: RHSHi, Cond: ISD::SETOEQ, Chain, IsSignaling);
2313 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2314 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSLo.getValueType()), LHS: LHSLo,
2315 RHS: RHSLo, Cond: CCCode, Chain: OutputChain, IsSignaling);
2316 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2317 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2318 Tmp1 =
2319 DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi, RHS: RHSHi,
2320 Cond: ISD::SETUNE, Chain: OutputChain, IsSignaling);
2321 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(R: 1) : SDValue();
2322 Tmp2 = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: LHSHi.getValueType()), LHS: LHSHi,
2323 RHS: RHSHi, Cond: CCCode, Chain: OutputChain, IsSignaling);
2324 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(R: 1) : SDValue();
2325 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp2);
2326 NewLHS = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Tmp1.getValueType(), N1: Tmp1, N2: Tmp3);
2327 NewRHS = SDValue(); // LHS is the result, not a compare.
2328 Chain = OutputChain;
2329}
2330
2331SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2332 SDValue NewLHS = N->getOperand(Num: 2), NewRHS = N->getOperand(Num: 3);
2333 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
2334 SDValue Chain;
2335 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2336
2337 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2338 // against zero to select between true and false values.
2339 if (!NewRHS.getNode()) {
2340 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2341 CCCode = ISD::SETNE;
2342 }
2343
2344 // Update N to have the operands specified.
2345 return SDValue(DAG.UpdateNodeOperands(N, Op1: N->getOperand(Num: 0),
2346 Op2: DAG.getCondCode(Cond: CCCode), Op3: NewLHS, Op4: NewRHS,
2347 Op5: N->getOperand(Num: 4)), 0);
2348}
2349
2350SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2351 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2352 "Logic only correct for ppcf128!");
2353 SDValue Lo, Hi;
2354 GetExpandedFloat(Op: N->getOperand(Num: 1), Lo, Hi);
2355 // The ppcf128 value is providing only the sign; take it from the
2356 // higher-order double (which must have the larger magnitude).
2357 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: SDLoc(N),
2358 VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: Hi);
2359}
2360
2361SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2362 bool IsStrict = N->isStrictFPOpcode();
2363 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2364 "Logic only correct for ppcf128!");
2365 SDValue Lo, Hi;
2366 GetExpandedFloat(Op: N->getOperand(Num: IsStrict ? 1 : 0), Lo, Hi);
2367
2368 if (!IsStrict)
2369 // Round it the rest of the way (e.g. to f32) if needed.
2370 return DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N),
2371 VT: N->getValueType(ResNo: 0), N1: Hi, N2: N->getOperand(Num: 1));
2372
2373 // Eliminate the node if the input float type is the same as the output float
2374 // type.
2375 if (Hi.getValueType() == N->getValueType(ResNo: 0)) {
2376 // Connect the output chain to the input chain, unlinking the node.
2377 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2378 ReplaceValueWith(From: SDValue(N, 0), To: Hi);
2379 return SDValue();
2380 }
2381
2382 SDValue Expansion = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: SDLoc(N),
2383 ResultTys: {N->getValueType(ResNo: 0), MVT::Other},
2384 Ops: {N->getOperand(Num: 0), Hi, N->getOperand(Num: 2)});
2385 ReplaceValueWith(From: SDValue(N, 1), To: Expansion.getValue(R: 1));
2386 ReplaceValueWith(From: SDValue(N, 0), To: Expansion);
2387 return SDValue();
2388}
2389
2390SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2391 EVT RVT = N->getValueType(ResNo: 0);
2392 SDLoc dl(N);
2393
2394 bool IsStrict = N->isStrictFPOpcode();
2395 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2396 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2397 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
2398 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2399
2400 EVT NVT;
2401 RTLIB::Libcall LC = findFPToIntLibcall(SrcVT: Op.getValueType(), RetVT: RVT, Promoted&: NVT, Signed);
2402 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2403 "Unsupported FP_TO_XINT!");
2404 TargetLowering::MakeLibCallOptions CallOptions;
2405 std::pair<SDValue, SDValue> Tmp =
2406 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
2407 if (!IsStrict)
2408 return Tmp.first;
2409
2410 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
2411 ReplaceValueWith(From: SDValue(N, 0), To: Tmp.first);
2412 return SDValue();
2413}
2414
2415SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2416 SDValue NewLHS = N->getOperand(Num: 0), NewRHS = N->getOperand(Num: 1);
2417 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
2418 SDValue Chain;
2419 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain);
2420
2421 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2422 // against zero to select between true and false values.
2423 if (!NewRHS.getNode()) {
2424 NewRHS = DAG.getConstant(Val: 0, DL: SDLoc(N), VT: NewLHS.getValueType());
2425 CCCode = ISD::SETNE;
2426 }
2427
2428 // Update N to have the operands specified.
2429 return SDValue(DAG.UpdateNodeOperands(N, Op1: NewLHS, Op2: NewRHS,
2430 Op3: N->getOperand(Num: 2), Op4: N->getOperand(Num: 3),
2431 Op5: DAG.getCondCode(Cond: CCCode)), 0);
2432}
2433
2434SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2435 bool IsStrict = N->isStrictFPOpcode();
2436 SDValue NewLHS = N->getOperand(Num: IsStrict ? 1 : 0);
2437 SDValue NewRHS = N->getOperand(Num: IsStrict ? 2 : 1);
2438 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
2439 ISD::CondCode CCCode =
2440 cast<CondCodeSDNode>(Val: N->getOperand(Num: IsStrict ? 3 : 2))->get();
2441 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, dl: SDLoc(N), Chain,
2442 IsSignaling: N->getOpcode() == ISD::STRICT_FSETCCS);
2443
2444 // FloatExpandSetCCOperands always returned a scalar.
2445 assert(!NewRHS.getNode() && "Expect to return scalar");
2446 assert(NewLHS.getValueType() == N->getValueType(0) &&
2447 "Unexpected setcc expansion!");
2448 if (Chain) {
2449 ReplaceValueWith(From: SDValue(N, 0), To: NewLHS);
2450 ReplaceValueWith(From: SDValue(N, 1), To: Chain);
2451 return SDValue();
2452 }
2453 return NewLHS;
2454}
2455
2456SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2457 if (ISD::isNormalStore(N))
2458 return ExpandOp_NormalStore(N, OpNo);
2459
2460 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2461 assert(OpNo == 1 && "Can only expand the stored value so far");
2462 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2463
2464 SDValue Chain = ST->getChain();
2465 SDValue Ptr = ST->getBasePtr();
2466
2467 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
2468 VT: ST->getValue().getValueType());
2469 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2470 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2471 (void)NVT;
2472
2473 SDValue Lo, Hi;
2474 GetExpandedOp(Op: ST->getValue(), Lo, Hi);
2475
2476 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Hi, Ptr,
2477 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
2478}
2479
2480SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2481 EVT RVT = N->getValueType(ResNo: 0);
2482 EVT RetVT = N->getOperand(Num: 0).getValueType();
2483 TargetLowering::MakeLibCallOptions CallOptions;
2484 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2485 Call_F32: RTLIB::LROUND_F32,
2486 Call_F64: RTLIB::LROUND_F64,
2487 Call_F80: RTLIB::LROUND_F80,
2488 Call_F128: RTLIB::LROUND_F128,
2489 Call_PPCF128: RTLIB::LROUND_PPCF128),
2490 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2491}
2492
2493SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2494 EVT RVT = N->getValueType(ResNo: 0);
2495 EVT RetVT = N->getOperand(Num: 0).getValueType();
2496 TargetLowering::MakeLibCallOptions CallOptions;
2497 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2498 Call_F32: RTLIB::LLROUND_F32,
2499 Call_F64: RTLIB::LLROUND_F64,
2500 Call_F80: RTLIB::LLROUND_F80,
2501 Call_F128: RTLIB::LLROUND_F128,
2502 Call_PPCF128: RTLIB::LLROUND_PPCF128),
2503 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2504}
2505
2506SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2507 EVT RVT = N->getValueType(ResNo: 0);
2508 EVT RetVT = N->getOperand(Num: 0).getValueType();
2509 TargetLowering::MakeLibCallOptions CallOptions;
2510 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2511 Call_F32: RTLIB::LRINT_F32,
2512 Call_F64: RTLIB::LRINT_F64,
2513 Call_F80: RTLIB::LRINT_F80,
2514 Call_F128: RTLIB::LRINT_F128,
2515 Call_PPCF128: RTLIB::LRINT_PPCF128),
2516 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2517}
2518
2519SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2520 EVT RVT = N->getValueType(ResNo: 0);
2521 EVT RetVT = N->getOperand(Num: 0).getValueType();
2522 TargetLowering::MakeLibCallOptions CallOptions;
2523 return TLI.makeLibCall(DAG, LC: GetFPLibCall(VT: RetVT,
2524 Call_F32: RTLIB::LLRINT_F32,
2525 Call_F64: RTLIB::LLRINT_F64,
2526 Call_F80: RTLIB::LLRINT_F80,
2527 Call_F128: RTLIB::LLRINT_F128,
2528 Call_PPCF128: RTLIB::LLRINT_PPCF128),
2529 RetVT: RVT, Ops: N->getOperand(Num: 0), CallOptions, dl: SDLoc(N)).first;
2530}
2531
2532//===----------------------------------------------------------------------===//
2533// Float Operand Promotion
2534//===----------------------------------------------------------------------===//
2535//
2536
2537static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2538 if (OpVT == MVT::f16)
2539 return ISD::FP16_TO_FP;
2540 if (RetVT == MVT::f16)
2541 return ISD::FP_TO_FP16;
2542 if (OpVT == MVT::bf16)
2543 return ISD::BF16_TO_FP;
2544 if (RetVT == MVT::bf16)
2545 return ISD::FP_TO_BF16;
2546 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2547}
2548
2549static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) {
2550 if (OpVT == MVT::f16)
2551 return ISD::STRICT_FP16_TO_FP;
2552 if (RetVT == MVT::f16)
2553 return ISD::STRICT_FP_TO_FP16;
2554 if (OpVT == MVT::bf16)
2555 return ISD::STRICT_BF16_TO_FP;
2556 if (RetVT == MVT::bf16)
2557 return ISD::STRICT_FP_TO_BF16;
2558 report_fatal_error(reason: "Attempt at an invalid promotion-related conversion");
2559}
2560
2561bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2562 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2563 SDValue R = SDValue();
2564
2565 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
2566 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2567 return false;
2568 }
2569
2570 // Nodes that use a promotion-requiring floating point operand, but doesn't
2571 // produce a promotion-requiring floating point result, need to be legalized
2572 // to use the promoted float operand. Nodes that produce at least one
2573 // promotion-requiring floating point result have their operands legalized as
2574 // a part of PromoteFloatResult.
2575 // clang-format off
2576 switch (N->getOpcode()) {
2577 default:
2578 #ifndef NDEBUG
2579 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2580 N->dump(&DAG); dbgs() << "\n";
2581 #endif
2582 report_fatal_error(reason: "Do not know how to promote this operator's operand!");
2583
2584 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2585 case ISD::FAKE_USE:
2586 R = PromoteFloatOp_FAKE_USE(N, OpNo);
2587 break;
2588 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2589 case ISD::FP_TO_SINT:
2590 case ISD::FP_TO_UINT:
2591 case ISD::LROUND:
2592 case ISD::LLROUND:
2593 case ISD::LRINT:
2594 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2595 case ISD::AssertNoFPClass: R = PromoteFloatOp_AssertNoFPClass(N, OpNo); break;
2596 case ISD::FP_TO_SINT_SAT:
2597 case ISD::FP_TO_UINT_SAT:
2598 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2599 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2600 case ISD::STRICT_FP_EXTEND:
2601 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2602 break;
2603 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2604 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2605 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2606 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2607 }
2608 // clang-format on
2609
2610 if (R.getNode())
2611 ReplaceValueWith(From: SDValue(N, 0), To: R);
2612 return false;
2613}
2614
2615SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2616 SDValue Op = N->getOperand(Num: 0);
2617 EVT OpVT = Op->getValueType(ResNo: 0);
2618
2619 SDValue Promoted = GetPromotedFloat(Op: N->getOperand(Num: 0));
2620 EVT PromotedVT = Promoted->getValueType(ResNo: 0);
2621
2622 // Convert the promoted float value to the desired IVT.
2623 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OpVT.getSizeInBits());
2624 SDValue Convert = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: PromotedVT, RetVT: OpVT), DL: SDLoc(N),
2625 VT: IVT, Operand: Promoted);
2626 // The final result type might not be an scalar so we need a bitcast. The
2627 // bitcast will be further legalized if needed.
2628 return DAG.getBitcast(VT: N->getValueType(ResNo: 0), V: Convert);
2629}
2630
2631SDValue DAGTypeLegalizer::PromoteFloatOp_FAKE_USE(SDNode *N, unsigned OpNo) {
2632 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2633 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: OpNo));
2634 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::Other, N1: N->getOperand(Num: 0),
2635 N2: Op);
2636}
2637
2638// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2639// PromoteFloatRes_FCOPYSIGN.
2640SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2641 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2642 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2643
2644 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2645 N1: N->getOperand(Num: 0), N2: Op1);
2646}
2647
2648// Convert the promoted float value to the desired integer type
2649SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2650 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2651 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op);
2652}
2653
2654// Convert the promoted float value to the desired integer type
2655SDValue DAGTypeLegalizer::PromoteFloatOp_AssertNoFPClass(SDNode *N,
2656 unsigned OpNo) {
2657 return GetPromotedFloat(Op: N->getOperand(Num: 0));
2658}
2659
2660SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2661 unsigned OpNo) {
2662 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2663 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op,
2664 N2: N->getOperand(Num: 1));
2665}
2666
2667SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2668 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
2669 EVT VT = N->getValueType(ResNo: 0);
2670
2671 // Desired VT is same as promoted type. Use promoted float directly.
2672 if (VT == Op->getValueType(ResNo: 0))
2673 return Op;
2674
2675 // Else, extend the promoted float value to the desired VT.
2676 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N), VT, Operand: Op);
2677}
2678
2679SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2680 unsigned OpNo) {
2681 assert(OpNo == 1 && "Promoting unpromotable operand");
2682
2683 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 1));
2684 EVT VT = N->getValueType(ResNo: 0);
2685
2686 // Desired VT is same as promoted type. Use promoted float directly.
2687 if (VT == Op->getValueType(ResNo: 0)) {
2688 ReplaceValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
2689 return Op;
2690 }
2691
2692 // Else, extend the promoted float value to the desired VT.
2693 SDValue Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: SDLoc(N), VTList: N->getVTList(),
2694 N1: N->getOperand(Num: 0), N2: Op);
2695 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
2696 return Res;
2697}
2698
2699// Promote the float operands used for comparison. The true- and false-
2700// operands have the same type as the result and are promoted, if needed, by
2701// PromoteFloatRes_SELECT_CC
2702SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2703 SDValue LHS = GetPromotedFloat(Op: N->getOperand(Num: 0));
2704 SDValue RHS = GetPromotedFloat(Op: N->getOperand(Num: 1));
2705
2706 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
2707 N1: LHS, N2: RHS, N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3),
2708 N5: N->getOperand(Num: 4));
2709}
2710
2711// Construct a SETCC that compares the promoted values and sets the conditional
2712// code.
2713SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2714 EVT VT = N->getValueType(ResNo: 0);
2715 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
2716 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
2717 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
2718
2719 return DAG.getSetCC(DL: SDLoc(N), VT, LHS: Op0, RHS: Op1, Cond: CCCode);
2720
2721}
2722
2723// Lower the promoted Float down to the integer value of same size and construct
2724// a STORE of the integer value.
2725SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2726 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
2727 SDValue Val = ST->getValue();
2728 SDLoc DL(N);
2729
2730 SDValue Promoted = GetPromotedFloat(Op: Val);
2731 EVT VT = ST->getOperand(Num: 1).getValueType();
2732 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2733
2734 SDValue NewVal;
2735 NewVal = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: Promoted.getValueType(), RetVT: VT), DL,
2736 VT: IVT, Operand: Promoted);
2737
2738 return DAG.getStore(Chain: ST->getChain(), dl: DL, Val: NewVal, Ptr: ST->getBasePtr(),
2739 MMO: ST->getMemOperand());
2740}
2741
2742SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2743 unsigned OpNo) {
2744 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
2745 SDValue Val = ST->getVal();
2746 SDLoc DL(N);
2747
2748 SDValue Promoted = GetPromotedFloat(Op: Val);
2749 EVT VT = ST->getOperand(Num: 1).getValueType();
2750 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2751
2752 SDValue NewVal = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: Promoted.getValueType(), RetVT: VT),
2753 DL, VT: IVT, Operand: Promoted);
2754
2755 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: DL, MemVT: IVT, Chain: ST->getChain(), Ptr: NewVal,
2756 Val: ST->getBasePtr(), MMO: ST->getMemOperand());
2757}
2758
2759//===----------------------------------------------------------------------===//
2760// Float Result Promotion
2761//===----------------------------------------------------------------------===//
2762
2763void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2764 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2765 SDValue R = SDValue();
2766
2767 // See if the target wants to custom expand this node.
2768 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
2769 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2770 return;
2771 }
2772
2773 switch (N->getOpcode()) {
2774 // These opcodes cannot appear if promotion of FP16 is done in the backend
2775 // instead of Clang
2776 case ISD::FP16_TO_FP:
2777 case ISD::FP_TO_FP16:
2778 default:
2779#ifndef NDEBUG
2780 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2781 N->dump(&DAG); dbgs() << "\n";
2782#endif
2783 report_fatal_error(reason: "Do not know how to promote this operator's result!");
2784
2785 case ISD::BITCAST:
2786 R = PromoteFloatRes_BITCAST(N);
2787 break;
2788 case ISD::FREEZE:
2789 R = PromoteFloatRes_FREEZE(N);
2790 break;
2791 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2792 case ISD::EXTRACT_VECTOR_ELT:
2793 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2794 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2795
2796 // Unary FP Operations
2797 case ISD::FABS:
2798 case ISD::FACOS:
2799 case ISD::FASIN:
2800 case ISD::FATAN:
2801 case ISD::FCBRT:
2802 case ISD::FCEIL:
2803 case ISD::FCOS:
2804 case ISD::FCOSH:
2805 case ISD::FEXP:
2806 case ISD::FEXP2:
2807 case ISD::FEXP10:
2808 case ISD::FFLOOR:
2809 case ISD::FLOG:
2810 case ISD::FLOG2:
2811 case ISD::FLOG10:
2812 case ISD::FNEARBYINT:
2813 case ISD::FNEG:
2814 case ISD::FRINT:
2815 case ISD::FROUND:
2816 case ISD::FROUNDEVEN:
2817 case ISD::FSIN:
2818 case ISD::FSINH:
2819 case ISD::FSQRT:
2820 case ISD::FTRUNC:
2821 case ISD::FTAN:
2822 case ISD::FTANH:
2823 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2824 case ISD::AssertNoFPClass:
2825 R = PromoteFloatRes_AssertNoFPClass(N);
2826 break;
2827
2828 // Binary FP Operations
2829 case ISD::FADD:
2830 case ISD::FDIV:
2831 case ISD::FMAXIMUM:
2832 case ISD::FMINIMUM:
2833 case ISD::FMAXIMUMNUM:
2834 case ISD::FMINIMUMNUM:
2835 case ISD::FMAXNUM:
2836 case ISD::FMINNUM:
2837 case ISD::FMAXNUM_IEEE:
2838 case ISD::FMINNUM_IEEE:
2839 case ISD::FMUL:
2840 case ISD::FPOW:
2841 case ISD::FATAN2:
2842 case ISD::FREM:
2843 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2844
2845 case ISD::FMA: // FMA is same as FMAD
2846 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2847
2848 case ISD::FPOWI:
2849 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2850 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2851
2852 case ISD::FMODF:
2853 case ISD::FSINCOS:
2854 case ISD::FSINCOSPI:
2855 R = PromoteFloatRes_UnaryWithTwoFPResults(N);
2856 break;
2857 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2858 case ISD::STRICT_FP_ROUND:
2859 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2860 break;
2861 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2862 case ISD::ATOMIC_LOAD:
2863 R = PromoteFloatRes_ATOMIC_LOAD(N);
2864 break;
2865 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2866 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2867
2868 case ISD::SINT_TO_FP:
2869 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2870 case ISD::POISON:
2871 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2872 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2873 case ISD::VECREDUCE_FADD:
2874 case ISD::VECREDUCE_FMUL:
2875 case ISD::VECREDUCE_FMIN:
2876 case ISD::VECREDUCE_FMAX:
2877 case ISD::VECREDUCE_FMAXIMUM:
2878 case ISD::VECREDUCE_FMINIMUM:
2879 R = PromoteFloatRes_VECREDUCE(N);
2880 break;
2881 case ISD::VECREDUCE_SEQ_FADD:
2882 case ISD::VECREDUCE_SEQ_FMUL:
2883 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2884 break;
2885 }
2886
2887 if (R.getNode())
2888 SetPromotedFloat(Op: SDValue(N, ResNo), Result: R);
2889}
2890
2891// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2892// At this point, it is not possible to determine if the bitcast value is
2893// eventually stored to memory or promoted to f32 or promoted to a floating
2894// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2895// STORE promotion handlers.
2896SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2897 EVT VT = N->getValueType(ResNo: 0);
2898 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2899 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2900 // bitcast will be legalized further if necessary.
2901 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
2902 BitWidth: N->getOperand(Num: 0).getValueType().getSizeInBits());
2903 SDValue Cast = DAG.getBitcast(VT: IVT, V: N->getOperand(Num: 0));
2904 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: Cast);
2905}
2906
2907SDValue DAGTypeLegalizer::PromoteFloatRes_FREEZE(SDNode *N) {
2908 EVT VT = N->getValueType(ResNo: 0);
2909 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2910 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2911 // bitcast will be legalized further if necessary.
2912 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
2913 BitWidth: N->getOperand(Num: 0).getValueType().getSizeInBits());
2914 SDValue Cast = DAG.getBitcast(VT: IVT, V: N->getOperand(Num: 0));
2915 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT,
2916 Operand: DAG.getFreeze(V: Cast));
2917}
2918
2919SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2920 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(Val: N);
2921 EVT VT = N->getValueType(ResNo: 0);
2922 SDLoc DL(N);
2923
2924 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2925 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
2926 SDValue C = DAG.getConstant(Val: CFPNode->getValueAPF().bitcastToAPInt(), DL,
2927 VT: IVT);
2928
2929 // Convert the Constant to the desired FP type
2930 // FIXME We might be able to do the conversion during compilation and get rid
2931 // of it from the object code
2932 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2933 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: C);
2934}
2935
2936// If the Index operand is a constant, try to redirect the extract operation to
2937// the correct legalized vector. If not, bit-convert the input vector to
2938// equivalent integer vector. Extract the element as an (bit-cast) integer
2939// value and convert it to the promoted type.
2940SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2941 SDLoc DL(N);
2942
2943 // If the index is constant, try to extract the value from the legalized
2944 // vector type.
2945 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
2946 SDValue Vec = N->getOperand(Num: 0);
2947 SDValue Idx = N->getOperand(Num: 1);
2948 EVT VecVT = Vec->getValueType(ResNo: 0);
2949 EVT EltVT = VecVT.getVectorElementType();
2950
2951 uint64_t IdxVal = Idx->getAsZExtVal();
2952
2953 switch (getTypeAction(VT: VecVT)) {
2954 default: break;
2955 case TargetLowering::TypeScalarizeVector: {
2956 SDValue Res = GetScalarizedVector(Op: N->getOperand(Num: 0));
2957 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2958 return SDValue();
2959 }
2960 case TargetLowering::TypeWidenVector: {
2961 Vec = GetWidenedVector(Op: Vec);
2962 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Vec, N2: Idx);
2963 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2964 return SDValue();
2965 }
2966 case TargetLowering::TypeSplitVector: {
2967 SDValue Lo, Hi;
2968 GetSplitVector(Op: Vec, Lo, Hi);
2969
2970 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2971 SDValue Res;
2972 if (IdxVal < LoElts)
2973 Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Lo, N2: Idx);
2974 else
2975 Res = DAG.getNode(Opcode: N->getOpcode(), DL, VT: EltVT, N1: Hi,
2976 N2: DAG.getConstant(Val: IdxVal - LoElts, DL,
2977 VT: Idx.getValueType()));
2978 ReplaceValueWith(From: SDValue(N, 0), To: Res);
2979 return SDValue();
2980 }
2981
2982 }
2983 }
2984
2985 // Bit-convert the input vector to the equivalent integer vector
2986 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
2987 EVT IVT = NewOp.getValueType().getVectorElementType();
2988
2989 // Extract the element as an (bit-cast) integer value
2990 SDValue NewVal = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: IVT,
2991 N1: NewOp, N2: N->getOperand(Num: 1));
2992
2993 // Convert the element to the desired FP type
2994 EVT VT = N->getValueType(ResNo: 0);
2995 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
2996 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: NewVal);
2997}
2998
2999// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
3000// needs promotion, so does the argument X. Note that Y, if needed, will be
3001// handled during operand promotion.
3002SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
3003 EVT VT = N->getValueType(ResNo: 0);
3004 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3005 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
3006
3007 SDValue Op1 = N->getOperand(Num: 1);
3008
3009 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1);
3010}
3011
3012// Unary operation where the result and the operand have PromoteFloat type
3013// action. Construct a new SDNode with the promoted float value of the old
3014// operand.
3015SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
3016 EVT VT = N->getValueType(ResNo: 0);
3017 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3018 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
3019 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, Operand: Op);
3020}
3021
3022// Unary operation with a more non-float operand where the result and the
3023// operand have PromoteFloat type action. Construct a new SDNode with the
3024// promoted float value of the old operand.
3025SDValue DAGTypeLegalizer::PromoteFloatRes_AssertNoFPClass(SDNode *N) {
3026 return GetPromotedFloat(Op: N->getOperand(Num: 0));
3027}
3028
3029// Binary operations where the result and both operands have PromoteFloat type
3030// action. Construct a new SDNode with the promoted float values of the old
3031// operands.
3032SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
3033 EVT VT = N->getValueType(ResNo: 0);
3034 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3035 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
3036 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
3037 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, Flags: N->getFlags());
3038}
3039
3040SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
3041 EVT VT = N->getValueType(ResNo: 0);
3042 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3043 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
3044 SDValue Op1 = GetPromotedFloat(Op: N->getOperand(Num: 1));
3045 SDValue Op2 = GetPromotedFloat(Op: N->getOperand(Num: 2));
3046
3047 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1, N3: Op2);
3048}
3049
3050// Promote the Float (first) operand and retain the Integer (second) operand
3051SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
3052 EVT VT = N->getValueType(ResNo: 0);
3053 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3054 SDValue Op0 = GetPromotedFloat(Op: N->getOperand(Num: 0));
3055 SDValue Op1 = N->getOperand(Num: 1);
3056
3057 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NVT, N1: Op0, N2: Op1);
3058}
3059
3060SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
3061 EVT VT = N->getValueType(ResNo: 0);
3062 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3063 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
3064 SDValue Res =
3065 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {NVT, N->getValueType(ResNo: 1)}, Ops: Op);
3066
3067 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3068 return Res;
3069}
3070
3071SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryWithTwoFPResults(SDNode *N) {
3072 EVT VT = N->getValueType(ResNo: 0);
3073 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3074 SDValue Op = GetPromotedFloat(Op: N->getOperand(Num: 0));
3075 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), ResultTys: {NVT, NVT}, Ops: Op);
3076
3077 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
3078 ++ResNum) {
3079 SetPromotedFloat(Op: SDValue(N, ResNum), Result: Res.getValue(R: ResNum));
3080 }
3081
3082 return SDValue();
3083}
3084
3085// Explicit operation to reduce precision. Reduce the value to half precision
3086// and promote it back to the legal type.
3087SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
3088 SDLoc DL(N);
3089
3090 SDValue Op = N->getOperand(Num: 0);
3091 EVT VT = N->getValueType(ResNo: 0);
3092 EVT OpVT = Op->getValueType(ResNo: 0);
3093 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
3094 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
3095
3096 // Round promoted float to desired precision
3097 SDValue Round = DAG.getNode(Opcode: GetPromotionOpcode(OpVT, RetVT: VT), DL, VT: IVT, Operand: Op);
3098 // Promote it back to the legal output type
3099 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL, VT: NVT, Operand: Round);
3100}
3101
3102// Explicit operation to reduce precision. Reduce the value to half precision
3103// and promote it back to the legal type.
3104SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
3105 SDLoc DL(N);
3106
3107 SDValue Chain = N->getOperand(Num: 0);
3108 SDValue Op = N->getOperand(Num: 1);
3109 EVT VT = N->getValueType(ResNo: 0);
3110 EVT OpVT = Op->getValueType(ResNo: 0);
3111 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: N->getValueType(ResNo: 0));
3112 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
3113
3114 // Round promoted float to desired precision
3115 SDValue Round = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT, RetVT: VT), DL,
3116 VTList: DAG.getVTList(VT1: IVT, VT2: MVT::Other), N1: Chain, N2: Op);
3117 // Promote it back to the legal output type
3118 SDValue Res =
3119 DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: VT, RetVT: NVT), DL,
3120 VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other), N1: Round.getValue(R: 1), N2: Round);
3121 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3122 return Res;
3123}
3124
3125SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
3126 LoadSDNode *L = cast<LoadSDNode>(Val: N);
3127 EVT VT = N->getValueType(ResNo: 0);
3128
3129 // Load the value as an integer value with the same number of bits.
3130 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
3131 SDValue newL = DAG.getLoad(
3132 AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: IVT, dl: SDLoc(N),
3133 Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(), PtrInfo: L->getPointerInfo(), MemVT: IVT,
3134 Alignment: L->getBaseAlign(), MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo());
3135 // Legalize the chain result by replacing uses of the old value chain with the
3136 // new one
3137 ReplaceValueWith(From: SDValue(N, 1), To: newL.getValue(R: 1));
3138
3139 // Convert the integer value to the desired FP type
3140 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3141 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NVT), DL: SDLoc(N), VT: NVT, Operand: newL);
3142}
3143
3144SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
3145 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
3146 EVT VT = AM->getValueType(ResNo: 0);
3147
3148 // Load the value as an integer value with the same number of bits.
3149 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
3150 SDValue newL = DAG.getAtomic(
3151 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: IVT, VTList: DAG.getVTList(VT1: IVT, VT2: MVT::Other),
3152 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
3153
3154 // Legalize the chain result by replacing uses of the old value chain with the
3155 // new one
3156 ReplaceValueWith(From: SDValue(N, 1), To: newL.getValue(R: 1));
3157
3158 // Convert the integer value to the desired FP type
3159 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3160 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: IVT), DL: SDLoc(N), VT: NVT, Operand: newL);
3161}
3162
3163// Construct a new SELECT node with the promoted true- and false- values.
3164SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
3165 SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 1));
3166 SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 2));
3167
3168 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT: TrueVal->getValueType(ResNo: 0),
3169 N1: N->getOperand(Num: 0), N2: TrueVal, N3: FalseVal);
3170}
3171
3172// Construct a new SELECT_CC node with the promoted true- and false- values.
3173// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
3174SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
3175 SDValue TrueVal = GetPromotedFloat(Op: N->getOperand(Num: 2));
3176 SDValue FalseVal = GetPromotedFloat(Op: N->getOperand(Num: 3));
3177
3178 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N),
3179 VT: TrueVal.getNode()->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3180 N2: N->getOperand(Num: 1), N3: TrueVal, N4: FalseVal, N5: N->getOperand(Num: 4));
3181}
3182
3183// Construct a SDNode that transforms the SINT or UINT operand to the promoted
3184// float type.
3185SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
3186 SDLoc DL(N);
3187 EVT VT = N->getValueType(ResNo: 0);
3188 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3189 SDValue NV = DAG.getNode(Opcode: N->getOpcode(), DL, VT: NVT, Operand: N->getOperand(Num: 0));
3190 // Round the value to the desired precision (that of the source type).
3191 return DAG.getNode(
3192 Opcode: ISD::FP_EXTEND, DL, VT: NVT,
3193 Operand: DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NV,
3194 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)));
3195}
3196
3197SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
3198 return DAG.getUNDEF(VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(),
3199 VT: N->getValueType(ResNo: 0)));
3200}
3201
3202SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
3203 // Expand and promote recursively.
3204 // TODO: This is non-optimal, but dealing with the concurrently happening
3205 // vector-legalization is non-trivial. We could do something similar to
3206 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
3207 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
3208 return SDValue();
3209}
3210
3211SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
3212 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3213 return SDValue();
3214}
3215
3216SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
3217 EVT VT = N->getValueType(ResNo: 0);
3218
3219 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
3220 SDLoc SL(N);
3221
3222 SDValue CastVal = BitConvertToInteger(Op: AM->getVal());
3223 EVT CastVT = CastVal.getValueType();
3224
3225 SDValue NewAtomic
3226 = DAG.getAtomic(Opcode: ISD::ATOMIC_SWAP, dl: SL, MemVT: CastVT,
3227 VTList: DAG.getVTList(VT1: CastVT, VT2: MVT::Other),
3228 Ops: { AM->getChain(), AM->getBasePtr(), CastVal },
3229 MMO: AM->getMemOperand());
3230
3231 SDValue Result = NewAtomic;
3232
3233 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
3234 EVT NFPVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT);
3235 Result = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: VT, RetVT: NFPVT), DL: SL, VT: NFPVT,
3236 Operand: NewAtomic);
3237 }
3238
3239 // Legalize the chain result by replacing uses of the old value chain with the
3240 // new one
3241 ReplaceValueWith(From: SDValue(N, 1), To: NewAtomic.getValue(R: 1));
3242
3243 return Result;
3244
3245}
3246
3247//===----------------------------------------------------------------------===//
3248// Half Result Soft Promotion
3249//===----------------------------------------------------------------------===//
3250
3251void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
3252 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
3253 N->dump(&DAG));
3254 SDValue R = SDValue();
3255
3256 // See if the target wants to custom expand this node.
3257 if (CustomLowerNode(N, VT: N->getValueType(ResNo), LegalizeResult: true)) {
3258 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
3259 return;
3260 }
3261
3262 switch (N->getOpcode()) {
3263 default:
3264#ifndef NDEBUG
3265 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
3266 N->dump(&DAG); dbgs() << "\n";
3267#endif
3268 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3269 "result!");
3270
3271 case ISD::ARITH_FENCE:
3272 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
3273 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
3274 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
3275 case ISD::EXTRACT_VECTOR_ELT:
3276 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
3277 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
3278 case ISD::STRICT_FP_ROUND:
3279 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
3280
3281 // Unary FP Operations
3282 case ISD::FABS:
3283 case ISD::FACOS:
3284 case ISD::FASIN:
3285 case ISD::FATAN:
3286 case ISD::FCBRT:
3287 case ISD::FCEIL:
3288 case ISD::FCOS:
3289 case ISD::FCOSH:
3290 case ISD::FEXP:
3291 case ISD::FEXP2:
3292 case ISD::FEXP10:
3293 case ISD::FFLOOR:
3294 case ISD::FLOG:
3295 case ISD::FLOG2:
3296 case ISD::FLOG10:
3297 case ISD::FNEARBYINT:
3298 case ISD::FNEG:
3299 case ISD::FREEZE:
3300 case ISD::FRINT:
3301 case ISD::FROUND:
3302 case ISD::FROUNDEVEN:
3303 case ISD::FSIN:
3304 case ISD::FSINH:
3305 case ISD::FSQRT:
3306 case ISD::FTRUNC:
3307 case ISD::FTAN:
3308 case ISD::FTANH:
3309 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
3310 case ISD::AssertNoFPClass:
3311 R = SoftPromoteHalfRes_AssertNoFPClass(N);
3312 break;
3313
3314 // Binary FP Operations
3315 case ISD::FADD:
3316 case ISD::FDIV:
3317 case ISD::FMAXIMUM:
3318 case ISD::FMINIMUM:
3319 case ISD::FMAXIMUMNUM:
3320 case ISD::FMINIMUMNUM:
3321 case ISD::FMAXNUM:
3322 case ISD::FMINNUM:
3323 case ISD::FMUL:
3324 case ISD::FPOW:
3325 case ISD::FATAN2:
3326 case ISD::FREM:
3327 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
3328
3329 case ISD::FMA: // FMA is same as FMAD
3330 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
3331
3332 case ISD::FPOWI:
3333 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
3334
3335 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
3336
3337 case ISD::FMODF:
3338 case ISD::FSINCOS:
3339 case ISD::FSINCOSPI:
3340 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N);
3341 break;
3342
3343 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
3344 case ISD::ATOMIC_LOAD:
3345 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
3346 break;
3347 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
3348 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
3349 case ISD::STRICT_SINT_TO_FP:
3350 case ISD::STRICT_UINT_TO_FP:
3351 case ISD::SINT_TO_FP:
3352 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
3353 case ISD::POISON:
3354 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
3355 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
3356 case ISD::VECREDUCE_FADD:
3357 case ISD::VECREDUCE_FMUL:
3358 case ISD::VECREDUCE_FMIN:
3359 case ISD::VECREDUCE_FMAX:
3360 case ISD::VECREDUCE_FMAXIMUM:
3361 case ISD::VECREDUCE_FMINIMUM:
3362 R = SoftPromoteHalfRes_VECREDUCE(N);
3363 break;
3364 case ISD::VECREDUCE_SEQ_FADD:
3365 case ISD::VECREDUCE_SEQ_FMUL:
3366 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
3367 break;
3368 }
3369
3370 if (R.getNode())
3371 SetSoftPromotedHalf(Op: SDValue(N, ResNo), Result: R);
3372}
3373
3374SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
3375 return DAG.getNode(Opcode: ISD::ARITH_FENCE, DL: SDLoc(N), VT: MVT::i16,
3376 Operand: BitConvertToInteger(Op: N->getOperand(Num: 0)));
3377}
3378
3379SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
3380 return BitConvertToInteger(Op: N->getOperand(Num: 0));
3381}
3382
3383SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
3384 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(Val: N);
3385
3386 // Get the (bit-cast) APInt of the APFloat and build an integer constant
3387 return DAG.getConstant(Val: CN->getValueAPF().bitcastToAPInt(), DL: SDLoc(CN),
3388 VT: MVT::i16);
3389}
3390
3391SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
3392 SDValue NewOp = BitConvertVectorToIntegerVector(Op: N->getOperand(Num: 0));
3393 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SDLoc(N),
3394 VT: NewOp.getValueType().getVectorElementType(), N1: NewOp,
3395 N2: N->getOperand(Num: 1));
3396}
3397
3398SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
3399 SDValue LHS = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3400 SDValue RHS = BitConvertToInteger(Op: N->getOperand(Num: 1));
3401 SDLoc dl(N);
3402
3403 EVT LVT = LHS.getValueType();
3404 EVT RVT = RHS.getValueType();
3405
3406 unsigned LSize = LVT.getSizeInBits();
3407 unsigned RSize = RVT.getSizeInBits();
3408
3409 // First get the sign bit of second operand.
3410 SDValue SignBit = DAG.getNode(
3411 Opcode: ISD::SHL, DL: dl, VT: RVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: RVT),
3412 N2: DAG.getConstant(Val: RSize - 1, DL: dl,
3413 VT: TLI.getShiftAmountTy(LHSTy: RVT, DL: DAG.getDataLayout())));
3414 SignBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: RVT, N1: RHS, N2: SignBit);
3415
3416 // Shift right or sign-extend it if the two operands have different types.
3417 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3418 if (SizeDiff > 0) {
3419 SignBit =
3420 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: RVT, N1: SignBit,
3421 N2: DAG.getConstant(Val: SizeDiff, DL: dl,
3422 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
3423 DL: DAG.getDataLayout())));
3424 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: LVT, Operand: SignBit);
3425 } else if (SizeDiff < 0) {
3426 SignBit = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: LVT, Operand: SignBit);
3427 SignBit =
3428 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: LVT, N1: SignBit,
3429 N2: DAG.getConstant(Val: -SizeDiff, DL: dl,
3430 VT: TLI.getShiftAmountTy(LHSTy: SignBit.getValueType(),
3431 DL: DAG.getDataLayout())));
3432 }
3433
3434 // Clear the sign bit of the first operand.
3435 SDValue Mask = DAG.getNode(
3436 Opcode: ISD::SHL, DL: dl, VT: LVT, N1: DAG.getConstant(Val: 1, DL: dl, VT: LVT),
3437 N2: DAG.getConstant(Val: LSize - 1, DL: dl,
3438 VT: TLI.getShiftAmountTy(LHSTy: LVT, DL: DAG.getDataLayout())));
3439 Mask = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: LVT, N1: Mask, N2: DAG.getConstant(Val: 1, DL: dl, VT: LVT));
3440 LHS = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: LVT, N1: LHS, N2: Mask);
3441
3442 // Or the value with the sign bit.
3443 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT: LVT, N1: LHS, N2: SignBit);
3444}
3445
3446SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3447 EVT OVT = N->getValueType(ResNo: 0);
3448 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3449 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3450 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3451 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3452 SDLoc dl(N);
3453
3454 // Promote to the larger FP type.
3455 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
3456 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3457 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3458 Op2 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op2);
3459
3460 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1, N3: Op2);
3461
3462 // Convert back to FP16 as an integer.
3463 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3464}
3465
3466SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3467 EVT OVT = N->getValueType(ResNo: 0);
3468 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3469 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3470 SDValue Op1 = N->getOperand(Num: 1);
3471 SDLoc dl(N);
3472
3473 // Promote to the larger FP type.
3474 Op0 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op0);
3475
3476 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3477
3478 // Convert back to FP16 as an integer.
3479 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3480}
3481
3482SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3483 EVT OVT = N->getValueType(ResNo: 0);
3484 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3485 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3486 SDLoc dl(N);
3487
3488 // Promote to the larger FP type.
3489 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3490
3491 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl,
3492 VTList: DAG.getVTList(VT1: NVT, VT2: N->getValueType(ResNo: 1)), N: Op);
3493
3494 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3495
3496 // Convert back to FP16 as an integer.
3497 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3498}
3499
3500SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) {
3501 EVT OVT = N->getValueType(ResNo: 0);
3502 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3503 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3504 SDLoc dl(N);
3505
3506 // Promote to the larger FP type.
3507 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3508 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Op);
3509
3510 // Convert back to FP16 as an integer.
3511 ISD::NodeType Truncate = GetPromotionOpcode(OpVT: NVT, RetVT: OVT);
3512 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues;
3513 ++ResNum) {
3514 SDValue Trunc = DAG.getNode(Opcode: Truncate, DL: dl, VT: MVT::i16, Operand: Res.getValue(R: ResNum));
3515 SetSoftPromotedHalf(Op: SDValue(N, ResNum), Result: Trunc);
3516 }
3517
3518 return SDValue();
3519}
3520
3521SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3522 EVT RVT = N->getValueType(ResNo: 0);
3523 bool IsStrict = N->isStrictFPOpcode();
3524 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3525 EVT SVT = Op.getValueType();
3526
3527 // If the input type needs to be softened, do that now so that call lowering
3528 // will see the f16 type.
3529 if (getTypeAction(VT: SVT) == TargetLowering::TypeSoftenFloat) {
3530 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: SVT, RetVT: RVT);
3531 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
3532
3533 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
3534 Op = GetSoftenedFloat(Op);
3535 TargetLowering::MakeLibCallOptions CallOptions;
3536 CallOptions.setTypeListBeforeSoften(OpsVT: SVT, RetVT: RVT, Value: true);
3537 std::pair<SDValue, SDValue> Tmp =
3538 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl: SDLoc(N), Chain);
3539 if (IsStrict)
3540 ReplaceValueWith(From: SDValue(N, 1), To: Tmp.second);
3541 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: MVT::i16, Operand: Tmp.first);
3542 }
3543
3544 if (IsStrict) {
3545 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
3546 ResultTys: {MVT::i16, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
3547 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3548 return Res;
3549 }
3550
3551 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: MVT::i16,
3552 Operand: N->getOperand(Num: 0));
3553}
3554
3555SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3556 LoadSDNode *L = cast<LoadSDNode>(Val: N);
3557
3558 // Load the value as an integer value with the same number of bits.
3559 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3560 SDValue NewL =
3561 DAG.getLoad(AM: L->getAddressingMode(), ExtType: L->getExtensionType(), VT: MVT::i16,
3562 dl: SDLoc(N), Chain: L->getChain(), Ptr: L->getBasePtr(), Offset: L->getOffset(),
3563 PtrInfo: L->getPointerInfo(), MemVT: MVT::i16, Alignment: L->getBaseAlign(),
3564 MMOFlags: L->getMemOperand()->getFlags(), AAInfo: L->getAAInfo());
3565 // Legalize the chain result by replacing uses of the old value chain with the
3566 // new one
3567 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3568 return NewL;
3569}
3570
3571SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3572 AtomicSDNode *AM = cast<AtomicSDNode>(Val: N);
3573
3574 // Load the value as an integer value with the same number of bits.
3575 SDValue NewL = DAG.getAtomic(
3576 Opcode: ISD::ATOMIC_LOAD, dl: SDLoc(N), MemVT: MVT::i16, VTList: DAG.getVTList(VT1: MVT::i16, VT2: MVT::Other),
3577 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
3578
3579 // Legalize the chain result by replacing uses of the old value chain with the
3580 // new one
3581 ReplaceValueWith(From: SDValue(N, 1), To: NewL.getValue(R: 1));
3582 return NewL;
3583}
3584
3585SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3586 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3587 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3588 return DAG.getSelect(DL: SDLoc(N), VT: Op1.getValueType(), Cond: N->getOperand(Num: 0), LHS: Op1,
3589 RHS: Op2);
3590}
3591
3592SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3593 SDValue Op2 = GetSoftPromotedHalf(Op: N->getOperand(Num: 2));
3594 SDValue Op3 = GetSoftPromotedHalf(Op: N->getOperand(Num: 3));
3595 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: Op2.getValueType(),
3596 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1), N3: Op2, N4: Op3,
3597 N5: N->getOperand(Num: 4));
3598}
3599
3600SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3601 EVT OVT = N->getValueType(ResNo: 0);
3602 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3603 SDLoc dl(N);
3604
3605 if (N->isStrictFPOpcode()) {
3606 SDValue Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
3607 Ops: {N->getOperand(Num: 0), N->getOperand(Num: 1)});
3608 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: NVT, RetVT: OVT), DL: dl,
3609 ResultTys: {MVT::i16, MVT::Other}, Ops: {Op.getValue(R: 1), Op});
3610 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3611 return Op;
3612 }
3613
3614 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: N->getOperand(Num: 0));
3615
3616 // Round the value to the softened type.
3617 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3618}
3619
3620SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3621 return DAG.getUNDEF(VT: MVT::i16);
3622}
3623
3624SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3625 EVT OVT = N->getValueType(ResNo: 0);
3626 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3627 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3628 SDLoc dl(N);
3629
3630 // Promote to the larger FP type.
3631 Op = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: OVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op);
3632
3633 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, Operand: Op);
3634
3635 // Convert back to FP16 as an integer.
3636 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3637}
3638
3639SDValue DAGTypeLegalizer::SoftPromoteHalfRes_AssertNoFPClass(SDNode *N) {
3640 return GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3641}
3642
3643SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3644 EVT OVT = N->getValueType(ResNo: 0);
3645 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: OVT);
3646 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3647 SDValue Op1 = GetSoftPromotedHalf(Op: N->getOperand(Num: 1));
3648 SDLoc dl(N);
3649
3650 // Promote to the larger FP type.
3651 auto PromotionOpcode = GetPromotionOpcode(OpVT: OVT, RetVT: NVT);
3652 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3653 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3654
3655 SDValue Res = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: NVT, N1: Op0, N2: Op1);
3656
3657 // Convert back to FP16 as an integer.
3658 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: NVT, RetVT: OVT), DL: dl, VT: MVT::i16, Operand: Res);
3659}
3660
3661SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3662 // Expand and soften recursively.
3663 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduce(Node: N, DAG));
3664 return SDValue();
3665}
3666
3667SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3668 // Expand and soften.
3669 ReplaceValueWith(From: SDValue(N, 0), To: TLI.expandVecReduceSeq(Node: N, DAG));
3670 return SDValue();
3671}
3672
3673//===----------------------------------------------------------------------===//
3674// Half Operand Soft Promotion
3675//===----------------------------------------------------------------------===//
3676
3677bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3678 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3679 N->dump(&DAG));
3680 SDValue Res = SDValue();
3681
3682 if (CustomLowerNode(N, VT: N->getOperand(Num: OpNo).getValueType(), LegalizeResult: false)) {
3683 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3684 return false;
3685 }
3686
3687 // Nodes that use a promotion-requiring floating point operand, but doesn't
3688 // produce a soft promotion-requiring floating point result, need to be
3689 // legalized to use the soft promoted float operand. Nodes that produce at
3690 // least one soft promotion-requiring floating point result have their
3691 // operands legalized as a part of PromoteFloatResult.
3692 switch (N->getOpcode()) {
3693 default:
3694 #ifndef NDEBUG
3695 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3696 N->dump(&DAG); dbgs() << "\n";
3697 #endif
3698 report_fatal_error(reason: "Do not know how to soft promote this operator's "
3699 "operand!");
3700
3701 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3702 case ISD::FAKE_USE:
3703 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo);
3704 break;
3705 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3706 case ISD::STRICT_FP_TO_SINT:
3707 case ISD::STRICT_FP_TO_UINT:
3708 case ISD::FP_TO_SINT:
3709 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3710 case ISD::FP_TO_SINT_SAT:
3711 case ISD::FP_TO_UINT_SAT:
3712 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3713 case ISD::STRICT_FP_EXTEND:
3714 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3715 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3716 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3717 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3718 case ISD::ATOMIC_STORE:
3719 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3720 break;
3721 case ISD::STACKMAP:
3722 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3723 break;
3724 case ISD::PATCHPOINT:
3725 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3726 break;
3727 }
3728
3729 if (!Res.getNode())
3730 return false;
3731
3732 assert(Res.getNode() != N && "Expected a new node!");
3733
3734 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3735 "Invalid operand expansion");
3736
3737 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3738 return false;
3739}
3740
3741SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3742 SDValue Op0 = GetSoftPromotedHalf(Op: N->getOperand(Num: 0));
3743
3744 return DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Op0);
3745}
3746
3747SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) {
3748 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3749 SDValue Op = GetSoftPromotedHalf(Op: N->getOperand(Num: OpNo));
3750 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: MVT::Other, N1: N->getOperand(Num: 0),
3751 N2: Op);
3752}
3753
3754SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3755 unsigned OpNo) {
3756 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3757 SDValue Op1 = N->getOperand(Num: 1);
3758 EVT RVT = Op1.getValueType();
3759 SDLoc dl(N);
3760
3761 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op1.getValueType());
3762
3763 Op1 = GetSoftPromotedHalf(Op: Op1);
3764 Op1 = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: RVT, RetVT: NVT), DL: dl, VT: NVT, Operand: Op1);
3765
3766 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
3767 N2: Op1);
3768}
3769
3770SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3771 EVT RVT = N->getValueType(ResNo: 0);
3772 bool IsStrict = N->isStrictFPOpcode();
3773 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3774 EVT SVT = Op.getValueType();
3775 Op = GetSoftPromotedHalf(Op: N->getOperand(Num: IsStrict ? 1 : 0));
3776
3777 if (IsStrict) {
3778 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: SDLoc(N),
3779 ResultTys: {RVT, MVT::Other}, Ops: {N->getOperand(Num: 0), Op});
3780 ReplaceValueWith(From: SDValue(N, 1), To: Res.getValue(R: 1));
3781 ReplaceValueWith(From: SDValue(N, 0), To: Res);
3782 return SDValue();
3783 }
3784
3785 return DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: SDLoc(N), VT: RVT, Operand: Op);
3786}
3787
3788SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3789 EVT RVT = N->getValueType(ResNo: 0);
3790 bool IsStrict = N->isStrictFPOpcode();
3791 SDValue Op = N->getOperand(Num: IsStrict ? 1 : 0);
3792 EVT SVT = Op.getValueType();
3793 SDLoc dl(N);
3794
3795 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3796 Op = GetSoftPromotedHalf(Op);
3797
3798 if (IsStrict) {
3799 Op = DAG.getNode(Opcode: GetPromotionOpcodeStrict(OpVT: SVT, RetVT: RVT), DL: dl, ResultTys: {NVT, MVT::Other},
3800 Ops: {N->getOperand(Num: 0), Op});
3801 Op = DAG.getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {RVT, MVT::Other},
3802 Ops: {Op.getValue(R: 1), Op});
3803 ReplaceValueWith(From: SDValue(N, 1), To: Op.getValue(R: 1));
3804 ReplaceValueWith(From: SDValue(N, 0), To: Op);
3805 return SDValue();
3806 }
3807
3808 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3809 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: RVT, Operand: Res);
3810}
3811
3812SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3813 EVT RVT = N->getValueType(ResNo: 0);
3814 SDValue Op = N->getOperand(Num: 0);
3815 EVT SVT = Op.getValueType();
3816 SDLoc dl(N);
3817
3818 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op.getValueType());
3819
3820 Op = GetSoftPromotedHalf(Op);
3821
3822 SDValue Res = DAG.getNode(Opcode: GetPromotionOpcode(OpVT: SVT, RetVT: RVT), DL: dl, VT: NVT, Operand: Op);
3823
3824 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), N1: Res,
3825 N2: N->getOperand(Num: 1));
3826}
3827
3828SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3829 unsigned OpNo) {
3830 assert(OpNo == 0 && "Can only soften the comparison values");
3831 SDValue Op0 = N->getOperand(Num: 0);
3832 SDValue Op1 = N->getOperand(Num: 1);
3833 SDLoc dl(N);
3834
3835 EVT SVT = Op0.getValueType();
3836 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
3837
3838 Op0 = GetSoftPromotedHalf(Op: Op0);
3839 Op1 = GetSoftPromotedHalf(Op: Op1);
3840
3841 // Promote to the larger FP type.
3842 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3843 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3844 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3845
3846 return DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: Op0, N2: Op1,
3847 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
3848}
3849
3850SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3851 SDValue Op0 = N->getOperand(Num: 0);
3852 SDValue Op1 = N->getOperand(Num: 1);
3853 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3854 SDLoc dl(N);
3855
3856 EVT SVT = Op0.getValueType();
3857 EVT NVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: Op0.getValueType());
3858
3859 Op0 = GetSoftPromotedHalf(Op: Op0);
3860 Op1 = GetSoftPromotedHalf(Op: Op1);
3861
3862 // Promote to the larger FP type.
3863 auto PromotionOpcode = GetPromotionOpcode(OpVT: SVT, RetVT: NVT);
3864 Op0 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op0);
3865 Op1 = DAG.getNode(Opcode: PromotionOpcode, DL: dl, VT: NVT, Operand: Op1);
3866
3867 return DAG.getSetCC(DL: SDLoc(N), VT: N->getValueType(ResNo: 0), LHS: Op0, RHS: Op1, Cond: CCCode);
3868}
3869
3870SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3871 assert(OpNo == 1 && "Can only soften the stored value!");
3872 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
3873 SDValue Val = ST->getValue();
3874 SDLoc dl(N);
3875
3876 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3877 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3878 return DAG.getStore(Chain: ST->getChain(), dl, Val: Promoted, Ptr: ST->getBasePtr(),
3879 MMO: ST->getMemOperand());
3880}
3881
3882SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3883 unsigned OpNo) {
3884 assert(OpNo == 1 && "Can only soften the stored value!");
3885 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
3886 SDValue Val = ST->getVal();
3887 SDLoc dl(N);
3888
3889 SDValue Promoted = GetSoftPromotedHalf(Op: Val);
3890 return DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl, MemVT: Promoted.getValueType(),
3891 Chain: ST->getChain(), Ptr: Promoted, Val: ST->getBasePtr(),
3892 MMO: ST->getMemOperand());
3893}
3894
3895SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3896 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3897 SmallVector<SDValue> NewOps(N->ops());
3898 SDValue Op = N->getOperand(Num: OpNo);
3899 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3900 SDValue NewNode =
3901 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3902
3903 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3904 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3905
3906 return SDValue(); // Signal that we replaced the node ourselves.
3907}
3908
3909SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3910 unsigned OpNo) {
3911 assert(OpNo >= 7);
3912 SmallVector<SDValue> NewOps(N->ops());
3913 SDValue Op = N->getOperand(Num: OpNo);
3914 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3915 SDValue NewNode =
3916 DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VTList: N->getVTList(), Ops: NewOps);
3917
3918 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3919 ReplaceValueWith(From: SDValue(N, ResNum), To: NewNode.getValue(R: ResNum));
3920
3921 return SDValue(); // Signal that we replaced the node ourselves.
3922}
3923