1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SelectionDAGNodes.h"
32#include "llvm/CodeGen/TargetFrameLowering.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/CodeGenTypes/MachineValueType.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Compiler.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/MathExtras.h"
49#include "llvm/Support/raw_ostream.h"
50#include "llvm/Target/TargetMachine.h"
51#include "llvm/Target/TargetOptions.h"
52#include <cassert>
53#include <cstdint>
54#include <tuple>
55#include <utility>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "legalizedag"
60
61namespace {
62
63/// Keeps track of state when getting the sign of a floating-point value as an
64/// integer.
65struct FloatSignAsInt {
66 EVT FloatVT;
67 SDValue Chain;
68 SDValue FloatPtr;
69 SDValue IntPtr;
70 MachinePointerInfo IntPointerInfo;
71 MachinePointerInfo FloatPointerInfo;
72 SDValue IntValue;
73 APInt SignMask;
74 uint8_t SignBit;
75};
76
77//===----------------------------------------------------------------------===//
78/// This takes an arbitrary SelectionDAG as input and
79/// hacks on it until the target machine can handle it. This involves
80/// eliminating value sizes the machine cannot handle (promoting small sizes to
81/// large sizes or splitting up large values into small values) as well as
82/// eliminating operations the machine cannot handle.
83///
84/// This code also does a small amount of optimization and recognition of idioms
85/// as part of its processing. For example, if a target does not support a
86/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
87/// will attempt merge setcc and brc instructions into brcc's.
88class SelectionDAGLegalize {
89 const TargetMachine &TM;
90 const TargetLowering &TLI;
91 SelectionDAG &DAG;
92
93 /// The set of nodes which have already been legalized. We hold a
94 /// reference to it in order to update as necessary on node deletion.
95 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
96
97 /// A set of all the nodes updated during legalization.
98 SmallSetVector<SDNode *, 16> *UpdatedNodes;
99
100 EVT getSetCCResultType(EVT VT) const {
101 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
102 }
103
104 // Libcall insertion helpers.
105
106public:
107 SelectionDAGLegalize(SelectionDAG &DAG,
108 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
109 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
110 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
112
113 /// Legalizes the given operation.
114 void LegalizeOp(SDNode *Node);
115
116private:
117 SDValue OptimizeFloatStore(StoreSDNode *ST);
118
119 void LegalizeLoadOps(SDNode *Node);
120 void LegalizeStoreOps(SDNode *Node);
121
122 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
123
124 /// Return a vector shuffle operation which
125 /// performs the same shuffe in terms of order or result bytes, but on a type
126 /// whose vector element type is narrower than the original shuffle type.
127 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
128 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
129 SDValue N1, SDValue N2,
130 ArrayRef<int> Mask) const;
131
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 TargetLowering::ArgListTy &&Args,
134 bool IsSigned, EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
136
137 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 SmallVectorImpl<SDValue> &Results);
139 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
140 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
141 RTLIB::Libcall Call_F128,
142 RTLIB::Libcall Call_PPCF128,
143 SmallVectorImpl<SDValue> &Results);
144
145 void
146 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
147 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
148 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
149 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
150 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
151 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
152 SmallVectorImpl<SDValue> &Results);
153
154 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
155 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
156 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
157 void ExpandArgFPLibCall(SDNode *Node,
158 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
159 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
160 RTLIB::Libcall Call_PPCF128,
161 SmallVectorImpl<SDValue> &Results);
162 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
163 RTLIB::Libcall CallI64,
164 RTLIB::Libcall CallI128);
165 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
166
167 SDValue ExpandSincosStretLibCall(SDNode *Node) const;
168
169 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
170 const SDLoc &dl);
171 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
172 const SDLoc &dl, SDValue ChainIn);
173 SDValue ExpandBUILD_VECTOR(SDNode *Node);
174 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
175 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
176 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
177 SmallVectorImpl<SDValue> &Results);
178 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
179 SDValue Value) const;
180 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
181 SDValue NewIntValue) const;
182 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
183 SDValue ExpandFABS(SDNode *Node) const;
184 SDValue ExpandFNEG(SDNode *Node) const;
185 SDValue expandLdexp(SDNode *Node) const;
186 SDValue expandFrexp(SDNode *Node) const;
187 SDValue expandModf(SDNode *Node) const;
188
189 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
190 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
191 SmallVectorImpl<SDValue> &Results);
192 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
193 SmallVectorImpl<SDValue> &Results);
194 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
195
196 /// Implements vector reduce operation promotion.
197 ///
198 /// All vector operands are promoted to a vector type with larger element
199 /// type, and the start value is promoted to a larger scalar type. Then the
200 /// result is truncated back to the original scalar type.
201 SDValue PromoteReduction(SDNode *Node);
202
203 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
204
205 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
206 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
207 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
208 SDValue ExpandConcatVectors(SDNode *Node);
209
210 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
211 SDValue ExpandConstant(ConstantSDNode *CP);
212
213 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
214 bool ExpandNode(SDNode *Node);
215 void ConvertNodeToLibcall(SDNode *Node);
216 void PromoteNode(SDNode *Node);
217
218public:
219 // Node replacement helpers
220
221 void ReplacedNode(SDNode *N) {
222 LegalizedNodes.erase(Ptr: N);
223 if (UpdatedNodes)
224 UpdatedNodes->insert(X: N);
225 }
226
227 void ReplaceNode(SDNode *Old, SDNode *New) {
228 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
229 dbgs() << " with: "; New->dump(&DAG));
230
231 assert(Old->getNumValues() == New->getNumValues() &&
232 "Replacing one node with another that produces a different number "
233 "of values!");
234 DAG.ReplaceAllUsesWith(From: Old, To: New);
235 if (UpdatedNodes)
236 UpdatedNodes->insert(X: New);
237 ReplacedNode(N: Old);
238 }
239
240 void ReplaceNode(SDValue Old, SDValue New) {
241 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
242 dbgs() << " with: "; New->dump(&DAG));
243
244 DAG.ReplaceAllUsesWith(From: Old, To: New);
245 if (UpdatedNodes)
246 UpdatedNodes->insert(X: New.getNode());
247 ReplacedNode(N: Old.getNode());
248 }
249
250 void ReplaceNode(SDNode *Old, const SDValue *New) {
251 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
252
253 DAG.ReplaceAllUsesWith(From: Old, To: New);
254 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
255 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
256 New[i]->dump(&DAG));
257 if (UpdatedNodes)
258 UpdatedNodes->insert(X: New[i].getNode());
259 }
260 ReplacedNode(N: Old);
261 }
262
263 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
264 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
265 dbgs() << " with: "; New->dump(&DAG));
266
267 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
268 if (UpdatedNodes)
269 UpdatedNodes->insert(X: New.getNode());
270 ReplacedNode(N: Old.getNode());
271 }
272};
273
274} // end anonymous namespace
275
276// Helper function that generates an MMO that considers the alignment of the
277// stack, and the size of the stack object
278static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
279 MachineFunction &MF,
280 bool isObjectScalable) {
281 auto &MFI = MF.getFrameInfo();
282 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
283 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
284 LocationSize ObjectSize = isObjectScalable
285 ? LocationSize::beforeOrAfterPointer()
286 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
287 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
288 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
289}
290
291/// Return a vector shuffle operation which
292/// performs the same shuffle in terms of order or result bytes, but on a type
293/// whose vector element type is narrower than the original shuffle type.
294/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
295SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
296 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
297 ArrayRef<int> Mask) const {
298 unsigned NumMaskElts = VT.getVectorNumElements();
299 unsigned NumDestElts = NVT.getVectorNumElements();
300 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
301
302 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
303
304 if (NumEltsGrowth == 1)
305 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
306
307 SmallVector<int, 8> NewMask;
308 for (unsigned i = 0; i != NumMaskElts; ++i) {
309 int Idx = Mask[i];
310 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
311 if (Idx < 0)
312 NewMask.push_back(Elt: -1);
313 else
314 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
315 }
316 }
317 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
318 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
319 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
320}
321
322/// Expands the ConstantFP node to an integer constant or
323/// a load from the constant pool.
324SDValue
325SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
326 bool Extend = false;
327 SDLoc dl(CFP);
328
329 // If a FP immediate is precise when represented as a float and if the
330 // target can do an extending load from float to double, we put it into
331 // the constant pool as a float, even if it's is statically typed as a
332 // double. This shrinks FP constants and canonicalizes them for targets where
333 // an FP extending load is the same cost as a normal load (such as on the x87
334 // fp stack or PPC FP unit).
335 EVT VT = CFP->getValueType(ResNo: 0);
336 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
337 if (!UseCP) {
338 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
339 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
340 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
341 }
342
343 APFloat APF = CFP->getValueAPF();
344 EVT OrigVT = VT;
345 EVT SVT = VT;
346
347 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
348 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
349 if (!APF.isSignaling()) {
350 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
351 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
352 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
353 // Only do this if the target has a native EXTLOAD instruction from
354 // smaller type.
355 TLI.isLoadLegal(
356 ValVT: OrigVT, MemVT: SVT,
357 Alignment: Align(DAG.getDataLayout().getPrefTypeAlign(
358 Ty: SVT.getTypeForEVT(Context&: *DAG.getContext()))),
359 AddrSpace: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction())
360 .getAddrSpace(),
361 ExtType: ISD::EXTLOAD, Atomic: false) &&
362 TLI.ShouldShrinkFPConstant(OrigVT)) {
363 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
364 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
365 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
366 VT = SVT;
367 Extend = true;
368 }
369 }
370 }
371
372 SDValue CPIdx =
373 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
374 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
375 if (Extend) {
376 SDValue Result = DAG.getExtLoad(
377 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
378 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
379 Alignment);
380 return Result;
381 }
382 SDValue Result = DAG.getLoad(
383 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
384 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
385 return Result;
386}
387
388/// Expands the Constant node to a load from the constant pool.
389SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
390 SDLoc dl(CP);
391 EVT VT = CP->getValueType(ResNo: 0);
392 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
393 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
394 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
395 SDValue Result = DAG.getLoad(
396 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
397 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
398 return Result;
399}
400
401SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
402 SDValue Vec = Op.getOperand(i: 0);
403 SDValue Val = Op.getOperand(i: 1);
404 SDValue Idx = Op.getOperand(i: 2);
405 SDLoc dl(Op);
406
407 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
408 // SCALAR_TO_VECTOR requires that the type of the value being inserted
409 // match the element type of the vector being created, except for
410 // integers in which case the inserted value can be over width.
411 EVT EltVT = Vec.getValueType().getVectorElementType();
412 if (Val.getValueType() == EltVT ||
413 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
414 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
415 VT: Vec.getValueType(), Operand: Val);
416
417 unsigned NumElts = Vec.getValueType().getVectorNumElements();
418 // We generate a shuffle of InVec and ScVec, so the shuffle mask
419 // should be 0,1,2,3,4,5... with the appropriate element replaced with
420 // elt 0 of the RHS.
421 SmallVector<int, 8> ShufOps;
422 for (unsigned i = 0; i != NumElts; ++i)
423 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
424
425 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
426 }
427 }
428 return ExpandInsertToVectorThroughStack(Op);
429}
430
431SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
432 if (!ISD::isNormalStore(N: ST))
433 return SDValue();
434
435 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
436 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
437 // FIXME: move this to the DAG Combiner! Note that we can't regress due
438 // to phase ordering between legalized code and the dag combiner. This
439 // probably means that we need to integrate dag combiner and legalizer
440 // together.
441 // We generally can't do this one for long doubles.
442 SDValue Chain = ST->getChain();
443 SDValue Ptr = ST->getBasePtr();
444 SDValue Value = ST->getValue();
445 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
446 AAMDNodes AAInfo = ST->getAAInfo();
447 SDLoc dl(ST);
448
449 // Don't optimise TargetConstantFP
450 if (Value.getOpcode() == ISD::TargetConstantFP)
451 return SDValue();
452
453 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
454 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
455 TLI.isTypeLegal(VT: MVT::i32)) {
456 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
457 bitcastToAPInt().zextOrTrunc(width: 32),
458 DL: SDLoc(CFP), VT: MVT::i32);
459 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
460 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
461 }
462
463 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
464 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
465 // If this target supports 64-bit registers, do a single 64-bit store.
466 if (TLI.isTypeLegal(VT: MVT::i64)) {
467 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
468 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
469 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
470 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
471 }
472
473 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
474 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
475 // stores. If the target supports neither 32- nor 64-bits, this
476 // xform is certainly not worth it.
477 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
478 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
479 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
480 if (DAG.getDataLayout().isBigEndian())
481 std::swap(a&: Lo, b&: Hi);
482
483 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
484 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
485 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
486 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
487 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
488 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
489
490 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
491 }
492 }
493 }
494 return SDValue();
495}
496
497void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
498 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
499 SDValue Chain = ST->getChain();
500 SDValue Ptr = ST->getBasePtr();
501 SDLoc dl(Node);
502
503 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
504 AAMDNodes AAInfo = ST->getAAInfo();
505
506 if (!ST->isTruncatingStore()) {
507 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
508 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
509 ReplaceNode(Old: ST, New: OptStore);
510 return;
511 }
512
513 SDValue Value = ST->getValue();
514 MVT VT = Value.getSimpleValueType();
515 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
516 default: llvm_unreachable("This action is not supported yet!");
517 case TargetLowering::Legal: {
518 // If this is an unaligned store and the target doesn't support it,
519 // expand it.
520 EVT MemVT = ST->getMemoryVT();
521 const DataLayout &DL = DAG.getDataLayout();
522 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
523 MMO: *ST->getMemOperand())) {
524 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
525 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
526 ReplaceNode(Old: SDValue(ST, 0), New: Result);
527 } else
528 LLVM_DEBUG(dbgs() << "Legal store\n");
529 break;
530 }
531 case TargetLowering::Custom: {
532 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
533 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
534 if (Res && Res != SDValue(Node, 0))
535 ReplaceNode(Old: SDValue(Node, 0), New: Res);
536 return;
537 }
538 case TargetLowering::Promote: {
539 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
540 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
541 "Can only promote stores to same size type");
542 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
543 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
544 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
545 ReplaceNode(Old: SDValue(Node, 0), New: Result);
546 break;
547 }
548 }
549 return;
550 }
551
552 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
553 SDValue Value = ST->getValue();
554 EVT StVT = ST->getMemoryVT();
555 TypeSize StWidth = StVT.getSizeInBits();
556 TypeSize StSize = StVT.getStoreSizeInBits();
557 auto &DL = DAG.getDataLayout();
558
559 if (StWidth != StSize) {
560 // Promote to a byte-sized store with upper bits zero if not
561 // storing an integral number of bytes. For example, promote
562 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
563 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
564 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
565 SDValue Result =
566 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
567 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
568 ReplaceNode(Old: SDValue(Node, 0), New: Result);
569 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
570 // If not storing a power-of-2 number of bits, expand as two stores.
571 assert(!StVT.isVector() && "Unsupported truncstore!");
572 unsigned StWidthBits = StWidth.getFixedValue();
573 unsigned LogStWidth = Log2_32(Value: StWidthBits);
574 assert(LogStWidth < 32);
575 unsigned RoundWidth = 1 << LogStWidth;
576 assert(RoundWidth < StWidthBits);
577 unsigned ExtraWidth = StWidthBits - RoundWidth;
578 assert(ExtraWidth < RoundWidth);
579 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
580 "Store size not an integral number of bytes!");
581 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
582 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
583 SDValue Lo, Hi;
584 unsigned IncrementSize;
585
586 if (DL.isLittleEndian()) {
587 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
588 // Store the bottom RoundWidth bits.
589 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
590 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
591
592 // Store the remaining ExtraWidth bits.
593 IncrementSize = RoundWidth / 8;
594 Ptr =
595 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
596 Hi = DAG.getNode(
597 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
598 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Value.getValueType(), DL: dl));
599 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
600 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
601 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
602 } else {
603 // Big endian - avoid unaligned stores.
604 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
605 // Store the top RoundWidth bits.
606 Hi = DAG.getNode(
607 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
608 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Value.getValueType(), DL: dl));
609 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
610 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
611
612 // Store the remaining ExtraWidth bits.
613 IncrementSize = RoundWidth / 8;
614 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
615 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
616 VT: Ptr.getValueType()));
617 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
618 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
619 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
620 }
621
622 // The order of the stores doesn't matter.
623 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
624 ReplaceNode(Old: SDValue(Node, 0), New: Result);
625 } else {
626 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT,
627 Alignment: ST->getAlign(), AddrSpace: ST->getAddressSpace())) {
628 default:
629 llvm_unreachable("This action is not supported yet!");
630 case TargetLowering::Legal: {
631 EVT MemVT = ST->getMemoryVT();
632 // If this is an unaligned store and the target doesn't support it,
633 // expand it.
634 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
635 MMO: *ST->getMemOperand())) {
636 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
637 ReplaceNode(Old: SDValue(ST, 0), New: Result);
638 }
639 break;
640 }
641 case TargetLowering::Custom: {
642 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
643 if (Res && Res != SDValue(Node, 0))
644 ReplaceNode(Old: SDValue(Node, 0), New: Res);
645 return;
646 }
647 case TargetLowering::Expand:
648 assert(!StVT.isVector() &&
649 "Vector Stores are handled in LegalizeVectorOps");
650
651 SDValue Result;
652
653 // TRUNCSTORE:i16 i32 -> STORE i16
654 if (TLI.isTypeLegal(VT: StVT)) {
655 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
656 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
657 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
658 } else {
659 // The in-memory type isn't legal. Truncate to the type it would promote
660 // to, and then do a truncstore.
661 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
662 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
663 Operand: Value);
664 Result = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
665 SVT: StVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
666 }
667
668 ReplaceNode(Old: SDValue(Node, 0), New: Result);
669 break;
670 }
671 }
672}
673
674void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
675 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
676 SDValue Chain = LD->getChain(); // The chain.
677 SDValue Ptr = LD->getBasePtr(); // The base pointer.
678 SDValue Value; // The value returned by the load op.
679 SDLoc dl(Node);
680
681 ISD::LoadExtType ExtType = LD->getExtensionType();
682 if (ExtType == ISD::NON_EXTLOAD) {
683 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
684 MVT VT = Node->getSimpleValueType(ResNo: 0);
685 SDValue RVal = SDValue(Node, 0);
686 SDValue RChain = SDValue(Node, 1);
687
688 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
689 default: llvm_unreachable("This action is not supported yet!");
690 case TargetLowering::Legal: {
691 EVT MemVT = LD->getMemoryVT();
692 const DataLayout &DL = DAG.getDataLayout();
693 // If this is an unaligned load and the target doesn't support it,
694 // expand it.
695 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
696 MMO: *LD->getMemOperand())) {
697 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
698 }
699 break;
700 }
701 case TargetLowering::Custom:
702 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
703 RVal = Res;
704 RChain = Res.getValue(R: 1);
705 }
706 break;
707
708 case TargetLowering::Promote: {
709 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
710 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
711 "Can only promote loads to same size type");
712
713 // If the range metadata type does not match the legalized memory
714 // operation type, remove the range metadata.
715 if (const MDNode *MD = LD->getRanges()) {
716 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
717 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
718 !NVT.isInteger())
719 LD->getMemOperand()->clearRanges();
720 }
721 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
722 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
723 RChain = Res.getValue(R: 1);
724 break;
725 }
726 }
727 if (RChain.getNode() != Node) {
728 assert(RVal.getNode() != Node && "Load must be completely replaced");
729 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
730 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
731 if (UpdatedNodes) {
732 UpdatedNodes->insert(X: RVal.getNode());
733 UpdatedNodes->insert(X: RChain.getNode());
734 }
735 ReplacedNode(N: Node);
736 }
737 return;
738 }
739
740 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
741 EVT SrcVT = LD->getMemoryVT();
742 TypeSize SrcWidth = SrcVT.getSizeInBits();
743 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
744 AAMDNodes AAInfo = LD->getAAInfo();
745
746 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
747 // Some targets pretend to have an i1 loading operation, and actually
748 // load an i8. This trick is correct for ZEXTLOAD because the top 7
749 // bits are guaranteed to be zero; it helps the optimizers understand
750 // that these bits are zero. It is also useful for EXTLOAD, since it
751 // tells the optimizers that those bits are undefined. It would be
752 // nice to have an effective generic way of getting these benefits...
753 // Until such a way is found, don't insist on promoting i1 here.
754 (SrcVT != MVT::i1 ||
755 TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1, Alignment: LD->getAlign(),
756 AddrSpace: LD->getAddressSpace(), ExtType,
757 Atomic: false) == TargetLowering::Promote)) {
758 // Promote to a byte-sized load if not loading an integral number of
759 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
760 unsigned NewWidth = SrcVT.getStoreSizeInBits();
761 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
762 SDValue Ch;
763
764 // The extra bits are guaranteed to be zero, since we stored them that
765 // way. A zext load from NVT thus automatically gives zext from SrcVT.
766
767 ISD::LoadExtType NewExtType =
768 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
769
770 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
771 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
772 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
773
774 Ch = Result.getValue(R: 1); // The chain.
775
776 if (ExtType == ISD::SEXTLOAD)
777 // Having the top bits zero doesn't help when sign extending.
778 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
779 VT: Result.getValueType(),
780 N1: Result, N2: DAG.getValueType(SrcVT));
781 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
782 // All the top bits are guaranteed to be zero - inform the optimizers.
783 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
784 VT: Result.getValueType(), N1: Result,
785 N2: DAG.getValueType(SrcVT));
786
787 Value = Result;
788 Chain = Ch;
789 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
790 // If not loading a power-of-2 number of bits, expand as two loads.
791 assert(!SrcVT.isVector() && "Unsupported extload!");
792 unsigned SrcWidthBits = SrcWidth.getFixedValue();
793 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
794 assert(LogSrcWidth < 32);
795 unsigned RoundWidth = 1 << LogSrcWidth;
796 assert(RoundWidth < SrcWidthBits);
797 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
798 assert(ExtraWidth < RoundWidth);
799 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
800 "Load size not an integral number of bytes!");
801 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
802 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
803 SDValue Lo, Hi, Ch;
804 unsigned IncrementSize;
805 auto &DL = DAG.getDataLayout();
806
807 if (DL.isLittleEndian()) {
808 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
809 // Load the bottom RoundWidth bits.
810 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
811 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
812 MMOFlags, AAInfo);
813
814 // Load the remaining ExtraWidth bits.
815 IncrementSize = RoundWidth / 8;
816 Ptr =
817 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
818 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
819 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
820 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
821
822 // Build a factor node to remember that this load is independent of
823 // the other one.
824 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
825 N2: Hi.getValue(R: 1));
826
827 // Move the top bits to the right place.
828 Hi = DAG.getNode(
829 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
830 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Hi.getValueType(), DL: dl));
831
832 // Join the hi and lo parts.
833 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
834 } else {
835 // Big endian - avoid unaligned loads.
836 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
837 // Load the top RoundWidth bits.
838 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
839 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
840 MMOFlags, AAInfo);
841
842 // Load the remaining ExtraWidth bits.
843 IncrementSize = RoundWidth / 8;
844 Ptr =
845 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
846 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
847 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
848 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
849
850 // Build a factor node to remember that this load is independent of
851 // the other one.
852 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
853 N2: Hi.getValue(R: 1));
854
855 // Move the top bits to the right place.
856 Hi = DAG.getNode(
857 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
858 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Hi.getValueType(), DL: dl));
859
860 // Join the hi and lo parts.
861 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
862 }
863
864 Chain = Ch;
865 } else {
866 bool isCustom = false;
867 switch (TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: SrcVT.getSimpleVT(),
868 Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(), ExtType,
869 Atomic: false)) {
870 default:
871 llvm_unreachable("This action is not supported yet!");
872 case TargetLowering::Custom:
873 isCustom = true;
874 [[fallthrough]];
875 case TargetLowering::Legal:
876 Value = SDValue(Node, 0);
877 Chain = SDValue(Node, 1);
878
879 if (isCustom) {
880 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
881 Value = Res;
882 Chain = Res.getValue(R: 1);
883 }
884 } else {
885 // If this is an unaligned load and the target doesn't support it,
886 // expand it.
887 EVT MemVT = LD->getMemoryVT();
888 const DataLayout &DL = DAG.getDataLayout();
889 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
890 MMO: *LD->getMemOperand())) {
891 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
892 }
893 }
894 break;
895
896 case TargetLowering::Expand: {
897 EVT DestVT = Node->getValueType(ResNo: 0);
898 if (!TLI.isLoadLegal(ValVT: DestVT, MemVT: SrcVT, Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(),
899 ExtType: ISD::EXTLOAD, Atomic: false)) {
900 // If the source type is not legal, see if there is a legal extload to
901 // an intermediate type that we can then extend further.
902 EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT());
903 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
904 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
905 TLI.isLoadLegal(ValVT: LoadVT, MemVT: SrcVT, Alignment: LD->getAlign(),
906 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false))) {
907 // If we are loading a legal type, this is a non-extload followed by a
908 // full extend.
909 ISD::LoadExtType MidExtType =
910 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
911
912 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
913 MemVT: SrcVT, MMO: LD->getMemOperand());
914 unsigned ExtendOp =
915 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
916 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
917 Chain = Load.getValue(R: 1);
918 break;
919 }
920
921 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
922 // normal undefined upper bits behavior to allow using an in-reg extend
923 // with the illegal FP type, so load as an integer and do the
924 // from-integer conversion.
925 EVT SVT = SrcVT.getScalarType();
926 if (SVT == MVT::f16 || SVT == MVT::bf16) {
927 EVT ISrcVT = SrcVT.changeTypeToInteger();
928 EVT IDestVT = DestVT.changeTypeToInteger();
929 EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT());
930
931 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
932 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
933 Value =
934 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
935 DL: dl, VT: DestVT, Operand: Result);
936 Chain = Result.getValue(R: 1);
937 break;
938 }
939 }
940
941 assert(!SrcVT.isVector() &&
942 "Vector Loads are handled in LegalizeVectorOps");
943
944 // FIXME: This does not work for vectors on most targets. Sign-
945 // and zero-extend operations are currently folded into extending
946 // loads, whether they are legal or not, and then we end up here
947 // without any support for legalizing them.
948 assert(ExtType != ISD::EXTLOAD &&
949 "EXTLOAD should always be supported!");
950 // Turn the unsupported load into an EXTLOAD followed by an
951 // explicit zero/sign extend inreg.
952 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
953 VT: Node->getValueType(ResNo: 0),
954 Chain, Ptr, MemVT: SrcVT,
955 MMO: LD->getMemOperand());
956 SDValue ValRes;
957 if (ExtType == ISD::SEXTLOAD)
958 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
959 VT: Result.getValueType(),
960 N1: Result, N2: DAG.getValueType(SrcVT));
961 else
962 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
963 Value = ValRes;
964 Chain = Result.getValue(R: 1);
965 break;
966 }
967 }
968 }
969
970 // Since loads produce two values, make sure to remember that we legalized
971 // both of them.
972 if (Chain.getNode() != Node) {
973 assert(Value.getNode() != Node && "Load must be completely replaced");
974 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
975 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
976 if (UpdatedNodes) {
977 UpdatedNodes->insert(X: Value.getNode());
978 UpdatedNodes->insert(X: Chain.getNode());
979 }
980 ReplacedNode(N: Node);
981 }
982}
983
984/// Return a legal replacement for the given operation, with all legal operands.
985void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
986 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
987
988 // Allow illegal target nodes and illegal registers.
989 if (Node->getOpcode() == ISD::TargetConstant ||
990 Node->getOpcode() == ISD::Register)
991 return;
992
993#ifndef NDEBUG
994 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
995 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
996 TargetLowering::TypeLegal &&
997 "Unexpected illegal type!");
998
999 for (const SDValue &Op : Node->op_values())
1000 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
1001 TargetLowering::TypeLegal ||
1002 Op.getOpcode() == ISD::TargetConstant ||
1003 Op.getOpcode() == ISD::Register) &&
1004 "Unexpected illegal type!");
1005#endif
1006
1007 // Figure out the correct action; the way to query this varies by opcode
1008 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1009 bool SimpleFinishLegalizing = true;
1010 switch (Node->getOpcode()) {
1011 case ISD::POISON: {
1012 // TODO: Currently, POISON is being lowered to UNDEF here. However, there is
1013 // an open concern that this transformation may not be ideal, as targets
1014 // should ideally handle POISON directly. Changing this behavior would
1015 // require adding support for POISON in TableGen, which is a large change.
1016 // Additionally, many existing test cases rely on the current behavior
1017 // (e.g., llvm/test/CodeGen/PowerPC/vec_shuffle.ll). A broader discussion
1018 // and incremental changes might be needed to properly support POISON
1019 // without breaking existing targets and tests.
1020 SDValue UndefNode = DAG.getUNDEF(VT: Node->getValueType(ResNo: 0));
1021 ReplaceNode(Old: Node, New: UndefNode.getNode());
1022 return;
1023 }
1024 case ISD::INTRINSIC_W_CHAIN:
1025 case ISD::INTRINSIC_WO_CHAIN:
1026 case ISD::INTRINSIC_VOID:
1027 case ISD::STACKSAVE:
1028 case ISD::STACKADDRESS:
1029 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1030 break;
1031 case ISD::GET_DYNAMIC_AREA_OFFSET:
1032 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1033 VT: Node->getValueType(ResNo: 0));
1034 break;
1035 case ISD::VAARG:
1036 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1037 VT: Node->getValueType(ResNo: 0));
1038 if (Action != TargetLowering::Promote)
1039 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1040 break;
1041 case ISD::SET_FPENV:
1042 case ISD::SET_FPMODE:
1043 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1044 VT: Node->getOperand(Num: 1).getValueType());
1045 break;
1046 case ISD::FP_TO_FP16:
1047 case ISD::FP_TO_BF16:
1048 case ISD::SINT_TO_FP:
1049 case ISD::UINT_TO_FP:
1050 case ISD::EXTRACT_VECTOR_ELT:
1051 case ISD::LROUND:
1052 case ISD::LLROUND:
1053 case ISD::LRINT:
1054 case ISD::LLRINT:
1055 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1056 VT: Node->getOperand(Num: 0).getValueType());
1057 break;
1058 case ISD::STRICT_FP_TO_FP16:
1059 case ISD::STRICT_FP_TO_BF16:
1060 case ISD::STRICT_SINT_TO_FP:
1061 case ISD::STRICT_UINT_TO_FP:
1062 case ISD::STRICT_LRINT:
1063 case ISD::STRICT_LLRINT:
1064 case ISD::STRICT_LROUND:
1065 case ISD::STRICT_LLROUND:
1066 // These pseudo-ops are the same as the other STRICT_ ops except
1067 // they are registered with setOperationAction() using the input type
1068 // instead of the output type.
1069 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1070 VT: Node->getOperand(Num: 1).getValueType());
1071 break;
1072 case ISD::SIGN_EXTEND_INREG: {
1073 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1074 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1075 break;
1076 }
1077 case ISD::ATOMIC_STORE:
1078 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1079 VT: Node->getOperand(Num: 1).getValueType());
1080 break;
1081 case ISD::SELECT_CC:
1082 case ISD::STRICT_FSETCC:
1083 case ISD::STRICT_FSETCCS:
1084 case ISD::SETCC:
1085 case ISD::SETCCCARRY:
1086 case ISD::VP_SETCC:
1087 case ISD::BR_CC: {
1088 unsigned Opc = Node->getOpcode();
1089 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1090 : Opc == ISD::STRICT_FSETCC ? 3
1091 : Opc == ISD::STRICT_FSETCCS ? 3
1092 : Opc == ISD::SETCCCARRY ? 3
1093 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1094 : 1;
1095 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1096 : Opc == ISD::STRICT_FSETCC ? 1
1097 : Opc == ISD::STRICT_FSETCCS ? 1
1098 : 0;
1099 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1100 ISD::CondCode CCCode =
1101 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1102 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1103 if (Action == TargetLowering::Legal) {
1104 if (Node->getOpcode() == ISD::SELECT_CC)
1105 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1106 VT: Node->getValueType(ResNo: 0));
1107 else
1108 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1109 }
1110 break;
1111 }
1112 case ISD::LOAD:
1113 case ISD::STORE:
1114 // FIXME: Model these properly. LOAD and STORE are complicated, and
1115 // STORE expects the unlegalized operand in some cases.
1116 SimpleFinishLegalizing = false;
1117 break;
1118 case ISD::CALLSEQ_START:
1119 case ISD::CALLSEQ_END:
1120 // FIXME: This shouldn't be necessary. These nodes have special properties
1121 // dealing with the recursive nature of legalization. Removing this
1122 // special case should be done as part of making LegalizeDAG non-recursive.
1123 SimpleFinishLegalizing = false;
1124 break;
1125 case ISD::EXTRACT_ELEMENT:
1126 case ISD::GET_ROUNDING:
1127 case ISD::MERGE_VALUES:
1128 case ISD::EH_RETURN:
1129 case ISD::FRAME_TO_ARGS_OFFSET:
1130 case ISD::EH_DWARF_CFA:
1131 case ISD::EH_SJLJ_SETJMP:
1132 case ISD::EH_SJLJ_LONGJMP:
1133 case ISD::EH_SJLJ_SETUP_DISPATCH:
1134 // These operations lie about being legal: when they claim to be legal,
1135 // they should actually be expanded.
1136 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1137 if (Action == TargetLowering::Legal)
1138 Action = TargetLowering::Expand;
1139 break;
1140 case ISD::INIT_TRAMPOLINE:
1141 case ISD::ADJUST_TRAMPOLINE:
1142 case ISD::FRAMEADDR:
1143 case ISD::RETURNADDR:
1144 case ISD::ADDROFRETURNADDR:
1145 case ISD::SPONENTRY:
1146 // These operations lie about being legal: when they claim to be legal,
1147 // they should actually be custom-lowered.
1148 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1149 if (Action == TargetLowering::Legal)
1150 Action = TargetLowering::Custom;
1151 break;
1152 case ISD::CLEAR_CACHE:
1153 // This operation is typically going to be LibCall unless the target wants
1154 // something differrent.
1155 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1156 break;
1157 case ISD::READCYCLECOUNTER:
1158 case ISD::READSTEADYCOUNTER:
1159 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1160 // legalization might have expanded that to several smaller types.
1161 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1162 break;
1163 case ISD::READ_REGISTER:
1164 case ISD::WRITE_REGISTER:
1165 // Named register is legal in the DAG, but blocked by register name
1166 // selection if not implemented by target (to chose the correct register)
1167 // They'll be converted to Copy(To/From)Reg.
1168 Action = TargetLowering::Legal;
1169 break;
1170 case ISD::UBSANTRAP:
1171 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1172 if (Action == TargetLowering::Expand) {
1173 // replace ISD::UBSANTRAP with ISD::TRAP
1174 SDValue NewVal;
1175 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1176 N: Node->getOperand(Num: 0));
1177 ReplaceNode(Old: Node, New: NewVal.getNode());
1178 LegalizeOp(Node: NewVal.getNode());
1179 return;
1180 }
1181 break;
1182 case ISD::DEBUGTRAP:
1183 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1184 if (Action == TargetLowering::Expand) {
1185 // replace ISD::DEBUGTRAP with ISD::TRAP
1186 SDValue NewVal;
1187 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1188 N: Node->getOperand(Num: 0));
1189 ReplaceNode(Old: Node, New: NewVal.getNode());
1190 LegalizeOp(Node: NewVal.getNode());
1191 return;
1192 }
1193 break;
1194 case ISD::SADDSAT:
1195 case ISD::UADDSAT:
1196 case ISD::SSUBSAT:
1197 case ISD::USUBSAT:
1198 case ISD::SSHLSAT:
1199 case ISD::USHLSAT:
1200 case ISD::SCMP:
1201 case ISD::UCMP:
1202 case ISD::FP_TO_SINT_SAT:
1203 case ISD::FP_TO_UINT_SAT:
1204 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1205 break;
1206 case ISD::SMULFIX:
1207 case ISD::SMULFIXSAT:
1208 case ISD::UMULFIX:
1209 case ISD::UMULFIXSAT:
1210 case ISD::SDIVFIX:
1211 case ISD::SDIVFIXSAT:
1212 case ISD::UDIVFIX:
1213 case ISD::UDIVFIXSAT: {
1214 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1215 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1216 VT: Node->getValueType(ResNo: 0), Scale);
1217 break;
1218 }
1219 case ISD::MSCATTER:
1220 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1221 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1222 break;
1223 case ISD::MSTORE:
1224 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1225 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1226 break;
1227 case ISD::VP_SCATTER:
1228 Action = TLI.getOperationAction(
1229 Op: Node->getOpcode(),
1230 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1231 break;
1232 case ISD::VP_STORE:
1233 Action = TLI.getOperationAction(
1234 Op: Node->getOpcode(),
1235 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1236 break;
1237 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1238 Action = TLI.getOperationAction(
1239 Op: Node->getOpcode(),
1240 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1241 break;
1242 case ISD::VECREDUCE_FADD:
1243 case ISD::VECREDUCE_FMUL:
1244 case ISD::VECREDUCE_ADD:
1245 case ISD::VECREDUCE_MUL:
1246 case ISD::VECREDUCE_AND:
1247 case ISD::VECREDUCE_OR:
1248 case ISD::VECREDUCE_XOR:
1249 case ISD::VECREDUCE_SMAX:
1250 case ISD::VECREDUCE_SMIN:
1251 case ISD::VECREDUCE_UMAX:
1252 case ISD::VECREDUCE_UMIN:
1253 case ISD::VECREDUCE_FMAX:
1254 case ISD::VECREDUCE_FMIN:
1255 case ISD::VECREDUCE_FMAXIMUM:
1256 case ISD::VECREDUCE_FMINIMUM:
1257 case ISD::IS_FPCLASS:
1258 Action = TLI.getOperationAction(
1259 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1260 break;
1261 case ISD::VECREDUCE_SEQ_FADD:
1262 case ISD::VECREDUCE_SEQ_FMUL:
1263 case ISD::VP_REDUCE_FADD:
1264 case ISD::VP_REDUCE_FMUL:
1265 case ISD::VP_REDUCE_ADD:
1266 case ISD::VP_REDUCE_MUL:
1267 case ISD::VP_REDUCE_AND:
1268 case ISD::VP_REDUCE_OR:
1269 case ISD::VP_REDUCE_XOR:
1270 case ISD::VP_REDUCE_SMAX:
1271 case ISD::VP_REDUCE_SMIN:
1272 case ISD::VP_REDUCE_UMAX:
1273 case ISD::VP_REDUCE_UMIN:
1274 case ISD::VP_REDUCE_FMAX:
1275 case ISD::VP_REDUCE_FMIN:
1276 case ISD::VP_REDUCE_FMAXIMUM:
1277 case ISD::VP_REDUCE_FMINIMUM:
1278 case ISD::VP_REDUCE_SEQ_FADD:
1279 case ISD::VP_REDUCE_SEQ_FMUL:
1280 Action = TLI.getOperationAction(
1281 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1282 break;
1283 case ISD::CTTZ_ELTS:
1284 case ISD::CTTZ_ELTS_ZERO_POISON:
1285 case ISD::VP_CTTZ_ELTS:
1286 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1287 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1288 VT: Node->getOperand(Num: 0).getValueType());
1289 break;
1290 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1291 Action = TLI.getOperationAction(
1292 Op: Node->getOpcode(),
1293 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1294 break;
1295 default:
1296 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1297 Action = TLI.getCustomOperationAction(Op&: *Node);
1298 } else {
1299 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1300 }
1301 break;
1302 }
1303
1304 if (SimpleFinishLegalizing) {
1305 SDNode *NewNode = Node;
1306 switch (Node->getOpcode()) {
1307 default: break;
1308 case ISD::SHL:
1309 case ISD::SRL:
1310 case ISD::SRA:
1311 case ISD::ROTL:
1312 case ISD::ROTR:
1313 case ISD::SSHLSAT:
1314 case ISD::USHLSAT: {
1315 // Legalizing shifts/rotates requires adjusting the shift amount
1316 // to the appropriate width.
1317 SDValue Op0 = Node->getOperand(Num: 0);
1318 SDValue Op1 = Node->getOperand(Num: 1);
1319 if (!Op1.getValueType().isVector()) {
1320 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1321 // The getShiftAmountOperand() may create a new operand node or
1322 // return the existing one. If new operand is created we need
1323 // to update the parent node.
1324 // Do not try to legalize SAO here! It will be automatically legalized
1325 // in the next round.
1326 if (SAO != Op1)
1327 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1328 }
1329 break;
1330 }
1331 case ISD::FSHL:
1332 case ISD::FSHR:
1333 case ISD::SRL_PARTS:
1334 case ISD::SRA_PARTS:
1335 case ISD::SHL_PARTS: {
1336 // Legalizing shifts/rotates requires adjusting the shift amount
1337 // to the appropriate width.
1338 SDValue Op0 = Node->getOperand(Num: 0);
1339 SDValue Op1 = Node->getOperand(Num: 1);
1340 SDValue Op2 = Node->getOperand(Num: 2);
1341 if (!Op2.getValueType().isVector()) {
1342 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1343 // The getShiftAmountOperand() may create a new operand node or
1344 // return the existing one. If new operand is created we need
1345 // to update the parent node.
1346 if (SAO != Op2)
1347 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1348 }
1349 break;
1350 }
1351 }
1352
1353 if (NewNode != Node) {
1354 ReplaceNode(Old: Node, New: NewNode);
1355 Node = NewNode;
1356 }
1357 switch (Action) {
1358 case TargetLowering::Legal:
1359 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1360 return;
1361 case TargetLowering::Custom:
1362 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1363 // FIXME: The handling for custom lowering with multiple results is
1364 // a complete mess.
1365 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1366 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1367 return;
1368
1369 if (Node->getNumValues() == 1) {
1370 // Verify the new types match the original. Glue is waived because
1371 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1372 assert((Res.getValueType() == Node->getValueType(0) ||
1373 Node->getValueType(0) == MVT::Glue) &&
1374 "Type mismatch for custom legalized operation");
1375 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1376 // We can just directly replace this node with the lowered value.
1377 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1378 return;
1379 }
1380
1381 SmallVector<SDValue, 8> ResultVals;
1382 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1383 // Verify the new types match the original. Glue is waived because
1384 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1385 assert((Res->getValueType(i) == Node->getValueType(i) ||
1386 Node->getValueType(i) == MVT::Glue) &&
1387 "Type mismatch for custom legalized operation");
1388 ResultVals.push_back(Elt: Res.getValue(R: i));
1389 }
1390 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1391 ReplaceNode(Old: Node, New: ResultVals.data());
1392 return;
1393 }
1394 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1395 [[fallthrough]];
1396 case TargetLowering::Expand:
1397 if (ExpandNode(Node))
1398 return;
1399 [[fallthrough]];
1400 case TargetLowering::LibCall:
1401 ConvertNodeToLibcall(Node);
1402 return;
1403 case TargetLowering::Promote:
1404 PromoteNode(Node);
1405 return;
1406 }
1407 }
1408
1409 switch (Node->getOpcode()) {
1410 default:
1411#ifndef NDEBUG
1412 dbgs() << "NODE: ";
1413 Node->dump( &DAG);
1414 dbgs() << "\n";
1415#endif
1416 llvm_unreachable("Do not know how to legalize this operator!");
1417
1418 case ISD::CALLSEQ_START:
1419 case ISD::CALLSEQ_END:
1420 break;
1421 case ISD::LOAD:
1422 return LegalizeLoadOps(Node);
1423 case ISD::STORE:
1424 return LegalizeStoreOps(Node);
1425 }
1426}
1427
1428SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1429 SDValue Vec = Op.getOperand(i: 0);
1430 SDValue Idx = Op.getOperand(i: 1);
1431 SDLoc dl(Op);
1432
1433 // Before we generate a new store to a temporary stack slot, see if there is
1434 // already one that we can use. There often is because when we scalarize
1435 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1436 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1437 // the vector. If all are expanded here, we don't want one store per vector
1438 // element.
1439
1440 // Caches for hasPredecessorHelper
1441 SmallPtrSet<const SDNode *, 32> Visited;
1442 SmallVector<const SDNode *, 16> Worklist;
1443 Visited.insert(Ptr: Op.getNode());
1444 Worklist.push_back(Elt: Idx.getNode());
1445 SDValue StackPtr, Ch;
1446 for (SDNode *User : Vec.getNode()->users()) {
1447 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1448 if (ST->isIndexed() || ST->isTruncatingStore() ||
1449 ST->getValue() != Vec)
1450 continue;
1451
1452 // Make sure that nothing else could have stored into the destination of
1453 // this store.
1454 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1455 continue;
1456
1457 // If the index is dependent on the store we will introduce a cycle when
1458 // creating the load (the load uses the index, and by replacing the chain
1459 // we will make the index dependent on the load). Also, the store might be
1460 // dependent on the extractelement and introduce a cycle when creating
1461 // the load.
1462 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1463 ST->hasPredecessor(N: Op.getNode()))
1464 continue;
1465
1466 StackPtr = ST->getBasePtr();
1467 Ch = SDValue(ST, 0);
1468 break;
1469 }
1470 }
1471
1472 EVT VecVT = Vec.getValueType();
1473
1474 if (!Ch.getNode()) {
1475 // Store the value to a temporary stack slot, then LOAD the returned part.
1476 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1477 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1478 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1479 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1480 }
1481
1482 SDValue NewLoad;
1483 Align ElementAlignment =
1484 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1485 b: DAG.getDataLayout().getPrefTypeAlign(
1486 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1487
1488 if (Op.getValueType().isVector()) {
1489 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1490 SubVecVT: Op.getValueType(), Index: Idx);
1491 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1492 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1493 } else {
1494 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1495 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1496 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1497 Alignment: ElementAlignment);
1498 }
1499
1500 // Replace the chain going out of the store, by the one out of the load.
1501 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1502
1503 // We introduced a cycle though, so update the loads operands, making sure
1504 // to use the original store's chain as an incoming chain.
1505 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1506 NewLoadOperands[0] = Ch;
1507 NewLoad =
1508 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1509 return NewLoad;
1510}
1511
1512SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1513 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1514
1515 SDValue Vec = Op.getOperand(i: 0);
1516 SDValue Part = Op.getOperand(i: 1);
1517 SDValue Idx = Op.getOperand(i: 2);
1518 SDLoc dl(Op);
1519
1520 // Store the value to a temporary stack slot, then LOAD the returned part.
1521 EVT VecVT = Vec.getValueType();
1522 EVT PartVT = Part.getValueType();
1523 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1524 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1525 MachinePointerInfo PtrInfo =
1526 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1527
1528 // First store the whole vector.
1529 Align BaseVecAlignment =
1530 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1531 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1532 Alignment: BaseVecAlignment);
1533
1534 // Freeze the index so we don't poison the clamping code we're about to emit.
1535 Idx = DAG.getFreeze(V: Idx);
1536
1537 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1538 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1539
1540 // Then store the inserted part.
1541 if (PartVT.isVector()) {
1542 SDValue SubStackPtr =
1543 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1544
1545 // Store the subvector.
1546 Ch = DAG.getStore(
1547 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1548 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1549 Alignment: PartAlignment);
1550 } else {
1551 SDValue SubStackPtr =
1552 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1553
1554 // Store the scalar value.
1555 Ch = DAG.getTruncStore(
1556 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1557 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1558 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1559 }
1560
1561 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1562 "ElementAlignment does not match!");
1563
1564 // Finally, load the updated vector.
1565 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1566 Alignment: BaseVecAlignment);
1567}
1568
1569SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1570 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1571 SDLoc DL(Node);
1572 SmallVector<SDValue, 16> Ops;
1573 unsigned NumOperands = Node->getNumOperands();
1574 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1575 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1576 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1577 EVT ElementValueType = TLI.getTypeToTransformTo(
1578 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1579 for (unsigned I = 0; I < NumOperands; ++I) {
1580 SDValue SubOp = Node->getOperand(Num: I);
1581 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1582 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1583 N1: SubOp,
1584 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1585 }
1586 }
1587 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1588}
1589
1590SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1591 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1592 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1593 "Unexpected opcode!");
1594
1595 // We can't handle this case efficiently. Allocate a sufficiently
1596 // aligned object on the stack, store each operand into it, then load
1597 // the result as a vector.
1598 // Create the stack frame object.
1599 EVT VT = Node->getValueType(ResNo: 0);
1600 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1601 : Node->getOperand(Num: 0).getValueType();
1602 SDLoc dl(Node);
1603 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1604 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1605 MachinePointerInfo PtrInfo =
1606 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1607
1608 // Emit a store of each element to the stack slot.
1609 SmallVector<SDValue, 8> Stores;
1610 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1611 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1612
1613 // If the destination vector element type of a BUILD_VECTOR is narrower than
1614 // the source element type, only store the bits necessary.
1615 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1616 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1617
1618 // Store (in the right endianness) the elements to memory.
1619 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1620 // Ignore undef elements.
1621 if (Node->getOperand(Num: i).isUndef()) continue;
1622
1623 unsigned Offset = TypeByteSize*i;
1624
1625 SDValue Idx =
1626 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1627
1628 if (Truncate)
1629 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1630 Val: Node->getOperand(Num: i), Ptr: Idx,
1631 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1632 else
1633 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1634 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1635 }
1636
1637 SDValue StoreChain;
1638 if (!Stores.empty()) // Not all undef elements?
1639 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1640 else
1641 StoreChain = DAG.getEntryNode();
1642
1643 // Result is a load from the stack slot.
1644 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1645}
1646
1647/// Bitcast a floating-point value to an integer value. Only bitcast the part
1648/// containing the sign bit if the target has no integer value capable of
1649/// holding all bits of the floating-point value.
1650void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1651 const SDLoc &DL,
1652 SDValue Value) const {
1653 EVT FloatVT = Value.getValueType();
1654 unsigned NumBits = FloatVT.getScalarSizeInBits();
1655 State.FloatVT = FloatVT;
1656 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1657 // Convert to an integer of the same size.
1658 if (TLI.isTypeLegal(VT: IVT)) {
1659 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1660 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1661 State.SignBit = NumBits - 1;
1662 return;
1663 }
1664
1665 auto &DataLayout = DAG.getDataLayout();
1666 // Store the float to memory, then load the sign part out as an integer.
1667 MVT LoadTy = TLI.getRegisterType(VT: MVT::i8);
1668 // First create a temporary that is aligned for both the load and store.
1669 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1670 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1671 // Then store the float to it.
1672 State.FloatPtr = StackPtr;
1673 MachineFunction &MF = DAG.getMachineFunction();
1674 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1675 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1676 PtrInfo: State.FloatPointerInfo);
1677
1678 SDValue IntPtr;
1679 if (DataLayout.isBigEndian()) {
1680 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1681 // Load out a legal integer with the same sign bit as the float.
1682 IntPtr = StackPtr;
1683 State.IntPointerInfo = State.FloatPointerInfo;
1684 } else {
1685 // Advance the pointer so that the loaded byte will contain the sign bit.
1686 unsigned ByteOffset = (NumBits / 8) - 1;
1687 IntPtr =
1688 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1689 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1690 Offset: ByteOffset);
1691 }
1692
1693 State.IntPtr = IntPtr;
1694 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1695 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1696 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1697 State.SignBit = 7;
1698}
1699
1700/// Replace the integer value produced by getSignAsIntValue() with a new value
1701/// and cast the result back to a floating-point type.
1702SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1703 const SDLoc &DL,
1704 SDValue NewIntValue) const {
1705 if (!State.Chain)
1706 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1707
1708 // Override the part containing the sign bit in the value stored on the stack.
1709 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1710 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1711 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1712 PtrInfo: State.FloatPointerInfo);
1713}
1714
1715SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1716 SDLoc DL(Node);
1717 SDValue Mag = Node->getOperand(Num: 0);
1718 SDValue Sign = Node->getOperand(Num: 1);
1719
1720 // Get sign bit into an integer value.
1721 FloatSignAsInt SignAsInt;
1722 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1723
1724 EVT IntVT = SignAsInt.IntValue.getValueType();
1725 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1726 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1727 N2: SignMask);
1728
1729 // If FABS is legal transform
1730 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1731 EVT FloatVT = Mag.getValueType();
1732 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1733 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1734 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1735 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1736 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1737 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1738 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1739 }
1740
1741 // Transform Mag value to integer, and clear the sign bit.
1742 FloatSignAsInt MagAsInt;
1743 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1744 EVT MagVT = MagAsInt.IntValue.getValueType();
1745 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1746 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1747 N2: ClearSignMask);
1748
1749 // Get the signbit at the right position for MagAsInt.
1750 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1751 EVT ShiftVT = IntVT;
1752 if (SignBit.getScalarValueSizeInBits() <
1753 ClearedSign.getScalarValueSizeInBits()) {
1754 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1755 ShiftVT = MagVT;
1756 }
1757 if (ShiftAmount > 0) {
1758 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1759 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1760 } else if (ShiftAmount < 0) {
1761 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1762 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1763 }
1764 if (SignBit.getScalarValueSizeInBits() >
1765 ClearedSign.getScalarValueSizeInBits()) {
1766 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1767 }
1768
1769 // Store the part with the modified sign and convert back to float.
1770 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1771 Flags: SDNodeFlags::Disjoint);
1772
1773 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1774}
1775
1776SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1777 // Get the sign bit as an integer.
1778 SDLoc DL(Node);
1779 FloatSignAsInt SignAsInt;
1780 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1781 EVT IntVT = SignAsInt.IntValue.getValueType();
1782
1783 // Flip the sign.
1784 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1785 SDValue SignFlip =
1786 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1787
1788 // Convert back to float.
1789 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1790}
1791
1792SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1793 SDLoc DL(Node);
1794 SDValue Value = Node->getOperand(Num: 0);
1795
1796 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1797 EVT FloatVT = Value.getValueType();
1798 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1799 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1800 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1801 }
1802
1803 // Transform value to integer, clear the sign bit and transform back.
1804 FloatSignAsInt ValueAsInt;
1805 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1806 EVT IntVT = ValueAsInt.IntValue.getValueType();
1807 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1808 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1809 N2: ClearSignMask);
1810 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1811}
1812
1813void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1814 SmallVectorImpl<SDValue> &Results) {
1815 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1816 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1817 " not tell us which reg is the stack pointer!");
1818 SDLoc dl(Node);
1819 EVT VT = Node->getValueType(ResNo: 0);
1820 SDValue Tmp1 = SDValue(Node, 0);
1821 SDValue Tmp2 = SDValue(Node, 1);
1822 SDValue Tmp3 = Node->getOperand(Num: 2);
1823 SDValue Chain = Tmp1.getOperand(i: 0);
1824
1825 // Chain the dynamic stack allocation so that it doesn't modify the stack
1826 // pointer when other instructions are using the stack.
1827 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1828
1829 SDValue Size = Tmp2.getOperand(i: 1);
1830 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1831 Chain = SP.getValue(R: 1);
1832 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1833 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1834 unsigned Opc =
1835 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1836 ISD::ADD : ISD::SUB;
1837
1838 Align StackAlign = TFL->getStackAlign();
1839 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1840 if (Alignment > StackAlign)
1841 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1842 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1843 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1844
1845 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1846
1847 Results.push_back(Elt: Tmp1);
1848 Results.push_back(Elt: Tmp2);
1849}
1850
1851/// Emit a store/load combination to the stack. This stores
1852/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1853/// a load from the stack slot to DestVT, extending it if needed.
1854/// The resultant code need not be legal.
1855SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1856 EVT DestVT, const SDLoc &dl) {
1857 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1858}
1859
1860SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1861 EVT DestVT, const SDLoc &dl,
1862 SDValue Chain) {
1863 EVT SrcVT = SrcOp.getValueType();
1864 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1865 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1866
1867 // Don't convert with stack if the load/store is expensive.
1868 if ((SrcVT.bitsGT(VT: SlotVT) && !TLI.isTruncStoreLegalOrCustom(
1869 ValVT: SrcOp.getValueType(), MemVT: SlotVT, Alignment: DestAlign,
1870 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace())) ||
1871 (SlotVT.bitsLT(VT: DestVT) &&
1872 !TLI.isLoadLegalOrCustom(ValVT: DestVT, MemVT: SlotVT, Alignment: DestAlign,
1873 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace(),
1874 ExtType: ISD::EXTLOAD, Atomic: false)))
1875 return SDValue();
1876
1877 // Create the stack frame object.
1878 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1879 Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1880 SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1881
1882 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1883 int SPFI = StackPtrFI->getIndex();
1884 MachinePointerInfo PtrInfo =
1885 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1886
1887 // Emit a store to the stack slot. Use a truncstore if the input value is
1888 // later than DestVT.
1889 SDValue Store;
1890
1891 if (SrcVT.bitsGT(VT: SlotVT))
1892 Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1893 SVT: SlotVT, Alignment: SrcAlign);
1894 else {
1895 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1896 Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1897 }
1898
1899 // Result is a load from the stack slot.
1900 if (SlotVT.bitsEq(VT: DestVT))
1901 return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1902
1903 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1904 return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1905 Alignment: DestAlign);
1906}
1907
1908SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1909 SDLoc dl(Node);
1910 // Create a vector sized/aligned stack slot, store the value to element #0,
1911 // then load the whole vector back out.
1912 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1913
1914 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1915 int SPFI = StackPtrFI->getIndex();
1916
1917 SDValue Ch = DAG.getTruncStore(
1918 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1919 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1920 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1921 return DAG.getLoad(
1922 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1923 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1924}
1925
1926static bool
1927ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1928 const TargetLowering &TLI, SDValue &Res) {
1929 unsigned NumElems = Node->getNumOperands();
1930 SDLoc dl(Node);
1931 EVT VT = Node->getValueType(ResNo: 0);
1932
1933 // Try to group the scalars into pairs, shuffle the pairs together, then
1934 // shuffle the pairs of pairs together, etc. until the vector has
1935 // been built. This will work only if all of the necessary shuffle masks
1936 // are legal.
1937
1938 // We do this in two phases; first to check the legality of the shuffles,
1939 // and next, assuming that all shuffles are legal, to create the new nodes.
1940 for (int Phase = 0; Phase < 2; ++Phase) {
1941 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1942 NewIntermedVals;
1943 for (unsigned i = 0; i < NumElems; ++i) {
1944 SDValue V = Node->getOperand(Num: i);
1945 if (V.isUndef())
1946 continue;
1947
1948 SDValue Vec;
1949 if (Phase)
1950 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1951 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1952 }
1953
1954 while (IntermedVals.size() > 2) {
1955 NewIntermedVals.clear();
1956 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1957 // This vector and the next vector are shuffled together (simply to
1958 // append the one to the other).
1959 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1960
1961 SmallVector<int, 16> FinalIndices;
1962 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1963 IntermedVals[i+1].second.size());
1964
1965 int k = 0;
1966 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1967 ++j, ++k) {
1968 ShuffleVec[k] = j;
1969 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1970 }
1971 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1972 ++j, ++k) {
1973 ShuffleVec[k] = NumElems + j;
1974 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1975 }
1976
1977 SDValue Shuffle;
1978 if (Phase)
1979 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1980 N2: IntermedVals[i+1].first,
1981 Mask: ShuffleVec);
1982 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1983 return false;
1984 NewIntermedVals.push_back(
1985 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1986 }
1987
1988 // If we had an odd number of defined values, then append the last
1989 // element to the array of new vectors.
1990 if ((IntermedVals.size() & 1) != 0)
1991 NewIntermedVals.push_back(Elt: IntermedVals.back());
1992
1993 IntermedVals.swap(RHS&: NewIntermedVals);
1994 }
1995
1996 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1997 "Invalid number of intermediate vectors");
1998 SDValue Vec1 = IntermedVals[0].first;
1999 SDValue Vec2;
2000 if (IntermedVals.size() > 1)
2001 Vec2 = IntermedVals[1].first;
2002 else if (Phase)
2003 Vec2 = DAG.getPOISON(VT);
2004
2005 SmallVector<int, 16> ShuffleVec(NumElems, -1);
2006 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
2007 ShuffleVec[IntermedVals[0].second[i]] = i;
2008 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
2009 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
2010
2011 if (Phase)
2012 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2013 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
2014 return false;
2015 }
2016
2017 return true;
2018}
2019
2020/// Expand a BUILD_VECTOR node on targets that don't
2021/// support the operation, but do support the resultant vector type.
2022SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2023 unsigned NumElems = Node->getNumOperands();
2024 SDValue Value1, Value2;
2025 SDLoc dl(Node);
2026 EVT VT = Node->getValueType(ResNo: 0);
2027 EVT OpVT = Node->getOperand(Num: 0).getValueType();
2028 EVT EltVT = VT.getVectorElementType();
2029
2030 // If the only non-undef value is the low element, turn this into a
2031 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
2032 bool isOnlyLowElement = true;
2033 bool MoreThanTwoValues = false;
2034 bool isConstant = true;
2035 for (unsigned i = 0; i < NumElems; ++i) {
2036 SDValue V = Node->getOperand(Num: i);
2037 if (V.isUndef())
2038 continue;
2039 if (i > 0)
2040 isOnlyLowElement = false;
2041 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2042 isConstant = false;
2043
2044 if (!Value1.getNode()) {
2045 Value1 = V;
2046 } else if (!Value2.getNode()) {
2047 if (V != Value1)
2048 Value2 = V;
2049 } else if (V != Value1 && V != Value2) {
2050 MoreThanTwoValues = true;
2051 }
2052 }
2053
2054 if (!Value1.getNode())
2055 return DAG.getUNDEF(VT);
2056
2057 if (isOnlyLowElement)
2058 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2059
2060 // If all elements are constants, create a load from the constant pool.
2061 if (isConstant) {
2062 SmallVector<Constant*, 16> CV;
2063 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2064 if (ConstantFPSDNode *V =
2065 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2066 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2067 } else if (ConstantSDNode *V =
2068 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2069 if (OpVT==EltVT)
2070 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2071 else {
2072 // If OpVT and EltVT don't match, EltVT is not legal and the
2073 // element values have been promoted/truncated earlier. Undo this;
2074 // we don't want a v16i8 to become a v16i32 for example.
2075 const ConstantInt *CI = V->getConstantIntValue();
2076 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2077 V: CI->getZExtValue(), /*IsSigned=*/false,
2078 /*ImplicitTrunc=*/true));
2079 }
2080 } else {
2081 assert(Node->getOperand(i).isUndef());
2082 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2083 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2084 }
2085 }
2086 Constant *CP = ConstantVector::get(V: CV);
2087 SDValue CPIdx =
2088 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2089 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2090 return DAG.getLoad(
2091 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2092 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2093 Alignment);
2094 }
2095
2096 SmallSet<SDValue, 16> DefinedValues;
2097 for (unsigned i = 0; i < NumElems; ++i) {
2098 if (Node->getOperand(Num: i).isUndef())
2099 continue;
2100 DefinedValues.insert(V: Node->getOperand(Num: i));
2101 }
2102
2103 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2104 if (!MoreThanTwoValues) {
2105 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2106 for (unsigned i = 0; i < NumElems; ++i) {
2107 SDValue V = Node->getOperand(Num: i);
2108 if (V.isUndef())
2109 continue;
2110 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2111 }
2112 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2113 // Get the splatted value into the low element of a vector register.
2114 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2115 SDValue Vec2;
2116 if (Value2.getNode())
2117 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2118 else
2119 Vec2 = DAG.getPOISON(VT);
2120
2121 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2122 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2123 }
2124 } else {
2125 SDValue Res;
2126 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2127 return Res;
2128 }
2129 }
2130
2131 // Otherwise, we can't handle this case efficiently.
2132 return ExpandVectorBuildThroughStack(Node);
2133}
2134
2135SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2136 SDLoc DL(Node);
2137 EVT VT = Node->getValueType(ResNo: 0);
2138 SDValue SplatVal = Node->getOperand(Num: 0);
2139
2140 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2141}
2142
2143// Expand a node into a call to a libcall, returning the value as the first
2144// result and the chain as the second. If the result value does not fit into a
2145// register, return the lo part and set the hi part to the by-reg argument in
2146// the first. If it does fit into a single register, return the result and
2147// leave the Hi part unset.
2148std::pair<SDValue, SDValue>
2149SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2150 TargetLowering::ArgListTy &&Args,
2151 bool IsSigned, EVT RetVT) {
2152 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2153 SDValue Callee;
2154 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2155 if (LCImpl != RTLIB::Unsupported)
2156 Callee = DAG.getExternalSymbol(LCImpl, VT: CodePtrTy);
2157 else {
2158 Callee = DAG.getPOISON(VT: CodePtrTy);
2159 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2160 Node->getOperationName(G: &DAG));
2161 }
2162
2163 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2164
2165 // By default, the input chain to this libcall is the entry node of the
2166 // function. If the libcall is going to be emitted as a tail call then
2167 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2168 // node which is being folded has a non-entry input chain.
2169 SDValue InChain = DAG.getEntryNode();
2170
2171 // isTailCall may be true since the callee does not reference caller stack
2172 // frame. Check if it's in the right position and that the return types match.
2173 SDValue TCChain = InChain;
2174 const Function &F = DAG.getMachineFunction().getFunction();
2175 bool isTailCall =
2176 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2177 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2178 if (isTailCall)
2179 InChain = TCChain;
2180
2181 TargetLowering::CallLoweringInfo CLI(DAG);
2182 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2183 CLI.setDebugLoc(SDLoc(Node))
2184 .setChain(InChain)
2185 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
2186 Target: Callee, ArgsList: std::move(Args))
2187 .setTailCall(isTailCall)
2188 .setSExtResult(signExtend)
2189 .setZExtResult(!signExtend)
2190 .setIsPostTypeLegalization(true);
2191
2192 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2193
2194 if (!CallInfo.second.getNode()) {
2195 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2196 // It's a tailcall, return the chain (which is the DAG root).
2197 return {DAG.getRoot(), DAG.getRoot()};
2198 }
2199
2200 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2201 return CallInfo;
2202}
2203
2204std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2205 bool isSigned) {
2206 TargetLowering::ArgListTy Args;
2207 for (const SDValue &Op : Node->op_values()) {
2208 EVT ArgVT = Op.getValueType();
2209 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2210 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2211 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2212 Entry.IsZExt = !Entry.IsSExt;
2213 Args.push_back(x: Entry);
2214 }
2215
2216 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2217 RetVT: Node->getValueType(ResNo: 0));
2218}
2219
2220void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2221 RTLIB::Libcall LC,
2222 SmallVectorImpl<SDValue> &Results) {
2223 if (LC == RTLIB::UNKNOWN_LIBCALL)
2224 llvm_unreachable("Can't create an unknown libcall!");
2225
2226 if (Node->isStrictFPOpcode()) {
2227 EVT RetVT = Node->getValueType(ResNo: 0);
2228 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2229 TargetLowering::MakeLibCallOptions CallOptions;
2230 CallOptions.IsPostTypeLegalization = true;
2231 // FIXME: This doesn't support tail calls.
2232 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2233 Ops, CallOptions,
2234 dl: SDLoc(Node),
2235 Chain: Node->getOperand(Num: 0));
2236 Results.push_back(Elt: Tmp.first);
2237 Results.push_back(Elt: Tmp.second);
2238 } else {
2239 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2240 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2241 Results.push_back(Elt: Tmp);
2242 }
2243}
2244
2245/// Expand the node to a libcall based on the result type.
2246void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2247 RTLIB::Libcall Call_F32,
2248 RTLIB::Libcall Call_F64,
2249 RTLIB::Libcall Call_F80,
2250 RTLIB::Libcall Call_F128,
2251 RTLIB::Libcall Call_PPCF128,
2252 SmallVectorImpl<SDValue> &Results) {
2253 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0),
2254 Call_F32, Call_F64, Call_F80,
2255 Call_F128, Call_PPCF128);
2256 ExpandFPLibCall(Node, LC, Results);
2257}
2258
2259void SelectionDAGLegalize::ExpandFastFPLibCall(
2260 SDNode *Node, bool IsFast,
2261 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2262 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2263 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2264 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2265 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2266 SmallVectorImpl<SDValue> &Results) {
2267
2268 EVT VT = Node->getSimpleValueType(ResNo: 0);
2269
2270 RTLIB::Libcall LC;
2271
2272 // FIXME: Probably should define fast to respect nan/inf and only be
2273 // approximate functions.
2274
2275 if (IsFast) {
2276 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.first, Call_F64: Call_F64.first, Call_F80: Call_F80.first,
2277 Call_F128: Call_F128.first, Call_PPCF128: Call_PPCF128.first);
2278 }
2279
2280 if (!IsFast || DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
2281 // Fall back if we don't have a fast implementation.
2282 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.second, Call_F64: Call_F64.second,
2283 Call_F80: Call_F80.second, Call_F128: Call_F128.second,
2284 Call_PPCF128: Call_PPCF128.second);
2285 }
2286
2287 ExpandFPLibCall(Node, LC, Results);
2288}
2289
2290SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2291 RTLIB::Libcall Call_I8,
2292 RTLIB::Libcall Call_I16,
2293 RTLIB::Libcall Call_I32,
2294 RTLIB::Libcall Call_I64,
2295 RTLIB::Libcall Call_I128) {
2296 RTLIB::Libcall LC;
2297 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2298 default: llvm_unreachable("Unexpected request for libcall!");
2299 case MVT::i8: LC = Call_I8; break;
2300 case MVT::i16: LC = Call_I16; break;
2301 case MVT::i32: LC = Call_I32; break;
2302 case MVT::i64: LC = Call_I64; break;
2303 case MVT::i128: LC = Call_I128; break;
2304 }
2305 return ExpandLibCall(LC, Node, isSigned).first;
2306}
2307
2308/// Expand the node to a libcall based on first argument type (for instance
2309/// lround and its variant).
2310void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2311 RTLIB::Libcall Call_F32,
2312 RTLIB::Libcall Call_F64,
2313 RTLIB::Libcall Call_F80,
2314 RTLIB::Libcall Call_F128,
2315 RTLIB::Libcall Call_PPCF128,
2316 SmallVectorImpl<SDValue> &Results) {
2317 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2318 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2319 Call_F32, Call_F64, Call_F80,
2320 Call_F128, Call_PPCF128);
2321 ExpandFPLibCall(Node, LC, Results);
2322}
2323
2324SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2325 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2326 RTLIB::Libcall CallI128) {
2327 RTLIB::Libcall LC;
2328 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2329 default:
2330 llvm_unreachable("Unexpected request for libcall!");
2331 case MVT::i32:
2332 LC = CallI32;
2333 break;
2334 case MVT::i64:
2335 LC = CallI64;
2336 break;
2337 case MVT::i128:
2338 LC = CallI128;
2339 break;
2340 }
2341
2342 // Bit-counting libcalls have one unsigned argument and return `int`.
2343 // Note that `int` may be illegal on this target; ExpandLibCall will
2344 // take care of promoting it to a legal type.
2345 SDValue Op = Node->getOperand(Num: 0);
2346 EVT IntVT =
2347 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2348
2349 EVT ArgVT = Op.getValueType();
2350 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2351 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2352 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2353 Arg.IsZExt = !Arg.IsSExt;
2354
2355 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2356 /*IsSigned=*/true, RetVT: IntVT)
2357 .first;
2358
2359 // If ExpandLibCall created a tail call, the result was already
2360 // of the correct type. Otherwise, we need to sign extend it.
2361 if (Res.getValueType() != MVT::Other)
2362 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2363 return Res;
2364}
2365
2366/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2367void
2368SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2369 SmallVectorImpl<SDValue> &Results) {
2370 unsigned Opcode = Node->getOpcode();
2371 bool isSigned = Opcode == ISD::SDIVREM;
2372
2373 RTLIB::Libcall LC;
2374 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2375 default: llvm_unreachable("Unexpected request for libcall!");
2376 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2377 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2378 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2379 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2380 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2381 }
2382
2383 // The input chain to this libcall is the entry node of the function.
2384 // Legalizing the call will automatically add the previous call to the
2385 // dependence.
2386 SDValue InChain = DAG.getEntryNode();
2387
2388 EVT RetVT = Node->getValueType(ResNo: 0);
2389 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2390
2391 TargetLowering::ArgListTy Args;
2392 for (const SDValue &Op : Node->op_values()) {
2393 EVT ArgVT = Op.getValueType();
2394 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2395 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2396 Entry.IsSExt = isSigned;
2397 Entry.IsZExt = !isSigned;
2398 Args.push_back(x: Entry);
2399 }
2400
2401 // Also pass the return address of the remainder.
2402 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2403 TargetLowering::ArgListEntry Entry(
2404 FIPtr, PointerType::getUnqual(C&: RetTy->getContext()));
2405 Entry.IsSExt = isSigned;
2406 Entry.IsZExt = !isSigned;
2407 Args.push_back(x: Entry);
2408
2409 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2410 if (LibcallImpl == RTLIB::Unsupported) {
2411 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2412 Node->getOperationName(G: &DAG));
2413 SDValue Poison = DAG.getPOISON(VT: RetVT);
2414 Results.push_back(Elt: Poison);
2415 Results.push_back(Elt: Poison);
2416 return;
2417 }
2418
2419 SDValue Callee =
2420 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2421
2422 SDLoc dl(Node);
2423 TargetLowering::CallLoweringInfo CLI(DAG);
2424 CLI.setDebugLoc(dl)
2425 .setChain(InChain)
2426 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
2427 ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2428 .setSExtResult(isSigned)
2429 .setZExtResult(!isSigned);
2430
2431 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2432
2433 // Remainder is loaded back from the stack frame.
2434 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
2435 MachinePointerInfo PtrInfo =
2436 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2437
2438 SDValue Rem = DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo);
2439 Results.push_back(Elt: CallInfo.first);
2440 Results.push_back(Elt: Rem);
2441}
2442
2443/// Return true if sincos or __sincos_stret libcall is available.
2444static bool isSinCosLibcallAvailable(SDNode *Node,
2445 const LibcallLoweringInfo &Libcalls) {
2446 MVT::SimpleValueType VT = Node->getSimpleValueType(ResNo: 0).SimpleTy;
2447 return Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS(RetVT: VT)) != RTLIB::Unsupported ||
2448 Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS_STRET(RetVT: VT)) !=
2449 RTLIB::Unsupported;
2450}
2451
2452/// Only issue sincos libcall if both sin and cos are needed.
2453static bool useSinCos(SDNode *Node) {
2454 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2455 ? ISD::FCOS : ISD::FSIN;
2456
2457 SDValue Op0 = Node->getOperand(Num: 0);
2458 for (const SDNode *User : Op0.getNode()->users()) {
2459 if (User == Node)
2460 continue;
2461 // The other user might have been turned into sincos already.
2462 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2463 return true;
2464 }
2465 return false;
2466}
2467
2468SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2469 // For iOS, we want to call an alternative entry point: __sincos_stret,
2470 // which returns the values in two S / D registers.
2471 SDLoc dl(Node);
2472 SDValue Arg = Node->getOperand(Num: 0);
2473 EVT ArgVT = Arg.getValueType();
2474 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(RetVT: ArgVT);
2475 RTLIB::LibcallImpl SincosStret = DAG.getLibcalls().getLibcallImpl(Call: LC);
2476 if (SincosStret == RTLIB::Unsupported)
2477 return SDValue();
2478
2479 /// There are 3 different ABI cases to handle:
2480 /// - Direct return of separate fields in registers
2481 /// - Single return as vector elements
2482 /// - sret struct
2483
2484 const RTLIB::RuntimeLibcallsInfo &CallsInfo = TLI.getRuntimeLibcallsInfo();
2485
2486 const DataLayout &DL = DAG.getDataLayout();
2487
2488 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2489 Ctx&: *DAG.getContext(), TT: TM.getTargetTriple(), DL, LibcallImpl: SincosStret);
2490
2491 Type *SincosStretRetTy = FuncTy->getReturnType();
2492 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(Call: SincosStret);
2493
2494 SDValue Callee =
2495 DAG.getExternalSymbol(LCImpl: SincosStret, VT: TLI.getProgramPointerTy(DL));
2496
2497 TargetLowering::ArgListTy Args;
2498 SDValue SRet;
2499
2500 int FrameIdx;
2501 if (FuncTy->getParamType(i: 0)->isPointerTy()) {
2502 // Uses sret
2503 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2504
2505 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(ArgNo: 0);
2506 Type *StructTy = PtrAttrs.getStructRetType();
2507 const uint64_t ByteSize = DL.getTypeAllocSize(Ty: StructTy);
2508 const Align StackAlign = DL.getPrefTypeAlign(Ty: StructTy);
2509
2510 FrameIdx = MFI.CreateStackObject(Size: ByteSize, Alignment: StackAlign, isSpillSlot: false);
2511 SRet = DAG.getFrameIndex(FI: FrameIdx, VT: TLI.getFrameIndexTy(DL));
2512
2513 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(i: 0));
2514 Entry.IsSRet = true;
2515 Entry.IndirectType = StructTy;
2516 Entry.Alignment = StackAlign;
2517
2518 Args.push_back(x: Entry);
2519 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 1));
2520 } else {
2521 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 0));
2522 }
2523
2524 TargetLowering::CallLoweringInfo CLI(DAG);
2525 CLI.setDebugLoc(dl)
2526 .setChain(DAG.getEntryNode())
2527 .setLibCallee(CC: CallConv, ResultType: SincosStretRetTy, Target: Callee, ArgsList: std::move(Args))
2528 .setIsPostTypeLegalization();
2529
2530 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2531
2532 if (SRet) {
2533 MachinePointerInfo PtrInfo =
2534 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2535 SDValue LoadSin = DAG.getLoad(VT: ArgVT, dl, Chain: CallResult.second, Ptr: SRet, PtrInfo);
2536
2537 TypeSize StoreSize = ArgVT.getStoreSize();
2538
2539 // Address of cos field.
2540 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr: SRet, Offset: StoreSize);
2541 SDValue LoadCos = DAG.getLoad(VT: ArgVT, dl, Chain: LoadSin.getValue(R: 1), Ptr: Add,
2542 PtrInfo: PtrInfo.getWithOffset(O: StoreSize));
2543
2544 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2545 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: LoadSin.getValue(R: 0),
2546 N2: LoadCos.getValue(R: 0));
2547 }
2548
2549 if (!CallResult.first.getValueType().isVector())
2550 return CallResult.first;
2551
2552 SDValue SinVal =
2553 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2554 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2555 SDValue CosVal =
2556 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2557 N2: DAG.getVectorIdxConstant(Val: 1, DL: dl));
2558 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2559 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: SinVal, N2: CosVal);
2560}
2561
2562SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2563 SDLoc dl(Node);
2564 EVT VT = Node->getValueType(ResNo: 0);
2565 SDValue X = Node->getOperand(Num: 0);
2566 SDValue N = Node->getOperand(Num: 1);
2567 EVT ExpVT = N.getValueType();
2568 EVT AsIntVT = VT.changeTypeToInteger();
2569 if (AsIntVT == EVT()) // TODO: How to handle f80?
2570 return SDValue();
2571
2572 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2573 return SDValue();
2574
2575 SDNodeFlags NSW;
2576 NSW.setNoSignedWrap(true);
2577 SDNodeFlags NUW_NSW;
2578 NUW_NSW.setNoUnsignedWrap(true);
2579 NUW_NSW.setNoSignedWrap(true);
2580
2581 EVT SetCCVT =
2582 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2583 const fltSemantics &FltSem = VT.getFltSemantics();
2584
2585 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2586 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2587 const int Precision = APFloat::semanticsPrecision(FltSem);
2588
2589 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2590 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2591
2592 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2593
2594 const APFloat One(FltSem, "1.0");
2595 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2596
2597 // Offset by precision to avoid denormal range.
2598 APFloat ScaleDownK =
2599 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2600
2601 // TODO: Should really introduce control flow and use a block for the >
2602 // MaxExp, < MinExp cases
2603
2604 // First, handle exponents Exp > MaxExp and scale down.
2605 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2606
2607 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2608 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2609 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2610 SDValue DecN1 =
2611 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2612
2613 SDValue ScaleUpTwice =
2614 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2615
2616 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2617 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2618 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2619
2620 SDValue SelectN_Big =
2621 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2622 SDValue SelectX_Big =
2623 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2624
2625 // Now handle exponents Exp < MinExp
2626 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2627
2628 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2629 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2630
2631 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2632
2633 SDValue ClampMinVal =
2634 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2635 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2636 SDValue IncN1 =
2637 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2638
2639 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2640 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2641 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2642
2643 SDValue ScaleDownTwice = DAG.getSetCC(
2644 DL: dl, VT: SetCCVT, LHS: N,
2645 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2646
2647 SDValue SelectN_Small =
2648 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2649 SDValue SelectX_Small =
2650 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2651
2652 // Now combine the two out of range exponent handling cases with the base
2653 // case.
2654 SDValue NewX = DAG.getNode(
2655 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2656 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2657
2658 SDValue NewN = DAG.getNode(
2659 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2660 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2661
2662 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2663
2664 SDValue ExponentShiftAmt =
2665 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2666 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2667
2668 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2669 N2: ExponentShiftAmt, Flags: NUW_NSW);
2670 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2671 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2672}
2673
2674SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2675 SDLoc dl(Node);
2676 SDValue Val = Node->getOperand(Num: 0);
2677 EVT VT = Val.getValueType();
2678 EVT ExpVT = Node->getValueType(ResNo: 1);
2679 EVT AsIntVT = VT.changeTypeToInteger();
2680 if (AsIntVT == EVT()) // TODO: How to handle f80?
2681 return SDValue();
2682
2683 const fltSemantics &FltSem = VT.getFltSemantics();
2684 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2685 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2686 const unsigned BitSize = VT.getScalarSizeInBits();
2687
2688 // TODO: Could introduce control flow and skip over the denormal handling.
2689
2690 // scale_up = fmul value, scalbn(1.0, precision + 1)
2691 // extracted_exp = (bitcast value to uint) >> precision - 1
2692 // biased_exp = extracted_exp + min_exp
2693 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2694 //
2695 // is_denormal = val < smallest_normalized
2696 // computed_fract = is_denormal ? scale_up : extracted_fract
2697 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2698 //
2699 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2700 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2701
2702 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2703 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2704 VT: AsIntVT);
2705
2706 SDValue SmallestNormalizedInt = DAG.getConstant(
2707 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2708 VT: AsIntVT);
2709
2710 // Masks out the exponent bits.
2711 SDValue ExpMask =
2712 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2713
2714 // Mask out the exponent part of the value.
2715 //
2716 // e.g, for f32 FractSignMaskVal = 0x807fffff
2717 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2718 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2719
2720 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2721 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2722
2723 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2724
2725 const APFloat One(FltSem, "1.0");
2726 // Scale a possible denormal input.
2727 // e.g., for f64, 0x1p+54
2728 APFloat ScaleUpKVal =
2729 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2730
2731 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2732 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2733
2734 EVT SetCCVT =
2735 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2736
2737 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2738
2739 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2740
2741 SDValue AddNegSmallestNormal =
2742 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2743 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2744 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2745
2746 SDValue IsDenormal =
2747 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2748
2749 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2750 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2751
2752 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2753 SDValue ScaledSelect =
2754 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2755
2756 SDValue ExpMaskScaled =
2757 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2758
2759 SDValue ScaledValue =
2760 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2761
2762 // Extract the exponent bits.
2763 SDValue ExponentShiftAmt =
2764 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2765 SDValue ShiftedExp =
2766 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2767 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2768
2769 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2770 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2771 SDValue DenormalExpBias =
2772 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2773
2774 SDValue MaskedFractAsInt =
2775 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2776 const APFloat Half(FltSem, "0.5");
2777 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2778 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2779 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2780
2781 SDValue ComputedExp =
2782 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2783
2784 SDValue Result0 =
2785 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2786
2787 SDValue Result1 =
2788 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2789
2790 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2791}
2792
2793SDValue SelectionDAGLegalize::expandModf(SDNode *Node) const {
2794 SDLoc dl(Node);
2795 SDValue Val = Node->getOperand(Num: 0);
2796 EVT VT = Val.getValueType();
2797 SDNodeFlags Flags = Node->getFlags();
2798
2799 SDValue IntPart = DAG.getNode(Opcode: ISD::FTRUNC, DL: dl, VT, Operand: Val, Flags);
2800 SDValue FracPart = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT, N1: Val, N2: IntPart, Flags);
2801
2802 SDValue FracToUse;
2803 if (Flags.hasNoInfs()) {
2804 FracToUse = FracPart;
2805 } else {
2806 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val, Flags);
2807 SDValue Inf =
2808 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
2809 EVT SetCCVT =
2810 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2811 SDValue IsInf = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: Inf, Cond: ISD::SETOEQ);
2812 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: dl, VT);
2813 FracToUse = DAG.getSelect(DL: dl, VT, Cond: IsInf, LHS: Zero, RHS: FracPart);
2814 }
2815
2816 SDValue ResultFrac =
2817 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT, N1: FracToUse, N2: Val, Flags);
2818 return DAG.getMergeValues(Ops: {ResultFrac, IntPart}, dl);
2819}
2820
2821/// This function is responsible for legalizing a
2822/// INT_TO_FP operation of the specified operand when the target requests that
2823/// we expand it. At this point, we know that the result and operand types are
2824/// legal for the target.
2825SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2826 SDValue &Chain) {
2827 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2828 Node->getOpcode() == ISD::SINT_TO_FP);
2829 EVT DestVT = Node->getValueType(ResNo: 0);
2830 SDLoc dl(Node);
2831 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2832 SDValue Op0 = Node->getOperand(Num: OpNo);
2833 EVT SrcVT = Op0.getValueType();
2834
2835 // TODO: Should any fast-math-flags be set for the created nodes?
2836 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2837 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2838 (DestVT.bitsLE(VT: MVT::f64) ||
2839 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2840 : ISD::FP_EXTEND,
2841 VT: DestVT))) {
2842 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2843 "expansion\n");
2844
2845 // Get the stack frame index of a 8 byte buffer.
2846 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2847
2848 SDValue Lo = Op0;
2849 // if signed map to unsigned space
2850 if (isSigned) {
2851 // Invert sign bit (signed to unsigned mapping).
2852 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2853 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2854 }
2855 // Initial hi portion of constructed double.
2856 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2857
2858 // If this a big endian target, swap the lo and high data.
2859 if (DAG.getDataLayout().isBigEndian())
2860 std::swap(a&: Lo, b&: Hi);
2861
2862 SDValue MemChain = DAG.getEntryNode();
2863
2864 // Store the lo of the constructed double.
2865 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2866 PtrInfo: MachinePointerInfo());
2867 // Store the hi of the constructed double.
2868 SDValue HiPtr =
2869 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2870 SDValue Store2 =
2871 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2872 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2873
2874 // load the constructed double
2875 SDValue Load =
2876 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2877 // FP constant to bias correct the final result
2878 SDValue Bias = DAG.getConstantFP(
2879 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2880 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2881 DL: dl, VT: MVT::f64);
2882 // Subtract the bias and get the final result.
2883 SDValue Sub;
2884 SDValue Result;
2885 if (Node->isStrictFPOpcode()) {
2886 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2887 Ops: {Node->getOperand(Num: 0), Load, Bias});
2888 Chain = Sub.getValue(R: 1);
2889 if (DestVT != Sub.getValueType()) {
2890 std::pair<SDValue, SDValue> ResultPair;
2891 ResultPair =
2892 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2893 Result = ResultPair.first;
2894 Chain = ResultPair.second;
2895 }
2896 else
2897 Result = Sub;
2898 } else {
2899 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2900 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2901 }
2902 return Result;
2903 }
2904
2905 if (isSigned)
2906 return SDValue();
2907
2908 // TODO: Generalize this for use with other types.
2909 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2910 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2911 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2912 // For unsigned conversions, convert them to signed conversions using the
2913 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2914 // should be valid for i32->f32 as well.
2915
2916 // More generally this transform should be valid if there are 3 more bits
2917 // in the integer type than the significand. Rounding uses the first bit
2918 // after the width of the significand and the OR of all bits after that. So
2919 // we need to be able to OR the shifted out bit into one of the bits that
2920 // participate in the OR.
2921
2922 // TODO: This really should be implemented using a branch rather than a
2923 // select. We happen to get lucky and machinesink does the right
2924 // thing most of the time. This would be a good candidate for a
2925 // pseudo-op, or, even better, for whole-function isel.
2926 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2927
2928 SDValue SignBitTest = DAG.getSetCC(
2929 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2930
2931 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL: dl);
2932 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2933 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2934 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2935 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2936
2937 SDValue Slow, Fast;
2938 if (Node->isStrictFPOpcode()) {
2939 // In strict mode, we must avoid spurious exceptions, and therefore
2940 // must make sure to only emit a single STRICT_SINT_TO_FP.
2941 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2942 // The STRICT_SINT_TO_FP inherits the exception mode from the
2943 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2944 // never raise any exception.
2945 SDNodeFlags Flags;
2946 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2947 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: {DestVT, MVT::Other},
2948 Ops: {Node->getOperand(Num: 0), InCvt}, Flags);
2949 Flags.setNoFPExcept(true);
2950 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: {DestVT, MVT::Other},
2951 Ops: {Fast.getValue(R: 1), Fast, Fast}, Flags);
2952 Chain = Slow.getValue(R: 1);
2953 } else {
2954 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2955 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2956 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2957 }
2958
2959 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2960 }
2961
2962 // Don't expand it if there isn't cheap fadd.
2963 if (!TLI.isOperationLegalOrCustom(
2964 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2965 return SDValue();
2966
2967 // The following optimization is valid only if every value in SrcVT (when
2968 // treated as signed) is representable in DestVT. Check that the mantissa
2969 // size of DestVT is >= than the number of bits in SrcVT -1.
2970 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2971 SrcVT.getSizeInBits() - 1 &&
2972 "Cannot perform lossless SINT_TO_FP!");
2973
2974 SDValue Tmp1;
2975 if (Node->isStrictFPOpcode()) {
2976 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2977 Ops: { Node->getOperand(Num: 0), Op0 });
2978 } else
2979 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2980
2981 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2982 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2983 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2984 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2985 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2986 Cond: SignSet, LHS: Four, RHS: Zero);
2987
2988 // If the sign bit of the integer is set, the large number will be treated
2989 // as a negative number. To counteract this, the dynamic code adds an
2990 // offset depending on the data type.
2991 uint64_t FF;
2992 switch (SrcVT.getSimpleVT().SimpleTy) {
2993 default:
2994 return SDValue();
2995 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2996 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2997 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2998 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2999 }
3000 if (DAG.getDataLayout().isLittleEndian())
3001 FF <<= 32;
3002 Constant *FudgeFactor = ConstantInt::get(
3003 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
3004
3005 SDValue CPIdx =
3006 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3007 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
3008 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
3009 Alignment = commonAlignment(A: Alignment, Offset: 4);
3010 SDValue FudgeInReg;
3011 if (DestVT == MVT::f32)
3012 FudgeInReg = DAG.getLoad(
3013 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
3014 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
3015 Alignment);
3016 else {
3017 SDValue Load = DAG.getExtLoad(
3018 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
3019 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
3020 Alignment);
3021 HandleSDNode Handle(Load);
3022 LegalizeOp(Node: Load.getNode());
3023 FudgeInReg = Handle.getValue();
3024 }
3025
3026 if (Node->isStrictFPOpcode()) {
3027 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
3028 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
3029 Chain = Result.getValue(R: 1);
3030 return Result;
3031 }
3032
3033 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
3034}
3035
3036/// This function is responsible for legalizing a
3037/// *INT_TO_FP operation of the specified operand when the target requests that
3038/// we promote it. At this point, we know that the result and operand types are
3039/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
3040/// operation that takes a larger input.
3041void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
3042 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
3043 bool IsStrict = N->isStrictFPOpcode();
3044 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
3045 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
3046 EVT DestVT = N->getValueType(ResNo: 0);
3047 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3048 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
3049 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
3050
3051 // First step, figure out the appropriate *INT_TO_FP operation to use.
3052 EVT NewInTy = LegalOp.getValueType();
3053
3054 unsigned OpToUse = 0;
3055
3056 // Scan for the appropriate larger type to use.
3057 while (true) {
3058 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
3059 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3060
3061 // If the target supports SINT_TO_FP of this type, use it.
3062 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
3063 OpToUse = SIntOp;
3064 break;
3065 }
3066 if (IsSigned)
3067 continue;
3068
3069 // If the target supports UINT_TO_FP of this type, use it.
3070 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
3071 OpToUse = UIntOp;
3072 break;
3073 }
3074
3075 // Otherwise, try a larger type.
3076 }
3077
3078 // Okay, we found the operation and type to use. Zero extend our input to the
3079 // desired type then run the operation on it.
3080 if (IsStrict) {
3081 SDValue Res =
3082 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
3083 Ops: {N->getOperand(Num: 0),
3084 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3085 DL: dl, VT: NewInTy, Operand: LegalOp)});
3086 Results.push_back(Elt: Res);
3087 Results.push_back(Elt: Res.getValue(R: 1));
3088 return;
3089 }
3090
3091 Results.push_back(
3092 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
3093 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3094 DL: dl, VT: NewInTy, Operand: LegalOp)));
3095}
3096
3097/// This function is responsible for legalizing a
3098/// FP_TO_*INT operation of the specified operand when the target requests that
3099/// we promote it. At this point, we know that the result and operand types are
3100/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
3101/// operation that returns a larger result.
3102void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3103 SmallVectorImpl<SDValue> &Results) {
3104 bool IsStrict = N->isStrictFPOpcode();
3105 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3106 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3107 EVT DestVT = N->getValueType(ResNo: 0);
3108 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3109 // First step, figure out the appropriate FP_TO*INT operation to use.
3110 EVT NewOutTy = DestVT;
3111
3112 unsigned OpToUse = 0;
3113
3114 // Scan for the appropriate larger type to use.
3115 while (true) {
3116 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
3117 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3118
3119 // A larger signed type can hold all unsigned values of the requested type,
3120 // so using FP_TO_SINT is valid
3121 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
3122 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3123 break;
3124
3125 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
3126 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
3127 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3128 break;
3129
3130 // Otherwise, try a larger type.
3131 }
3132
3133 // Okay, we found the operation and type to use.
3134 SDValue Operation;
3135 if (IsStrict) {
3136 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
3137 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
3138 } else
3139 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
3140
3141 // Truncate the result of the extended FP_TO_*INT operation to the desired
3142 // size.
3143 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
3144 Results.push_back(Elt: Trunc);
3145 if (IsStrict)
3146 Results.push_back(Elt: Operation.getValue(R: 1));
3147}
3148
3149/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
3150/// the result and operand types are legal and there must be a legal
3151/// FP_TO_*INT_SAT operation for a larger result type.
3152SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3153 const SDLoc &dl) {
3154 unsigned Opcode = Node->getOpcode();
3155
3156 // Scan for the appropriate larger type to use.
3157 EVT NewOutTy = Node->getValueType(ResNo: 0);
3158 while (true) {
3159 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
3160 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3161
3162 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
3163 break;
3164 }
3165
3166 // Saturation width is determined by second operand, so we don't have to
3167 // perform any fixup and can directly truncate the result.
3168 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
3169 N2: Node->getOperand(Num: 1));
3170 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
3171}
3172
3173/// Open code the operations for PARITY of the specified operation.
3174SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3175 EVT VT = Op.getValueType();
3176 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3177 unsigned Sz = VT.getScalarSizeInBits();
3178
3179 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3180 SDValue Result;
3181 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3182 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3183 } else {
3184 Result = Op;
3185 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3186 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3187 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3188 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3189 }
3190 }
3191
3192 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3193}
3194
3195SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3196 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3197 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3198 : Node->getOperand(Num: 0).getSimpleValueType();
3199 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3200 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3201 MVT NewScalarVT = NewVecVT.getVectorElementType();
3202
3203 SDLoc DL(Node);
3204 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3205
3206 // FIXME: Support integer.
3207 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3208 "Only FP promotion is supported");
3209
3210 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3211 if (Node->getOperand(Num: j).getValueType().isVector() &&
3212 !(IsVPOpcode &&
3213 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3214 // promote the vector operand.
3215 // FIXME: Support integer.
3216 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3217 "Only FP promotion is supported");
3218 Operands[j] =
3219 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3220 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3221 // promote the initial value.
3222 Operands[j] =
3223 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3224 } else {
3225 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3226 }
3227
3228 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3229 Flags: Node->getFlags());
3230
3231 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3232 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3233 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3234}
3235
3236bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3237 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3238 SmallVector<SDValue, 8> Results;
3239 SDLoc dl(Node);
3240 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3241 bool NeedInvert;
3242 switch (Node->getOpcode()) {
3243 case ISD::ABS:
3244 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3245 Results.push_back(Elt: Tmp1);
3246 break;
3247 case ISD::ABDS:
3248 case ISD::ABDU:
3249 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3250 Results.push_back(Elt: Tmp1);
3251 break;
3252 case ISD::AVGCEILS:
3253 case ISD::AVGCEILU:
3254 case ISD::AVGFLOORS:
3255 case ISD::AVGFLOORU:
3256 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3257 Results.push_back(Elt: Tmp1);
3258 break;
3259 case ISD::CTPOP:
3260 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3261 Results.push_back(Elt: Tmp1);
3262 break;
3263 case ISD::CTLZ:
3264 case ISD::CTLZ_ZERO_UNDEF:
3265 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3266 Results.push_back(Elt: Tmp1);
3267 break;
3268 case ISD::CTLS:
3269 if ((Tmp1 = TLI.expandCTLS(N: Node, DAG)))
3270 Results.push_back(Elt: Tmp1);
3271 break;
3272 case ISD::CTTZ:
3273 case ISD::CTTZ_ZERO_UNDEF:
3274 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3275 Results.push_back(Elt: Tmp1);
3276 break;
3277 case ISD::BITREVERSE:
3278 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3279 Results.push_back(Elt: Tmp1);
3280 break;
3281 case ISD::BSWAP:
3282 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3283 Results.push_back(Elt: Tmp1);
3284 break;
3285 case ISD::PARITY:
3286 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3287 break;
3288 case ISD::FRAMEADDR:
3289 case ISD::RETURNADDR:
3290 case ISD::FRAME_TO_ARGS_OFFSET:
3291 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3292 break;
3293 case ISD::EH_DWARF_CFA: {
3294 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3295 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3296 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3297 VT: CfaArg.getValueType(),
3298 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3299 VT: CfaArg.getValueType()),
3300 N2: CfaArg);
3301 SDValue FA = DAG.getNode(
3302 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3303 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3304 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3305 N1: FA, N2: Offset));
3306 break;
3307 }
3308 case ISD::GET_ROUNDING:
3309 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3310 Results.push_back(Elt: Node->getOperand(Num: 0));
3311 break;
3312 case ISD::EH_RETURN:
3313 case ISD::EH_LABEL:
3314 case ISD::PREFETCH:
3315 case ISD::VAEND:
3316 case ISD::EH_SJLJ_LONGJMP:
3317 // If the target didn't expand these, there's nothing to do, so just
3318 // preserve the chain and be done.
3319 Results.push_back(Elt: Node->getOperand(Num: 0));
3320 break;
3321 case ISD::READCYCLECOUNTER:
3322 case ISD::READSTEADYCOUNTER:
3323 // If the target didn't expand this, just return 'zero' and preserve the
3324 // chain.
3325 Results.append(NumInputs: Node->getNumValues() - 1,
3326 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3327 Results.push_back(Elt: Node->getOperand(Num: 0));
3328 break;
3329 case ISD::EH_SJLJ_SETJMP:
3330 // If the target didn't expand this, just return 'zero' and preserve the
3331 // chain.
3332 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3333 Results.push_back(Elt: Node->getOperand(Num: 0));
3334 break;
3335 case ISD::ATOMIC_LOAD: {
3336 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3337 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3338 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3339 SDValue Swap = DAG.getAtomicCmpSwap(
3340 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3341 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3342 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3343 Results.push_back(Elt: Swap.getValue(R: 0));
3344 Results.push_back(Elt: Swap.getValue(R: 1));
3345 break;
3346 }
3347 case ISD::ATOMIC_STORE: {
3348 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3349 SDValue Swap = DAG.getAtomic(
3350 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3351 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3352 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3353 Results.push_back(Elt: Swap.getValue(R: 1));
3354 break;
3355 }
3356 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3357 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3358 // splits out the success value as a comparison. Expanding the resulting
3359 // ATOMIC_CMP_SWAP will produce a libcall.
3360 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3361 SDValue Res = DAG.getAtomicCmpSwap(
3362 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3363 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3364 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3365
3366 SDValue ExtRes = Res;
3367 SDValue LHS = Res;
3368 SDValue RHS = Node->getOperand(Num: 1);
3369
3370 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3371 EVT OuterType = Node->getValueType(ResNo: 0);
3372 switch (TLI.getExtendForAtomicOps()) {
3373 case ISD::SIGN_EXTEND:
3374 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3375 N2: DAG.getValueType(AtomicType));
3376 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3377 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3378 ExtRes = LHS;
3379 break;
3380 case ISD::ZERO_EXTEND:
3381 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3382 N2: DAG.getValueType(AtomicType));
3383 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3384 ExtRes = LHS;
3385 break;
3386 case ISD::ANY_EXTEND:
3387 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3388 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3389 break;
3390 default:
3391 llvm_unreachable("Invalid atomic op extension");
3392 }
3393
3394 SDValue Success =
3395 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3396
3397 Results.push_back(Elt: ExtRes.getValue(R: 0));
3398 Results.push_back(Elt: Success);
3399 Results.push_back(Elt: Res.getValue(R: 1));
3400 break;
3401 }
3402 case ISD::ATOMIC_LOAD_SUB: {
3403 SDLoc DL(Node);
3404 EVT VT = Node->getValueType(ResNo: 0);
3405 SDValue RHS = Node->getOperand(Num: 2);
3406 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3407 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3408 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3409 RHS = RHS->getOperand(Num: 0);
3410 SDValue NewRHS =
3411 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3412 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3413 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3414 Val: NewRHS, MMO: AN->getMemOperand());
3415 Results.push_back(Elt: Res);
3416 Results.push_back(Elt: Res.getValue(R: 1));
3417 break;
3418 }
3419 case ISD::DYNAMIC_STACKALLOC:
3420 ExpandDYNAMIC_STACKALLOC(Node, Results);
3421 break;
3422 case ISD::MERGE_VALUES:
3423 for (unsigned i = 0; i < Node->getNumValues(); i++)
3424 Results.push_back(Elt: Node->getOperand(Num: i));
3425 break;
3426 case ISD::POISON:
3427 case ISD::UNDEF: {
3428 EVT VT = Node->getValueType(ResNo: 0);
3429 if (VT.isInteger())
3430 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3431 else {
3432 assert(VT.isFloatingPoint() && "Unknown value type!");
3433 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3434 }
3435 break;
3436 }
3437 case ISD::STRICT_FP_ROUND:
3438 // When strict mode is enforced we can't do expansion because it
3439 // does not honor the "strict" properties. Only libcall is allowed.
3440 if (TLI.isStrictFPEnabled())
3441 break;
3442 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3443 // since this operation is more efficient than stack operation.
3444 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3445 VT: Node->getValueType(ResNo: 0))
3446 == TargetLowering::Legal)
3447 break;
3448 // We fall back to use stack operation when the FP_ROUND operation
3449 // isn't available.
3450 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3451 DestVT: Node->getValueType(ResNo: 0), dl,
3452 Chain: Node->getOperand(Num: 0)))) {
3453 ReplaceNode(Old: Node, New: Tmp1.getNode());
3454 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3455 return true;
3456 }
3457 break;
3458 case ISD::FP_ROUND: {
3459 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3460 Results.push_back(Elt: Tmp1);
3461 break;
3462 }
3463
3464 [[fallthrough]];
3465 }
3466 case ISD::BITCAST:
3467 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3468 DestVT: Node->getValueType(ResNo: 0), dl)))
3469 Results.push_back(Elt: Tmp1);
3470 break;
3471 case ISD::STRICT_FP_EXTEND:
3472 // When strict mode is enforced we can't do expansion because it
3473 // does not honor the "strict" properties. Only libcall is allowed.
3474 if (TLI.isStrictFPEnabled())
3475 break;
3476 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3477 // since this operation is more efficient than stack operation.
3478 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3479 VT: Node->getValueType(ResNo: 0))
3480 == TargetLowering::Legal)
3481 break;
3482 // We fall back to use stack operation when the FP_EXTEND operation
3483 // isn't available.
3484 if ((Tmp1 = EmitStackConvert(
3485 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3486 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3487 ReplaceNode(Old: Node, New: Tmp1.getNode());
3488 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3489 return true;
3490 }
3491 break;
3492 case ISD::FP_EXTEND: {
3493 SDValue Op = Node->getOperand(Num: 0);
3494 EVT SrcVT = Op.getValueType();
3495 EVT DstVT = Node->getValueType(ResNo: 0);
3496 if (SrcVT.getScalarType() == MVT::bf16) {
3497 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3498 break;
3499 }
3500
3501 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3502 Results.push_back(Elt: Tmp1);
3503 break;
3504 }
3505 case ISD::BF16_TO_FP: {
3506 // Always expand bf16 to f32 casts, they lower to ext + shift.
3507 //
3508 // Note that the operand of this code can be bf16 or an integer type in case
3509 // bf16 is not supported on the target and was softened.
3510 SDValue Op = Node->getOperand(Num: 0);
3511 if (Op.getValueType() == MVT::bf16) {
3512 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3513 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3514 } else {
3515 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3516 }
3517 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3518 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3519 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3520 // Add fp_extend in case the output is bigger than f32.
3521 if (Node->getValueType(ResNo: 0) != MVT::f32)
3522 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3523 Results.push_back(Elt: Op);
3524 break;
3525 }
3526 case ISD::FP_TO_BF16: {
3527 SDValue Op = Node->getOperand(Num: 0);
3528 if (Op.getValueType() != MVT::f32)
3529 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3530 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3531 // Certain SNaNs will turn into infinities if we do a simple shift right.
3532 if (!DAG.isKnownNeverSNaN(Op)) {
3533 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3534 }
3535 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
3536 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3537 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3538 // The result of this node can be bf16 or an integer type in case bf16 is
3539 // not supported on the target and was softened to i16 for storage.
3540 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3541 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3542 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3543 } else {
3544 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3545 }
3546 Results.push_back(Elt: Op);
3547 break;
3548 }
3549 case ISD::CONVERT_FROM_ARBITRARY_FP: {
3550 // Expand conversion from arbitrary FP format stored in an integer to a
3551 // native IEEE float type using integer bit manipulation.
3552 //
3553 // TODO: currently only conversions from FP4, FP6 and FP8 formats from OCP
3554 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3555 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3556 // Float8E8M0FNU.
3557 EVT DstVT = Node->getValueType(ResNo: 0);
3558
3559 SDValue IntVal = Node->getOperand(Num: 0);
3560 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
3561 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
3562
3563 // Supported source formats.
3564 switch (Sem) {
3565 case APFloatBase::S_Float8E5M2:
3566 case APFloatBase::S_Float8E4M3FN:
3567 case APFloatBase::S_Float6E3M2FN:
3568 case APFloatBase::S_Float6E2M3FN:
3569 case APFloatBase::S_Float4E2M1FN:
3570 break;
3571 default:
3572 DAG.getContext()->emitError(ErrorStr: "CONVERT_FROM_ARBITRARY_FP: not implemented "
3573 "source format (semantics enum " +
3574 Twine(SemEnum) + ")");
3575 Results.push_back(Elt: DAG.getPOISON(VT: DstVT));
3576 break;
3577 }
3578 if (!Results.empty())
3579 break;
3580
3581 const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(S: Sem);
3582
3583 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
3584 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
3585 const unsigned SrcMant = SrcPrecision - 1;
3586 const unsigned SrcExp = SrcBits - SrcMant - 1;
3587 const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
3588
3589 const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;
3590
3591 // Destination format parameters.
3592 const fltSemantics &DstSem = DstVT.getFltSemantics();
3593
3594 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
3595 const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
3596 const unsigned DstExpBits = DstBits - DstMant - 1;
3597 const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
3598 const int DstBias = 1 - DstMinExp;
3599 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
3600
3601 // Work in an integer type matching the destination float width.
3602 // Use zero-extend to preserve the raw bit-pattern.
3603 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstBits);
3604 SDValue Src = DAG.getZExtOrTrunc(Op: IntVal, DL: dl, VT: IntVT);
3605
3606 EVT SetCCVT = getSetCCResultType(VT: IntVT);
3607
3608 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
3609 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
3610
3611 // Extract bit fields.
3612 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
3613 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
3614
3615 SDValue MantField = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Src,
3616 N2: DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT));
3617
3618 SDValue ExpField =
3619 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT,
3620 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
3621 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl)),
3622 N2: DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT));
3623
3624 SDValue SignBit =
3625 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
3626 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl));
3627
3628 // Precompute sign shifted to MSB of destination.
3629 SDValue SignShifted =
3630 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: SignBit,
3631 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl));
3632
3633 // Classify the input value based on compile-time format properties.
3634 SDValue ExpAllOnes = DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT);
3635 SDValue IsExpAllOnes =
3636 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: ExpAllOnes, Cond: ISD::SETEQ);
3637 SDValue IsExpZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: Zero, Cond: ISD::SETEQ);
3638 SDValue IsMantZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETEQ);
3639 SDValue IsMantNonZero =
3640 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETNE);
3641
3642 // NaN detection.
3643 SDValue IsNaN;
3644 if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
3645 // FiniteOnly formats (E2M1FN, E3M2FN, E2M3FN) never produce NaN.
3646 IsNaN = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
3647 } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
3648 // E5M2 produces NaN when exp == all-ones AND mantissa != 0.
3649 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantNonZero);
3650 } else {
3651 // NanOnly + AllOnes (E4M3FN): NaN when all exp and mantissa bits are 1.
3652 assert(SrcSem.nanEncoding == fltNanEncoding::AllOnes);
3653 SDValue MantAllOnes = DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT);
3654 SDValue IsMantAllOnes =
3655 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: MantAllOnes, Cond: ISD::SETEQ);
3656 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantAllOnes);
3657 }
3658
3659 // Inf detection.
3660 SDValue IsInf;
3661 if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
3662 // E5M2: Inf when exp == all-ones AND mantissa == 0.
3663 IsInf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantZero);
3664 } else {
3665 // NanOnly and FiniteOnly formats have no Inf representation.
3666 IsInf = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
3667 }
3668
3669 // Zero detection.
3670 SDValue IsZero = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantZero);
3671
3672 // Denorm detection: exp == 0 AND mant != 0.
3673 SDValue IsDenorm =
3674 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantNonZero);
3675
3676 // Normal value conversion.
3677 // dst_exp = exp_field + (DstBias - SrcBias)
3678 // dst_mant = mant << (DstMant - SrcMant)
3679 const int BiasAdjust = DstBias - SrcBias;
3680 SDValue NormDstExp = DAG.getNode(
3681 Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: ExpField,
3682 N2: DAG.getConstant(Val: APInt(DstBits, BiasAdjust, true), DL: dl, VT: IntVT));
3683
3684 SDValue NormDstMant;
3685 if (DstMant > SrcMant) {
3686 SDValue NormDstMantShift =
3687 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
3688 NormDstMant =
3689 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: MantField, N2: NormDstMantShift);
3690 } else {
3691 NormDstMant = MantField;
3692 }
3693
3694 // Assemble normal result.
3695 SDValue DstMantShift = DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl);
3696 SDValue NormExpShifted =
3697 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: NormDstExp, N2: DstMantShift);
3698 SDValue NormResult = DAG.getNode(
3699 Opcode: ISD::OR, DL: dl, VT: IntVT,
3700 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted),
3701 N2: NormDstMant);
3702
3703 // Denormal value conversion.
3704 // For a denormal source (exp_field == 0, mant != 0), normalize by finding
3705 // the MSB position of mant using CTLZ, then compute the correct
3706 // exponent and mantissa for the destination format.
3707 SDValue DenormResult;
3708 {
3709 const unsigned IntVTBits = DstBits;
3710 SDValue LeadingZeros =
3711 DAG.getNode(Opcode: ISD::CTLZ_ZERO_UNDEF, DL: dl, VT: IntVT, Operand: MantField);
3712
3713 // dst_exp_denorm = (IntVTBits + DstBias - SrcBias - SrcMant) -
3714 // LeadingZeros
3715 const int DenormExpConst =
3716 (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
3717 SDValue DenormDstExp = DAG.getNode(
3718 Opcode: ISD::SUB, DL: dl, VT: IntVT,
3719 N1: DAG.getConstant(Val: APInt(DstBits, DenormExpConst, true), DL: dl, VT: IntVT),
3720 N2: LeadingZeros);
3721
3722 // MSB position of the mantissa (0-indexed from LSB).
3723 SDValue MantMSB =
3724 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT,
3725 N1: DAG.getConstant(Val: IntVTBits - 1, DL: dl, VT: IntVT), N2: LeadingZeros);
3726
3727 // leading_one = 1 << MantMSB
3728 SDValue LeadingOne = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: MantMSB);
3729
3730 // frac = mant XOR leading_one (strip the implicit 1)
3731 SDValue Frac = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: IntVT, N1: MantField, N2: LeadingOne);
3732
3733 // shift_amount = DstMant - MantMSB
3734 // = DstMant - (IntVTBits - 1 - LeadingZeros)
3735 // = LeadingZeros - (IntVTBits - 1 - DstMant)
3736 const unsigned ShiftSub = IntVTBits - 1 - DstMant; // always >= 0
3737 SDValue ShiftAmount = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: LeadingZeros,
3738 N2: DAG.getConstant(Val: ShiftSub, DL: dl, VT: IntVT));
3739
3740 SDValue DenormDstMant =
3741 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: Frac, N2: ShiftAmount);
3742
3743 // Assemble denorm as sign | (denorm_dst_exp << DstMant) | denorm_dst_mant
3744 SDValue DenormExpShifted =
3745 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormDstExp, N2: DstMantShift);
3746 DenormResult = DAG.getNode(
3747 Opcode: ISD::OR, DL: dl, VT: IntVT,
3748 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
3749 N2: DenormDstMant);
3750 }
3751
3752 // Select between normal and denorm paths.
3753 SDValue FiniteResult =
3754 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsDenorm, LHS: DenormResult, RHS: NormResult);
3755
3756 // Build special-value results.
3757 // NaN -> canonical quiet NaN: sign=0, exp=all-ones, qNaN bit set.
3758 // Encoding: (DstExpAllOnes << DstMant) | (1 << (DstMant - 1))
3759 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
3760 SDValue NaNResult =
3761 DAG.getConstant(Val: (DstExpAllOnes << DstMant) | QNaNBit, DL: dl, VT: IntVT);
3762
3763 // Inf -> destination Inf.
3764 // sign | (DstExpAllOnes << DstMant)
3765 SDValue InfResult =
3766 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
3767 N2: DAG.getConstant(Val: DstExpAllOnes << DstMant, DL: dl, VT: IntVT));
3768
3769 // Zero -> signed zero.
3770 // Sign bit only.
3771 SDValue ZeroResult = SignShifted;
3772
3773 // Final selection goes in order: NaN takes priority, then Inf, then Zero.
3774 SDValue Result = FiniteResult;
3775 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
3776 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
3777 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
3778
3779 // Bitcast integer result to destination float type.
3780 Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT, Operand: Result);
3781
3782 Results.push_back(Elt: Result);
3783 break;
3784 }
3785 case ISD::FCANONICALIZE: {
3786 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
3787 Results.push_back(Elt: Mul);
3788 break;
3789 }
3790 case ISD::SIGN_EXTEND_INREG: {
3791 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3792 EVT VT = Node->getValueType(ResNo: 0);
3793
3794 // An in-register sign-extend of a boolean is a negation:
3795 // 'true' (1) sign-extended is -1.
3796 // 'false' (0) sign-extended is 0.
3797 // However, we must mask the high bits of the source operand because the
3798 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3799
3800 // TODO: Do this for vectors too?
3801 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3802 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3803 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3804 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3805 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3806 Results.push_back(Elt: Neg);
3807 break;
3808 }
3809
3810 // NOTE: we could fall back on load/store here too for targets without
3811 // SRA. However, it is doubtful that any exist.
3812 unsigned BitsDiff = VT.getScalarSizeInBits() -
3813 ExtraVT.getScalarSizeInBits();
3814 SDValue ShiftCst = DAG.getShiftAmountConstant(Val: BitsDiff, VT, DL: dl);
3815 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: ShiftCst);
3816 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Tmp1, N2: ShiftCst);
3817 Results.push_back(Elt: Tmp1);
3818 break;
3819 }
3820 case ISD::UINT_TO_FP:
3821 case ISD::STRICT_UINT_TO_FP:
3822 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3823 Results.push_back(Elt: Tmp1);
3824 if (Node->isStrictFPOpcode())
3825 Results.push_back(Elt: Tmp2);
3826 break;
3827 }
3828 [[fallthrough]];
3829 case ISD::SINT_TO_FP:
3830 case ISD::STRICT_SINT_TO_FP:
3831 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3832 Results.push_back(Elt: Tmp1);
3833 if (Node->isStrictFPOpcode())
3834 Results.push_back(Elt: Tmp2);
3835 }
3836 break;
3837 case ISD::FP_TO_SINT:
3838 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3839 Results.push_back(Elt: Tmp1);
3840 break;
3841 case ISD::STRICT_FP_TO_SINT:
3842 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3843 ReplaceNode(Old: Node, New: Tmp1.getNode());
3844 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3845 return true;
3846 }
3847 break;
3848 case ISD::FP_TO_UINT:
3849 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3850 Results.push_back(Elt: Tmp1);
3851 break;
3852 case ISD::STRICT_FP_TO_UINT:
3853 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3854 // Relink the chain.
3855 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3856 // Replace the new UINT result.
3857 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3858 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3859 return true;
3860 }
3861 break;
3862 case ISD::FP_TO_SINT_SAT:
3863 case ISD::FP_TO_UINT_SAT:
3864 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3865 break;
3866 case ISD::LROUND:
3867 case ISD::LLROUND: {
3868 SDValue Arg = Node->getOperand(Num: 0);
3869 EVT ArgVT = Arg.getValueType();
3870 EVT ResVT = Node->getValueType(ResNo: 0);
3871 SDLoc dl(Node);
3872 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3873 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3874 break;
3875 }
3876 case ISD::VAARG:
3877 Results.push_back(Elt: DAG.expandVAArg(Node));
3878 Results.push_back(Elt: Results[0].getValue(R: 1));
3879 break;
3880 case ISD::VACOPY:
3881 Results.push_back(Elt: DAG.expandVACopy(Node));
3882 break;
3883 case ISD::EXTRACT_VECTOR_ELT:
3884 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3885 // This must be an access of the only element. Return it.
3886 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3887 Operand: Node->getOperand(Num: 0));
3888 else
3889 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3890 Results.push_back(Elt: Tmp1);
3891 break;
3892 case ISD::EXTRACT_SUBVECTOR:
3893 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3894 break;
3895 case ISD::INSERT_SUBVECTOR:
3896 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3897 break;
3898 case ISD::CONCAT_VECTORS:
3899 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3900 VectorValueType.isScalableVector() ||
3901 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3902 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3903 else
3904 Results.push_back(Elt: ExpandConcatVectors(Node));
3905 break;
3906 case ISD::SCALAR_TO_VECTOR:
3907 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3908 break;
3909 case ISD::INSERT_VECTOR_ELT:
3910 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3911 break;
3912 case ISD::VECTOR_SHUFFLE: {
3913 SmallVector<int, 32> NewMask;
3914 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3915
3916 EVT VT = Node->getValueType(ResNo: 0);
3917 EVT EltVT = VT.getVectorElementType();
3918 SDValue Op0 = Node->getOperand(Num: 0);
3919 SDValue Op1 = Node->getOperand(Num: 1);
3920 if (!TLI.isTypeLegal(VT: EltVT)) {
3921 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3922
3923 // BUILD_VECTOR operands are allowed to be wider than the element type.
3924 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3925 // it.
3926 if (NewEltVT.bitsLT(VT: EltVT)) {
3927 // Convert shuffle node.
3928 // If original node was v4i64 and the new EltVT is i32,
3929 // cast operands to v8i32 and re-build the mask.
3930
3931 // Calculate new VT, the size of the new VT should be equal to original.
3932 EVT NewVT =
3933 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3934 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3935 assert(NewVT.bitsEq(VT));
3936
3937 // cast operands to new VT
3938 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3939 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3940
3941 // Convert the shuffle mask
3942 unsigned int factor =
3943 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3944
3945 // EltVT gets smaller
3946 assert(factor > 0);
3947
3948 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3949 if (Mask[i] < 0) {
3950 for (unsigned fi = 0; fi < factor; ++fi)
3951 NewMask.push_back(Elt: Mask[i]);
3952 }
3953 else {
3954 for (unsigned fi = 0; fi < factor; ++fi)
3955 NewMask.push_back(Elt: Mask[i]*factor+fi);
3956 }
3957 }
3958 Mask = NewMask;
3959 VT = NewVT;
3960 }
3961 EltVT = NewEltVT;
3962 }
3963 unsigned NumElems = VT.getVectorNumElements();
3964 SmallVector<SDValue, 16> Ops;
3965 for (unsigned i = 0; i != NumElems; ++i) {
3966 if (Mask[i] < 0) {
3967 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3968 continue;
3969 }
3970 unsigned Idx = Mask[i];
3971 if (Idx < NumElems)
3972 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3973 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3974 else
3975 Ops.push_back(
3976 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3977 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3978 }
3979
3980 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3981 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3982 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3983 Results.push_back(Elt: Tmp1);
3984 break;
3985 }
3986 case ISD::VECTOR_SPLICE_LEFT:
3987 case ISD::VECTOR_SPLICE_RIGHT: {
3988 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3989 break;
3990 }
3991 case ISD::VECTOR_DEINTERLEAVE: {
3992 unsigned Factor = Node->getNumOperands();
3993 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
3994 break;
3995 SmallVector<SDValue, 8> Ops(Node->ops());
3996 EVT VecVT = Node->getValueType(ResNo: 0);
3997 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3998 // Deinterleave at Factor/2 so each result contains two factors interleaved:
3999 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
4000 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
4001 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
4002 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
4003 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
4004 Results.resize(N: Factor);
4005 // Deinterleave the 2 factors out:
4006 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
4007 for (unsigned I = 0; I < Factor / 2; I++) {
4008 SDValue Deinterleave =
4009 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
4010 Ops: {L.getValue(R: I), R.getValue(R: I)});
4011 Results[I] = Deinterleave.getValue(R: 0);
4012 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
4013 }
4014 break;
4015 }
4016 case ISD::VECTOR_INTERLEAVE: {
4017 unsigned Factor = Node->getNumOperands();
4018 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
4019 break;
4020 EVT VecVT = Node->getValueType(ResNo: 0);
4021 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
4022 SmallVector<SDValue, 8> LOps, ROps;
4023 // Interleave so we have 2 factors per result:
4024 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
4025 for (unsigned I = 0; I < Factor / 2; I++) {
4026 SDValue Interleave =
4027 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
4028 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
4029 LOps.push_back(Elt: Interleave.getValue(R: 0));
4030 ROps.push_back(Elt: Interleave.getValue(R: 1));
4031 }
4032 // Interleave at Factor/2:
4033 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
4034 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: LOps);
4035 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: ROps);
4036 for (unsigned I = 0; I < Factor / 2; I++)
4037 Results.push_back(Elt: L.getValue(R: I));
4038 for (unsigned I = 0; I < Factor / 2; I++)
4039 Results.push_back(Elt: R.getValue(R: I));
4040 break;
4041 }
4042 case ISD::EXTRACT_ELEMENT: {
4043 EVT OpTy = Node->getOperand(Num: 0).getValueType();
4044 if (Node->getConstantOperandVal(Num: 1)) {
4045 // 1 -> Hi
4046 Tmp1 = DAG.getNode(
4047 Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
4048 N2: DAG.getShiftAmountConstant(Val: OpTy.getSizeInBits() / 2, VT: OpTy, DL: dl));
4049 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
4050 } else {
4051 // 0 -> Lo
4052 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
4053 Operand: Node->getOperand(Num: 0));
4054 }
4055 Results.push_back(Elt: Tmp1);
4056 break;
4057 }
4058 case ISD::STACKADDRESS:
4059 case ISD::STACKSAVE:
4060 // Expand to CopyFromReg if the target set
4061 // StackPointerRegisterToSaveRestore.
4062 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
4063 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
4064 VT: Node->getValueType(ResNo: 0)));
4065 Results.push_back(Elt: Results[0].getValue(R: 1));
4066 } else {
4067 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
4068 Results.push_back(Elt: Node->getOperand(Num: 0));
4069
4070 StringRef IntrinsicName = Node->getOpcode() == ISD::STACKADDRESS
4071 ? "llvm.stackaddress"
4072 : "llvm.stacksave";
4073 DAG.getContext()->diagnose(DI: DiagnosticInfoLegalizationFailure(
4074 Twine(IntrinsicName) + " is not supported on this target.",
4075 DAG.getMachineFunction().getFunction(), dl.getDebugLoc()));
4076 }
4077 break;
4078 case ISD::STACKRESTORE:
4079 // Expand to CopyToReg if the target set
4080 // StackPointerRegisterToSaveRestore.
4081 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
4082 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
4083 N: Node->getOperand(Num: 1)));
4084 } else {
4085 Results.push_back(Elt: Node->getOperand(Num: 0));
4086 }
4087 break;
4088 case ISD::GET_DYNAMIC_AREA_OFFSET:
4089 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
4090 Results.push_back(Elt: Results[0].getValue(R: 0));
4091 break;
4092 case ISD::FCOPYSIGN:
4093 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
4094 break;
4095 case ISD::FNEG:
4096 Results.push_back(Elt: ExpandFNEG(Node));
4097 break;
4098 case ISD::FABS:
4099 Results.push_back(Elt: ExpandFABS(Node));
4100 break;
4101 case ISD::IS_FPCLASS: {
4102 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
4103 if (SDValue Expanded =
4104 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
4105 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
4106 Results.push_back(Elt: Expanded);
4107 break;
4108 }
4109 case ISD::SMIN:
4110 case ISD::SMAX:
4111 case ISD::UMIN:
4112 case ISD::UMAX: {
4113 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
4114 ISD::CondCode Pred;
4115 switch (Node->getOpcode()) {
4116 default: llvm_unreachable("How did we get here?");
4117 case ISD::SMAX: Pred = ISD::SETGT; break;
4118 case ISD::SMIN: Pred = ISD::SETLT; break;
4119 case ISD::UMAX: Pred = ISD::SETUGT; break;
4120 case ISD::UMIN: Pred = ISD::SETULT; break;
4121 }
4122 Tmp1 = Node->getOperand(Num: 0);
4123 Tmp2 = Node->getOperand(Num: 1);
4124 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
4125 Results.push_back(Elt: Tmp1);
4126 break;
4127 }
4128 case ISD::FMINNUM:
4129 case ISD::FMAXNUM: {
4130 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
4131 Results.push_back(Elt: Expanded);
4132 break;
4133 }
4134 case ISD::FMINIMUM:
4135 case ISD::FMAXIMUM: {
4136 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
4137 Results.push_back(Elt: Expanded);
4138 break;
4139 }
4140 case ISD::FMINIMUMNUM:
4141 case ISD::FMAXIMUMNUM: {
4142 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
4143 break;
4144 }
4145 case ISD::FSIN:
4146 case ISD::FCOS: {
4147 EVT VT = Node->getValueType(ResNo: 0);
4148 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
4149 // fcos which share the same operand and both are used.
4150 if ((TLI.isOperationLegal(Op: ISD::FSINCOS, VT) ||
4151 isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls())) &&
4152 useSinCos(Node)) {
4153 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4154 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
4155 if (Node->getOpcode() == ISD::FCOS)
4156 Tmp1 = Tmp1.getValue(R: 1);
4157 Results.push_back(Elt: Tmp1);
4158 }
4159 break;
4160 }
4161 case ISD::FLDEXP:
4162 case ISD::STRICT_FLDEXP: {
4163 EVT VT = Node->getValueType(ResNo: 0);
4164 RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT);
4165 // Use the LibCall instead, it is very likely faster
4166 // FIXME: Use separate LibCall action.
4167 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
4168 break;
4169
4170 if (SDValue Expanded = expandLdexp(Node)) {
4171 Results.push_back(Elt: Expanded);
4172 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
4173 Results.push_back(Elt: Expanded.getValue(R: 1));
4174 }
4175
4176 break;
4177 }
4178 case ISD::FFREXP: {
4179 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0));
4180 // Use the LibCall instead, it is very likely faster
4181 // FIXME: Use separate LibCall action.
4182 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
4183 break;
4184
4185 if (SDValue Expanded = expandFrexp(Node)) {
4186 Results.push_back(Elt: Expanded);
4187 Results.push_back(Elt: Expanded.getValue(R: 1));
4188 }
4189 break;
4190 }
4191 case ISD::FMODF: {
4192 RTLIB::Libcall LC = RTLIB::getMODF(VT: Node->getValueType(ResNo: 0));
4193 // Use the LibCall instead, it is very likely faster
4194 // FIXME: Use separate LibCall action.
4195 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
4196 break;
4197
4198 if (SDValue Expanded = expandModf(Node)) {
4199 Results.push_back(Elt: Expanded);
4200 Results.push_back(Elt: Expanded.getValue(R: 1));
4201 }
4202 break;
4203 }
4204 case ISD::FSINCOS: {
4205 if (isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls()))
4206 break;
4207 EVT VT = Node->getValueType(ResNo: 0);
4208 SDValue Op = Node->getOperand(Num: 0);
4209 SDNodeFlags Flags = Node->getFlags();
4210 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
4211 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
4212 Results.append(IL: {Tmp1, Tmp2});
4213 break;
4214 }
4215 case ISD::FMAD:
4216 llvm_unreachable("Illegal fmad should never be formed");
4217
4218 case ISD::FP16_TO_FP:
4219 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4220 // We can extend to types bigger than f32 in two steps without changing
4221 // the result. Since "f16 -> f32" is much more commonly available, give
4222 // CodeGen the option of emitting that before resorting to a libcall.
4223 SDValue Res =
4224 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
4225 Results.push_back(
4226 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
4227 }
4228 break;
4229 case ISD::STRICT_BF16_TO_FP:
4230 case ISD::STRICT_FP16_TO_FP:
4231 if (Node->getValueType(ResNo: 0) != MVT::f32) {
4232 // We can extend to types bigger than f32 in two steps without changing
4233 // the result. Since "f16 -> f32" is much more commonly available, give
4234 // CodeGen the option of emitting that before resorting to a libcall.
4235 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
4236 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
4237 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
4238 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
4239 Ops: {Res.getValue(R: 1), Res});
4240 Results.push_back(Elt: Res);
4241 Results.push_back(Elt: Res.getValue(R: 1));
4242 }
4243 break;
4244 case ISD::FP_TO_FP16:
4245 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
4246 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
4247 SDValue Op = Node->getOperand(Num: 0);
4248 MVT SVT = Op.getSimpleValueType();
4249 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
4250 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
4251 // Under fastmath, we can expand this node into a fround followed by
4252 // a float-half conversion.
4253 SDValue FloatVal =
4254 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
4255 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
4256 Results.push_back(
4257 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
4258 }
4259 }
4260 break;
4261 case ISD::ConstantFP: {
4262 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
4263 // Check to see if this FP immediate is already legal.
4264 // If this is a legal constant, turn it into a TargetConstantFP node.
4265 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
4266 ForCodeSize: DAG.shouldOptForSize()))
4267 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
4268 break;
4269 }
4270 case ISD::Constant: {
4271 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
4272 Results.push_back(Elt: ExpandConstant(CP));
4273 break;
4274 }
4275 case ISD::FSUB: {
4276 EVT VT = Node->getValueType(ResNo: 0);
4277 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
4278 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
4279 const SDNodeFlags Flags = Node->getFlags();
4280 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
4281 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
4282 Results.push_back(Elt: Tmp1);
4283 }
4284 break;
4285 }
4286 case ISD::SUB: {
4287 EVT VT = Node->getValueType(ResNo: 0);
4288 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
4289 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
4290 "Don't know how to expand this subtraction!");
4291 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
4292 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
4293 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
4294 break;
4295 }
4296 case ISD::UREM:
4297 case ISD::SREM:
4298 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
4299 Results.push_back(Elt: Tmp1);
4300 break;
4301 case ISD::UDIV:
4302 case ISD::SDIV: {
4303 bool isSigned = Node->getOpcode() == ISD::SDIV;
4304 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4305 EVT VT = Node->getValueType(ResNo: 0);
4306 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
4307 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4308 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4309 N2: Node->getOperand(Num: 1));
4310 Results.push_back(Elt: Tmp1);
4311 }
4312 break;
4313 }
4314 case ISD::MULHU:
4315 case ISD::MULHS: {
4316 unsigned ExpandOpcode =
4317 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
4318 EVT VT = Node->getValueType(ResNo: 0);
4319 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4320
4321 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4322 N2: Node->getOperand(Num: 1));
4323 Results.push_back(Elt: Tmp1.getValue(R: 1));
4324 break;
4325 }
4326 case ISD::UMUL_LOHI:
4327 case ISD::SMUL_LOHI: {
4328 SDValue LHS = Node->getOperand(Num: 0);
4329 SDValue RHS = Node->getOperand(Num: 1);
4330 EVT VT = LHS.getValueType();
4331 unsigned MULHOpcode =
4332 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
4333
4334 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
4335 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
4336 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
4337 break;
4338 }
4339
4340 SmallVector<SDValue, 4> Halves;
4341 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4342 assert(TLI.isTypeLegal(HalfType));
4343 if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
4344 HiLoVT: HalfType, DAG,
4345 Kind: TargetLowering::MulExpansionKind::Always)) {
4346 for (unsigned i = 0; i < 2; ++i) {
4347 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
4348 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
4349 SDValue Shift =
4350 DAG.getShiftAmountConstant(Val: HalfType.getScalarSizeInBits(), VT, DL: dl);
4351 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4352 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4353 }
4354 break;
4355 }
4356 break;
4357 }
4358 case ISD::MUL: {
4359 EVT VT = Node->getValueType(ResNo: 0);
4360 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4361 // See if multiply or divide can be lowered using two-result operations.
4362 // We just need the low half of the multiply; try both the signed
4363 // and unsigned forms. If the target supports both SMUL_LOHI and
4364 // UMUL_LOHI, form a preference by checking which forms of plain
4365 // MULH it supports.
4366 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
4367 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
4368 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
4369 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
4370 unsigned OpToUse = 0;
4371 if (HasSMUL_LOHI && !HasMULHS) {
4372 OpToUse = ISD::SMUL_LOHI;
4373 } else if (HasUMUL_LOHI && !HasMULHU) {
4374 OpToUse = ISD::UMUL_LOHI;
4375 } else if (HasSMUL_LOHI) {
4376 OpToUse = ISD::SMUL_LOHI;
4377 } else if (HasUMUL_LOHI) {
4378 OpToUse = ISD::UMUL_LOHI;
4379 }
4380 if (OpToUse) {
4381 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4382 N2: Node->getOperand(Num: 1)));
4383 break;
4384 }
4385
4386 SDValue Lo, Hi;
4387 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4388 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
4389 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
4390 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
4391 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
4392 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
4393 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4394 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
4395 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
4396 SDValue Shift =
4397 DAG.getShiftAmountConstant(Val: HalfType.getSizeInBits(), VT, DL: dl);
4398 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4399 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4400 }
4401 break;
4402 }
4403 case ISD::FSHL:
4404 case ISD::FSHR:
4405 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
4406 Results.push_back(Elt: Expanded);
4407 break;
4408 case ISD::ROTL:
4409 case ISD::ROTR:
4410 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
4411 Results.push_back(Elt: Expanded);
4412 break;
4413 case ISD::CLMUL:
4414 case ISD::CLMULR:
4415 case ISD::CLMULH:
4416 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG))
4417 Results.push_back(Elt: Expanded);
4418 break;
4419 case ISD::SADDSAT:
4420 case ISD::UADDSAT:
4421 case ISD::SSUBSAT:
4422 case ISD::USUBSAT:
4423 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
4424 break;
4425 case ISD::SCMP:
4426 case ISD::UCMP:
4427 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
4428 break;
4429 case ISD::SSHLSAT:
4430 case ISD::USHLSAT:
4431 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
4432 break;
4433 case ISD::SMULFIX:
4434 case ISD::SMULFIXSAT:
4435 case ISD::UMULFIX:
4436 case ISD::UMULFIXSAT:
4437 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
4438 break;
4439 case ISD::SDIVFIX:
4440 case ISD::SDIVFIXSAT:
4441 case ISD::UDIVFIX:
4442 case ISD::UDIVFIXSAT:
4443 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
4444 LHS: Node->getOperand(Num: 0),
4445 RHS: Node->getOperand(Num: 1),
4446 Scale: Node->getConstantOperandVal(Num: 2),
4447 DAG)) {
4448 Results.push_back(Elt: V);
4449 break;
4450 }
4451 // FIXME: We might want to retry here with a wider type if we fail, if that
4452 // type is legal.
4453 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4454 // <= 128 (which is the case for all of the default Embedded-C types),
4455 // we will only get here with types and scales that we could always expand
4456 // if we were allowed to generate libcalls to division functions of illegal
4457 // type. But we cannot do that.
4458 llvm_unreachable("Cannot expand DIVFIX!");
4459 case ISD::UADDO_CARRY:
4460 case ISD::USUBO_CARRY: {
4461 SDValue LHS = Node->getOperand(Num: 0);
4462 SDValue RHS = Node->getOperand(Num: 1);
4463 SDValue Carry = Node->getOperand(Num: 2);
4464
4465 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4466
4467 // Initial add of the 2 operands.
4468 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4469 EVT VT = LHS.getValueType();
4470 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4471
4472 // Initial check for overflow.
4473 EVT CarryType = Node->getValueType(ResNo: 1);
4474 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4475 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4476 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4477
4478 // Add of the sum and the carry.
4479 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4480 SDValue CarryExt =
4481 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4482 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4483
4484 // Second check for overflow. If we are adding, we can only overflow if the
4485 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4486 // If we are subtracting, we can only overflow if the initial sum is 0 and
4487 // the carry is set, resulting in a new sum of all 1s.
4488 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4489 SDValue Overflow2 =
4490 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4491 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4492 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4493 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4494
4495 SDValue ResultCarry =
4496 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4497
4498 Results.push_back(Elt: Sum2);
4499 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4500 break;
4501 }
4502 case ISD::SADDO:
4503 case ISD::SSUBO: {
4504 SDValue Result, Overflow;
4505 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4506 Results.push_back(Elt: Result);
4507 Results.push_back(Elt: Overflow);
4508 break;
4509 }
4510 case ISD::UADDO:
4511 case ISD::USUBO: {
4512 SDValue Result, Overflow;
4513 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4514 Results.push_back(Elt: Result);
4515 Results.push_back(Elt: Overflow);
4516 break;
4517 }
4518 case ISD::UMULO:
4519 case ISD::SMULO: {
4520 SDValue Result, Overflow;
4521 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4522 Results.push_back(Elt: Result);
4523 Results.push_back(Elt: Overflow);
4524 }
4525 break;
4526 }
4527 case ISD::BUILD_PAIR: {
4528 EVT PairTy = Node->getValueType(ResNo: 0);
4529 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4530 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4531 Tmp2 = DAG.getNode(
4532 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4533 N2: DAG.getShiftAmountConstant(Val: PairTy.getSizeInBits() / 2, VT: PairTy, DL: dl));
4534 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4535 break;
4536 }
4537 case ISD::SELECT:
4538 Tmp1 = Node->getOperand(Num: 0);
4539 Tmp2 = Node->getOperand(Num: 1);
4540 Tmp3 = Node->getOperand(Num: 2);
4541 if (Tmp1.getOpcode() == ISD::SETCC) {
4542 Tmp1 = DAG.getSelectCC(
4543 DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), True: Tmp2, False: Tmp3,
4544 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get(), Flags: Node->getFlags());
4545 } else {
4546 Tmp1 =
4547 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4548 True: Tmp2, False: Tmp3, Cond: ISD::SETNE, Flags: Node->getFlags());
4549 }
4550 Results.push_back(Elt: Tmp1);
4551 break;
4552 case ISD::BR_JT: {
4553 SDValue Chain = Node->getOperand(Num: 0);
4554 SDValue Table = Node->getOperand(Num: 1);
4555 SDValue Index = Node->getOperand(Num: 2);
4556 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4557
4558 const DataLayout &TD = DAG.getDataLayout();
4559 EVT PTy = TLI.getPointerTy(DL: TD);
4560
4561 unsigned EntrySize =
4562 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4563
4564 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4565 // This transformation needs to be done here since otherwise the MIPS
4566 // backend will end up emitting a three instruction multiply sequence
4567 // instead of a single shift and MSP430 will call a runtime function.
4568 if (llvm::isPowerOf2_32(Value: EntrySize))
4569 Index = DAG.getNode(
4570 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4571 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4572 else
4573 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4574 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4575 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4576
4577 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4578 SDValue LD = DAG.getExtLoad(
4579 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4580 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4581 Addr = LD;
4582 if (TLI.isJumpTableRelative()) {
4583 // For PIC, the sequence is:
4584 // BRIND(RelocBase + load(Jumptable + index))
4585 // RelocBase can be JumpTable, GOT or some sort of global base.
4586 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4587 Offset: Addr, DL: dl);
4588 }
4589
4590 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4591 Results.push_back(Elt: Tmp1);
4592 break;
4593 }
4594 case ISD::BRCOND:
4595 // Expand brcond's setcc into its constituent parts and create a BR_CC
4596 // Node.
4597 Tmp1 = Node->getOperand(Num: 0);
4598 Tmp2 = Node->getOperand(Num: 1);
4599 if (Tmp2.getOpcode() == ISD::SETCC &&
4600 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4601 VT: Tmp2.getOperand(i: 0).getValueType())) {
4602 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4603 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4604 N5: Node->getOperand(Num: 2));
4605 } else {
4606 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4607 if (Tmp2.isUndef() ||
4608 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4609 Tmp3 = Tmp2;
4610 else
4611 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4612 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4613 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4614 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4615 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4616 N5: Node->getOperand(Num: 2));
4617 }
4618 Results.push_back(Elt: Tmp1);
4619 break;
4620 case ISD::SETCC:
4621 case ISD::VP_SETCC:
4622 case ISD::STRICT_FSETCC:
4623 case ISD::STRICT_FSETCCS: {
4624 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4625 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4626 Node->getOpcode() == ISD::STRICT_FSETCCS;
4627 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4628 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4629 unsigned Offset = IsStrict ? 1 : 0;
4630 Tmp1 = Node->getOperand(Num: 0 + Offset);
4631 Tmp2 = Node->getOperand(Num: 1 + Offset);
4632 Tmp3 = Node->getOperand(Num: 2 + Offset);
4633 SDValue Mask, EVL;
4634 if (IsVP) {
4635 Mask = Node->getOperand(Num: 3 + Offset);
4636 EVL = Node->getOperand(Num: 4 + Offset);
4637 }
4638 bool Legalized = TLI.LegalizeSetCCCondCode(
4639 DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl,
4640 Chain, IsSignaling);
4641
4642 if (Legalized) {
4643 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4644 // condition code, create a new SETCC node.
4645 if (Tmp3.getNode()) {
4646 if (IsStrict) {
4647 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4648 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4649 Chain = Tmp1.getValue(R: 1);
4650 } else if (IsVP) {
4651 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0),
4652 Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags());
4653 } else {
4654 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4655 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4656 }
4657 }
4658
4659 // If we expanded the SETCC by inverting the condition code, then wrap
4660 // the existing SETCC in a NOT to restore the intended condition.
4661 if (NeedInvert) {
4662 if (!IsVP)
4663 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4664 else
4665 Tmp1 =
4666 DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0));
4667 }
4668
4669 Results.push_back(Elt: Tmp1);
4670 if (IsStrict)
4671 Results.push_back(Elt: Chain);
4672
4673 break;
4674 }
4675
4676 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4677 // understand if this code is useful for strict nodes.
4678 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4679
4680 // Otherwise, SETCC for the given comparison type must be completely
4681 // illegal; expand it into a SELECT_CC.
4682 // FIXME: This drops the mask/evl for VP_SETCC.
4683 EVT VT = Node->getValueType(ResNo: 0);
4684 EVT Tmp1VT = Tmp1.getValueType();
4685 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4686 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4687 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3,
4688 Flags: Node->getFlags());
4689 Results.push_back(Elt: Tmp1);
4690 break;
4691 }
4692 case ISD::SELECT_CC: {
4693 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4694 Tmp1 = Node->getOperand(Num: 0); // LHS
4695 Tmp2 = Node->getOperand(Num: 1); // RHS
4696 Tmp3 = Node->getOperand(Num: 2); // True
4697 Tmp4 = Node->getOperand(Num: 3); // False
4698 EVT VT = Node->getValueType(ResNo: 0);
4699 SDValue Chain;
4700 SDValue CC = Node->getOperand(Num: 4);
4701 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4702
4703 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4704 // If the condition code is legal, then we need to expand this
4705 // node using SETCC and SELECT.
4706 EVT CmpVT = Tmp1.getValueType();
4707 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4708 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4709 "expanded.");
4710 EVT CCVT = getSetCCResultType(VT: CmpVT);
4711 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4712 Results.push_back(
4713 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4714 break;
4715 }
4716
4717 // SELECT_CC is legal, so the condition code must not be.
4718 bool Legalized = false;
4719 // Try to legalize by inverting the condition. This is for targets that
4720 // might support an ordered version of a condition, but not the unordered
4721 // version (or vice versa).
4722 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4723 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4724 // Use the new condition code and swap true and false
4725 Legalized = true;
4726 Tmp1 =
4727 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC, Flags: Node->getFlags());
4728 } else {
4729 // If The inverse is not legal, then try to swap the arguments using
4730 // the inverse condition code.
4731 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4732 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4733 // The swapped inverse condition is legal, so swap true and false,
4734 // lhs and rhs.
4735 Legalized = true;
4736 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC,
4737 Flags: Node->getFlags());
4738 }
4739 }
4740
4741 if (!Legalized) {
4742 Legalized = TLI.LegalizeSetCCCondCode(
4743 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4744 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4745
4746 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4747
4748 // If we expanded the SETCC by inverting the condition code, then swap
4749 // the True/False operands to match.
4750 if (NeedInvert)
4751 std::swap(a&: Tmp3, b&: Tmp4);
4752
4753 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4754 // condition code, create a new SELECT_CC node.
4755 if (CC.getNode()) {
4756 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4757 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4758 } else {
4759 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4760 CC = DAG.getCondCode(Cond: ISD::SETNE);
4761 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4762 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4763 }
4764 }
4765 Results.push_back(Elt: Tmp1);
4766 break;
4767 }
4768 case ISD::BR_CC: {
4769 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4770 SDValue Chain;
4771 Tmp1 = Node->getOperand(Num: 0); // Chain
4772 Tmp2 = Node->getOperand(Num: 2); // LHS
4773 Tmp3 = Node->getOperand(Num: 3); // RHS
4774 Tmp4 = Node->getOperand(Num: 1); // CC
4775
4776 bool Legalized = TLI.LegalizeSetCCCondCode(
4777 DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4,
4778 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4779 (void)Legalized;
4780 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4781
4782 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4783 // node.
4784 if (Tmp4.getNode()) {
4785 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4786
4787 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4788 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4789 } else {
4790 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4791 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4792 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4793 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4794 }
4795 Results.push_back(Elt: Tmp1);
4796 break;
4797 }
4798 case ISD::BUILD_VECTOR:
4799 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4800 break;
4801 case ISD::SPLAT_VECTOR:
4802 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4803 break;
4804 case ISD::SRA:
4805 case ISD::SRL:
4806 case ISD::SHL: {
4807 // Scalarize vector SRA/SRL/SHL.
4808 EVT VT = Node->getValueType(ResNo: 0);
4809 assert(VT.isVector() && "Unable to legalize non-vector shift");
4810 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4811 unsigned NumElem = VT.getVectorNumElements();
4812
4813 SmallVector<SDValue, 8> Scalars;
4814 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4815 SDValue Ex =
4816 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4817 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4818 SDValue Sh =
4819 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4820 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4821 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4822 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4823 }
4824
4825 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4826 Results.push_back(Elt: Result);
4827 break;
4828 }
4829 case ISD::VECREDUCE_FADD:
4830 case ISD::VECREDUCE_FMUL:
4831 case ISD::VECREDUCE_ADD:
4832 case ISD::VECREDUCE_MUL:
4833 case ISD::VECREDUCE_AND:
4834 case ISD::VECREDUCE_OR:
4835 case ISD::VECREDUCE_XOR:
4836 case ISD::VECREDUCE_SMAX:
4837 case ISD::VECREDUCE_SMIN:
4838 case ISD::VECREDUCE_UMAX:
4839 case ISD::VECREDUCE_UMIN:
4840 case ISD::VECREDUCE_FMAX:
4841 case ISD::VECREDUCE_FMIN:
4842 case ISD::VECREDUCE_FMAXIMUM:
4843 case ISD::VECREDUCE_FMINIMUM:
4844 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4845 break;
4846 case ISD::VP_CTTZ_ELTS:
4847 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4848 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4849 break;
4850 case ISD::CLEAR_CACHE:
4851 // The default expansion of llvm.clear_cache is simply a no-op for those
4852 // targets where it is not needed.
4853 Results.push_back(Elt: Node->getOperand(Num: 0));
4854 break;
4855 case ISD::LRINT:
4856 case ISD::LLRINT: {
4857 SDValue Arg = Node->getOperand(Num: 0);
4858 EVT ArgVT = Arg.getValueType();
4859 EVT ResVT = Node->getValueType(ResNo: 0);
4860 SDLoc dl(Node);
4861 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL: dl, VT: ArgVT, Operand: Arg);
4862 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
4863 break;
4864 }
4865 case ISD::ADDRSPACECAST:
4866 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4867 break;
4868 case ISD::GLOBAL_OFFSET_TABLE:
4869 case ISD::GlobalAddress:
4870 case ISD::GlobalTLSAddress:
4871 case ISD::ExternalSymbol:
4872 case ISD::ConstantPool:
4873 case ISD::JumpTable:
4874 case ISD::INTRINSIC_W_CHAIN:
4875 case ISD::INTRINSIC_WO_CHAIN:
4876 case ISD::INTRINSIC_VOID:
4877 // FIXME: Custom lowering for these operations shouldn't return null!
4878 // Return true so that we don't call ConvertNodeToLibcall which also won't
4879 // do anything.
4880 return true;
4881 }
4882
4883 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4884 // FIXME: We were asked to expand a strict floating-point operation,
4885 // but there is currently no expansion implemented that would preserve
4886 // the "strict" properties. For now, we just fall back to the non-strict
4887 // version if that is legal on the target. The actual mutation of the
4888 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4889 switch (Node->getOpcode()) {
4890 default:
4891 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4892 VT: Node->getValueType(ResNo: 0))
4893 == TargetLowering::Legal)
4894 return true;
4895 break;
4896 case ISD::STRICT_FSUB: {
4897 if (TLI.getStrictFPOperationAction(
4898 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4899 return true;
4900 if (TLI.getStrictFPOperationAction(
4901 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4902 break;
4903
4904 EVT VT = Node->getValueType(ResNo: 0);
4905 const SDNodeFlags Flags = Node->getFlags();
4906 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4907 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4908 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4909 Flags);
4910
4911 Results.push_back(Elt: Fadd);
4912 Results.push_back(Elt: Fadd.getValue(R: 1));
4913 break;
4914 }
4915 case ISD::STRICT_SINT_TO_FP:
4916 case ISD::STRICT_UINT_TO_FP:
4917 case ISD::STRICT_LRINT:
4918 case ISD::STRICT_LLRINT:
4919 case ISD::STRICT_LROUND:
4920 case ISD::STRICT_LLROUND:
4921 // These are registered by the operand type instead of the value
4922 // type. Reflect that here.
4923 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4924 VT: Node->getOperand(Num: 1).getValueType())
4925 == TargetLowering::Legal)
4926 return true;
4927 break;
4928 }
4929 }
4930
4931 // Replace the original node with the legalized result.
4932 if (Results.empty()) {
4933 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4934 return false;
4935 }
4936
4937 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4938 ReplaceNode(Old: Node, New: Results.data());
4939 return true;
4940}
4941
4942/// Return if we can use the FAST_* variant of a math libcall for the node.
4943/// FIXME: This is just guessing, we probably should have unique specific sets
4944/// flags required per libcall.
4945static bool canUseFastMathLibcall(const SDNode *Node) {
4946 // FIXME: Probably should define fast to respect nan/inf and only be
4947 // approximate functions.
4948
4949 SDNodeFlags Flags = Node->getFlags();
4950 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4951 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4952}
4953
4954void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4955 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4956 SmallVector<SDValue, 8> Results;
4957 SDLoc dl(Node);
4958 TargetLowering::MakeLibCallOptions CallOptions;
4959 CallOptions.IsPostTypeLegalization = true;
4960 // FIXME: Check flags on the node to see if we can use a finite call.
4961 unsigned Opc = Node->getOpcode();
4962 switch (Opc) {
4963 case ISD::ATOMIC_FENCE: {
4964 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4965 // FIXME: handle "fence singlethread" more efficiently.
4966 TargetLowering::ArgListTy Args;
4967
4968 TargetLowering::CallLoweringInfo CLI(DAG);
4969 CLI.setDebugLoc(dl)
4970 .setChain(Node->getOperand(Num: 0))
4971 .setLibCallee(
4972 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4973 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4974 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4975 ArgsList: std::move(Args));
4976
4977 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4978
4979 Results.push_back(Elt: CallResult.second);
4980 break;
4981 }
4982 // By default, atomic intrinsics are marked Legal and lowered. Targets
4983 // which don't support them directly, however, may want libcalls, in which
4984 // case they mark them Expand, and we get here.
4985 case ISD::ATOMIC_SWAP:
4986 case ISD::ATOMIC_LOAD_ADD:
4987 case ISD::ATOMIC_LOAD_SUB:
4988 case ISD::ATOMIC_LOAD_AND:
4989 case ISD::ATOMIC_LOAD_CLR:
4990 case ISD::ATOMIC_LOAD_OR:
4991 case ISD::ATOMIC_LOAD_XOR:
4992 case ISD::ATOMIC_LOAD_NAND:
4993 case ISD::ATOMIC_LOAD_MIN:
4994 case ISD::ATOMIC_LOAD_MAX:
4995 case ISD::ATOMIC_LOAD_UMIN:
4996 case ISD::ATOMIC_LOAD_UMAX:
4997 case ISD::ATOMIC_CMP_SWAP: {
4998 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4999 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
5000 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
5001 EVT RetVT = Node->getValueType(ResNo: 0);
5002 SmallVector<SDValue, 4> Ops;
5003 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported) {
5004 // If outline atomic available, prepare its arguments and expand.
5005 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
5006 Ops.push_back(Elt: Node->getOperand(Num: 1));
5007
5008 } else {
5009 LC = RTLIB::getSYNC(Opc, VT);
5010 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5011 "Unexpected atomic op or value type!");
5012 // Arguments for expansion to sync libcall
5013 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
5014 }
5015 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
5016 Ops, CallOptions,
5017 dl: SDLoc(Node),
5018 Chain: Node->getOperand(Num: 0));
5019 Results.push_back(Elt: Tmp.first);
5020 Results.push_back(Elt: Tmp.second);
5021 break;
5022 }
5023 case ISD::TRAP: {
5024 // If this operation is not supported, lower it to 'abort()' call
5025 TargetLowering::ArgListTy Args;
5026 TargetLowering::CallLoweringInfo CLI(DAG);
5027 CLI.setDebugLoc(dl)
5028 .setChain(Node->getOperand(Num: 0))
5029 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
5030 Target: DAG.getExternalSymbol(
5031 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
5032 ArgsList: std::move(Args));
5033 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
5034
5035 Results.push_back(Elt: CallResult.second);
5036 break;
5037 }
5038 case ISD::CLEAR_CACHE: {
5039 SDValue InputChain = Node->getOperand(Num: 0);
5040 SDValue StartVal = Node->getOperand(Num: 1);
5041 SDValue EndVal = Node->getOperand(Num: 2);
5042 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5043 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
5044 dl: SDLoc(Node), Chain: InputChain);
5045 Results.push_back(Elt: Tmp.second);
5046 break;
5047 }
5048 case ISD::FMINNUM:
5049 case ISD::STRICT_FMINNUM:
5050 ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
5051 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
5052 Call_PPCF128: RTLIB::FMIN_PPCF128, Results);
5053 break;
5054 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
5055 // libcall legalization for these nodes, but there is no default expasion for
5056 // these nodes either (see PR63267 for example).
5057 case ISD::FMAXNUM:
5058 case ISD::STRICT_FMAXNUM:
5059 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
5060 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
5061 Call_PPCF128: RTLIB::FMAX_PPCF128, Results);
5062 break;
5063 case ISD::FMINIMUMNUM:
5064 ExpandFPLibCall(Node, Call_F32: RTLIB::FMINIMUM_NUM_F32, Call_F64: RTLIB::FMINIMUM_NUM_F64,
5065 Call_F80: RTLIB::FMINIMUM_NUM_F80, Call_F128: RTLIB::FMINIMUM_NUM_F128,
5066 Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128, Results);
5067 break;
5068 case ISD::FMAXIMUMNUM:
5069 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAXIMUM_NUM_F32, Call_F64: RTLIB::FMAXIMUM_NUM_F64,
5070 Call_F80: RTLIB::FMAXIMUM_NUM_F80, Call_F128: RTLIB::FMAXIMUM_NUM_F128,
5071 Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128, Results);
5072 break;
5073 case ISD::FSQRT:
5074 case ISD::STRICT_FSQRT: {
5075 // FIXME: Probably should define fast to respect nan/inf and only be
5076 // approximate functions.
5077 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5078 Call_F32: {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
5079 Call_F64: {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
5080 Call_F80: {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
5081 Call_F128: {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
5082 Call_PPCF128: {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
5083 Results);
5084 break;
5085 }
5086 case ISD::FCBRT:
5087 ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64,
5088 Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128,
5089 Call_PPCF128: RTLIB::CBRT_PPCF128, Results);
5090 break;
5091 case ISD::FSIN:
5092 case ISD::STRICT_FSIN:
5093 ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
5094 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
5095 Call_PPCF128: RTLIB::SIN_PPCF128, Results);
5096 break;
5097 case ISD::FCOS:
5098 case ISD::STRICT_FCOS:
5099 ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
5100 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
5101 Call_PPCF128: RTLIB::COS_PPCF128, Results);
5102 break;
5103 case ISD::FTAN:
5104 case ISD::STRICT_FTAN:
5105 ExpandFPLibCall(Node, Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
5106 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128, Results);
5107 break;
5108 case ISD::FASIN:
5109 case ISD::STRICT_FASIN:
5110 ExpandFPLibCall(Node, Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
5111 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128, Results);
5112 break;
5113 case ISD::FACOS:
5114 case ISD::STRICT_FACOS:
5115 ExpandFPLibCall(Node, Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
5116 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128, Results);
5117 break;
5118 case ISD::FATAN:
5119 case ISD::STRICT_FATAN:
5120 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
5121 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128, Results);
5122 break;
5123 case ISD::FATAN2:
5124 case ISD::STRICT_FATAN2:
5125 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN2_F32, Call_F64: RTLIB::ATAN2_F64, Call_F80: RTLIB::ATAN2_F80,
5126 Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128, Results);
5127 break;
5128 case ISD::FSINH:
5129 case ISD::STRICT_FSINH:
5130 ExpandFPLibCall(Node, Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
5131 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128, Results);
5132 break;
5133 case ISD::FCOSH:
5134 case ISD::STRICT_FCOSH:
5135 ExpandFPLibCall(Node, Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
5136 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128, Results);
5137 break;
5138 case ISD::FTANH:
5139 case ISD::STRICT_FTANH:
5140 ExpandFPLibCall(Node, Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
5141 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128, Results);
5142 break;
5143 case ISD::FSINCOS:
5144 case ISD::FSINCOSPI: {
5145 EVT VT = Node->getValueType(ResNo: 0);
5146
5147 if (Node->getOpcode() == ISD::FSINCOS) {
5148 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(RetVT: VT);
5149 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
5150 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
5151 Results.push_back(Elt: Expanded);
5152 Results.push_back(Elt: Expanded.getValue(R: 1));
5153 break;
5154 }
5155 }
5156 }
5157
5158 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
5159 ? RTLIB::getSINCOS(RetVT: VT)
5160 : RTLIB::getSINCOSPI(RetVT: VT);
5161 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);
5162 if (!Expanded) {
5163 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
5164 Node->getOperationName(G: &DAG));
5165 SDValue Poison = DAG.getPOISON(VT);
5166 Results.push_back(Elt: Poison);
5167 Results.push_back(Elt: Poison);
5168 }
5169
5170 break;
5171 }
5172 case ISD::FLOG:
5173 case ISD::STRICT_FLOG:
5174 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80,
5175 Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results);
5176 break;
5177 case ISD::FLOG2:
5178 case ISD::STRICT_FLOG2:
5179 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80,
5180 Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results);
5181 break;
5182 case ISD::FLOG10:
5183 case ISD::STRICT_FLOG10:
5184 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80,
5185 Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results);
5186 break;
5187 case ISD::FEXP:
5188 case ISD::STRICT_FEXP:
5189 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80,
5190 Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results);
5191 break;
5192 case ISD::FEXP2:
5193 case ISD::STRICT_FEXP2:
5194 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80,
5195 Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results);
5196 break;
5197 case ISD::FEXP10:
5198 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
5199 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results);
5200 break;
5201 case ISD::FTRUNC:
5202 case ISD::STRICT_FTRUNC:
5203 ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
5204 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
5205 Call_PPCF128: RTLIB::TRUNC_PPCF128, Results);
5206 break;
5207 case ISD::FFLOOR:
5208 case ISD::STRICT_FFLOOR:
5209 ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
5210 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
5211 Call_PPCF128: RTLIB::FLOOR_PPCF128, Results);
5212 break;
5213 case ISD::FCEIL:
5214 case ISD::STRICT_FCEIL:
5215 ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
5216 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
5217 Call_PPCF128: RTLIB::CEIL_PPCF128, Results);
5218 break;
5219 case ISD::FRINT:
5220 case ISD::STRICT_FRINT:
5221 ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
5222 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
5223 Call_PPCF128: RTLIB::RINT_PPCF128, Results);
5224 break;
5225 case ISD::FNEARBYINT:
5226 case ISD::STRICT_FNEARBYINT:
5227 ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32,
5228 Call_F64: RTLIB::NEARBYINT_F64,
5229 Call_F80: RTLIB::NEARBYINT_F80,
5230 Call_F128: RTLIB::NEARBYINT_F128,
5231 Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results);
5232 break;
5233 case ISD::FROUND:
5234 case ISD::STRICT_FROUND:
5235 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32,
5236 Call_F64: RTLIB::ROUND_F64,
5237 Call_F80: RTLIB::ROUND_F80,
5238 Call_F128: RTLIB::ROUND_F128,
5239 Call_PPCF128: RTLIB::ROUND_PPCF128, Results);
5240 break;
5241 case ISD::FROUNDEVEN:
5242 case ISD::STRICT_FROUNDEVEN:
5243 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32,
5244 Call_F64: RTLIB::ROUNDEVEN_F64,
5245 Call_F80: RTLIB::ROUNDEVEN_F80,
5246 Call_F128: RTLIB::ROUNDEVEN_F128,
5247 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results);
5248 break;
5249 case ISD::FLDEXP:
5250 case ISD::STRICT_FLDEXP:
5251 ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80,
5252 Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results);
5253 break;
5254 case ISD::FMODF:
5255 case ISD::FFREXP: {
5256 EVT VT = Node->getValueType(ResNo: 0);
5257 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
5258 : RTLIB::getFREXP(RetVT: VT);
5259 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
5260 /*CallRetResNo=*/0);
5261 if (!Expanded)
5262 llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");
5263 break;
5264 }
5265 case ISD::FPOWI:
5266 case ISD::STRICT_FPOWI: {
5267 RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0));
5268 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
5269 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
5270 // Some targets don't have a powi libcall; use pow instead.
5271 if (Node->isStrictFPOpcode()) {
5272 SDValue Exponent =
5273 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
5274 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5275 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5276 SDValue FPOW =
5277 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
5278 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5279 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
5280 Results.push_back(Elt: FPOW);
5281 Results.push_back(Elt: FPOW.getValue(R: 1));
5282 } else {
5283 SDValue Exponent =
5284 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
5285 Operand: Node->getOperand(Num: 1));
5286 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
5287 VT: Node->getValueType(ResNo: 0),
5288 N1: Node->getOperand(Num: 0), N2: Exponent));
5289 }
5290 break;
5291 }
5292 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
5293 bool ExponentHasSizeOfInt =
5294 DAG.getLibInfo().getIntSize() ==
5295 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
5296 if (!ExponentHasSizeOfInt) {
5297 // If the exponent does not match with sizeof(int) a libcall to
5298 // RTLIB::POWI would use the wrong type for the argument.
5299 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
5300 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
5301 break;
5302 }
5303 ExpandFPLibCall(Node, LC, Results);
5304 break;
5305 }
5306 case ISD::FPOW:
5307 case ISD::STRICT_FPOW:
5308 ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80,
5309 Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results);
5310 break;
5311 case ISD::LROUND:
5312 case ISD::STRICT_LROUND:
5313 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
5314 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
5315 Call_F128: RTLIB::LROUND_F128,
5316 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
5317 break;
5318 case ISD::LLROUND:
5319 case ISD::STRICT_LLROUND:
5320 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
5321 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
5322 Call_F128: RTLIB::LLROUND_F128,
5323 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
5324 break;
5325 case ISD::LRINT:
5326 case ISD::STRICT_LRINT:
5327 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
5328 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
5329 Call_F128: RTLIB::LRINT_F128,
5330 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
5331 break;
5332 case ISD::LLRINT:
5333 case ISD::STRICT_LLRINT:
5334 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
5335 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
5336 Call_F128: RTLIB::LLRINT_F128,
5337 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
5338 break;
5339 case ISD::FDIV:
5340 case ISD::STRICT_FDIV: {
5341 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5342 Call_F32: {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5343 Call_F64: {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5344 Call_F80: {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5345 Call_F128: {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5346 Call_PPCF128: {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5347 break;
5348 }
5349 case ISD::FREM:
5350 case ISD::STRICT_FREM:
5351 ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
5352 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
5353 Call_PPCF128: RTLIB::REM_PPCF128, Results);
5354 break;
5355 case ISD::FMA:
5356 case ISD::STRICT_FMA:
5357 ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64,
5358 Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128,
5359 Call_PPCF128: RTLIB::FMA_PPCF128, Results);
5360 break;
5361 case ISD::FADD:
5362 case ISD::STRICT_FADD: {
5363 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5364 Call_F32: {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5365 Call_F64: {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5366 Call_F80: {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5367 Call_F128: {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5368 Call_PPCF128: {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5369 break;
5370 }
5371 case ISD::FMUL:
5372 case ISD::STRICT_FMUL: {
5373 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5374 Call_F32: {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5375 Call_F64: {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5376 Call_F80: {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5377 Call_F128: {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5378 Call_PPCF128: {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5379 break;
5380 }
5381 case ISD::FP16_TO_FP:
5382 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5383 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
5384 }
5385 break;
5386 case ISD::STRICT_BF16_TO_FP:
5387 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5388 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5389 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
5390 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5391 Results.push_back(Elt: Tmp.first);
5392 Results.push_back(Elt: Tmp.second);
5393 }
5394 break;
5395 case ISD::STRICT_FP16_TO_FP: {
5396 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5397 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5398 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
5399 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5400 Results.push_back(Elt: Tmp.first);
5401 Results.push_back(Elt: Tmp.second);
5402 }
5403 break;
5404 }
5405 case ISD::FP_TO_FP16: {
5406 RTLIB::Libcall LC =
5407 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
5408 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5409 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5410 break;
5411 }
5412 case ISD::FP_TO_BF16: {
5413 RTLIB::Libcall LC =
5414 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
5415 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5416 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5417 break;
5418 }
5419 case ISD::STRICT_SINT_TO_FP:
5420 case ISD::STRICT_UINT_TO_FP:
5421 case ISD::SINT_TO_FP:
5422 case ISD::UINT_TO_FP: {
5423 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
5424 bool IsStrict = Node->isStrictFPOpcode();
5425 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
5426 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
5427 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
5428 EVT RVT = Node->getValueType(ResNo: 0);
5429 EVT NVT = EVT();
5430 SDLoc dl(Node);
5431
5432 // Even if the input is legal, no libcall may exactly match, eg. we don't
5433 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
5434 // eg: i13 -> fp. Then, look for an appropriate libcall.
5435 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5436 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5437 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5438 ++t) {
5439 NVT = (MVT::SimpleValueType)t;
5440 // The source needs to big enough to hold the operand.
5441 if (NVT.bitsGE(VT: SVT))
5442 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
5443 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
5444 }
5445 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5446
5447 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5448 // Sign/zero extend the argument if the libcall takes a larger type.
5449 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
5450 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
5451 CallOptions.setIsSigned(Signed);
5452 std::pair<SDValue, SDValue> Tmp =
5453 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
5454 Results.push_back(Elt: Tmp.first);
5455 if (IsStrict)
5456 Results.push_back(Elt: Tmp.second);
5457 break;
5458 }
5459 case ISD::FP_TO_SINT:
5460 case ISD::FP_TO_UINT:
5461 case ISD::STRICT_FP_TO_SINT:
5462 case ISD::STRICT_FP_TO_UINT: {
5463 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
5464 bool IsStrict = Node->isStrictFPOpcode();
5465 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
5466 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
5467
5468 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5469 EVT SVT = Op.getValueType();
5470 EVT RVT = Node->getValueType(ResNo: 0);
5471 EVT NVT = EVT();
5472 SDLoc dl(Node);
5473
5474 // Even if the result is legal, no libcall may exactly match, eg. we don't
5475 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
5476 // eg: fp -> i32. Then, look for an appropriate libcall.
5477 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5478 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5479 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5480 ++IntVT) {
5481 NVT = (MVT::SimpleValueType)IntVT;
5482 // The type needs to big enough to hold the result.
5483 if (NVT.bitsGE(VT: RVT))
5484 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
5485 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
5486 }
5487 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5488
5489 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5490 std::pair<SDValue, SDValue> Tmp =
5491 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
5492
5493 // Truncate the result if the libcall returns a larger type.
5494 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
5495 if (IsStrict)
5496 Results.push_back(Elt: Tmp.second);
5497 break;
5498 }
5499
5500 case ISD::FP_ROUND:
5501 case ISD::STRICT_FP_ROUND: {
5502 // X = FP_ROUND(Y, TRUNC)
5503 // TRUNC is a flag, which is always an integer that is zero or one.
5504 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5505 // is known to not change the value of Y.
5506 // We can only expand it into libcall if the TRUNC is 0.
5507 bool IsStrict = Node->isStrictFPOpcode();
5508 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5509 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5510 EVT VT = Node->getValueType(ResNo: 0);
5511 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5512 "Unable to expand as libcall if it is not normal rounding");
5513
5514 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5515 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5516
5517 std::pair<SDValue, SDValue> Tmp =
5518 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5519 Results.push_back(Elt: Tmp.first);
5520 if (IsStrict)
5521 Results.push_back(Elt: Tmp.second);
5522 break;
5523 }
5524 case ISD::FP_EXTEND: {
5525 Results.push_back(
5526 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5527 RetVT: Node->getValueType(ResNo: 0)),
5528 Node, isSigned: false).first);
5529 break;
5530 }
5531 case ISD::STRICT_FP_EXTEND:
5532 case ISD::STRICT_FP_TO_FP16:
5533 case ISD::STRICT_FP_TO_BF16: {
5534 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5535 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5536 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5537 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5538 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5539 else
5540 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5541 RetVT: Node->getValueType(ResNo: 0));
5542
5543 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5544
5545 std::pair<SDValue, SDValue> Tmp =
5546 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5547 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5548 Results.push_back(Elt: Tmp.first);
5549 Results.push_back(Elt: Tmp.second);
5550 break;
5551 }
5552 case ISD::FSUB:
5553 case ISD::STRICT_FSUB: {
5554 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5555 Call_F32: {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5556 Call_F64: {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5557 Call_F80: {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5558 Call_F128: {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5559 Call_PPCF128: {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5560 break;
5561 }
5562 case ISD::SREM:
5563 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5564 Call_I8: RTLIB::SREM_I8,
5565 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5566 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5567 break;
5568 case ISD::UREM:
5569 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5570 Call_I8: RTLIB::UREM_I8,
5571 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5572 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5573 break;
5574 case ISD::SDIV:
5575 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5576 Call_I8: RTLIB::SDIV_I8,
5577 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5578 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5579 break;
5580 case ISD::UDIV:
5581 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5582 Call_I8: RTLIB::UDIV_I8,
5583 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5584 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5585 break;
5586 case ISD::SDIVREM:
5587 case ISD::UDIVREM:
5588 // Expand into divrem libcall
5589 ExpandDivRemLibCall(Node, Results);
5590 break;
5591 case ISD::MUL:
5592 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5593 Call_I8: RTLIB::MUL_I8,
5594 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5595 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5596 break;
5597 case ISD::CTLZ_ZERO_UNDEF:
5598 Results.push_back(Elt: ExpandBitCountingLibCall(
5599 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5600 break;
5601 case ISD::CTPOP:
5602 Results.push_back(Elt: ExpandBitCountingLibCall(
5603 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5604 break;
5605 case ISD::RESET_FPENV: {
5606 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5607 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5608 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5609 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5610 SDValue Chain = Node->getOperand(Num: 0);
5611 Results.push_back(
5612 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
5613 break;
5614 }
5615 case ISD::GET_FPENV_MEM: {
5616 SDValue Chain = Node->getOperand(Num: 0);
5617 SDValue EnvPtr = Node->getOperand(Num: 1);
5618 Results.push_back(
5619 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5620 break;
5621 }
5622 case ISD::SET_FPENV_MEM: {
5623 SDValue Chain = Node->getOperand(Num: 0);
5624 SDValue EnvPtr = Node->getOperand(Num: 1);
5625 Results.push_back(
5626 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5627 break;
5628 }
5629 case ISD::GET_FPMODE: {
5630 // Call fegetmode, which saves control modes into a stack slot. Then load
5631 // the value to return from the stack.
5632 EVT ModeVT = Node->getValueType(ResNo: 0);
5633 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5634 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5635 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5636 InChain: Node->getOperand(Num: 0), DLoc: dl);
5637 SDValue LdInst = DAG.getLoad(
5638 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5639 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5640 Results.push_back(Elt: LdInst);
5641 Results.push_back(Elt: LdInst.getValue(R: 1));
5642 break;
5643 }
5644 case ISD::SET_FPMODE: {
5645 // Move control modes to stack slot and then call fesetmode with the pointer
5646 // to the slot as argument.
5647 SDValue Mode = Node->getOperand(Num: 1);
5648 EVT ModeVT = Mode.getValueType();
5649 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5650 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5651 SDValue StInst = DAG.getStore(
5652 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5653 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5654 Results.push_back(
5655 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
5656 break;
5657 }
5658 case ISD::RESET_FPMODE: {
5659 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5660 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5661 // target must provide custom lowering.
5662 const DataLayout &DL = DAG.getDataLayout();
5663 EVT PtrTy = TLI.getPointerTy(DL);
5664 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5665 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5666 InChain: Node->getOperand(Num: 0), DLoc: dl));
5667 break;
5668 }
5669 }
5670
5671 // Replace the original node with the legalized result.
5672 if (!Results.empty()) {
5673 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5674 ReplaceNode(Old: Node, New: Results.data());
5675 } else
5676 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5677}
5678
5679// Determine the vector type to use in place of an original scalar element when
5680// promoting equally sized vectors.
5681static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5682 MVT EltVT, MVT NewEltVT) {
5683 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5684 MVT MidVT = OldEltsPerNewElt == 1
5685 ? NewEltVT
5686 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5687 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5688 return MidVT;
5689}
5690
5691void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5692 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5693 SmallVector<SDValue, 8> Results;
5694 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5695 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5696 Node->getOpcode() == ISD::SINT_TO_FP ||
5697 Node->getOpcode() == ISD::SETCC ||
5698 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5699 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5700 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5701 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5702 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5703 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM) {
5704 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5705 }
5706 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5707 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5708 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5709 Node->getOpcode() == ISD::STRICT_FSETCC ||
5710 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5711 Node->getOpcode() == ISD::STRICT_LRINT ||
5712 Node->getOpcode() == ISD::STRICT_LLRINT ||
5713 Node->getOpcode() == ISD::STRICT_LROUND ||
5714 Node->getOpcode() == ISD::STRICT_LLROUND ||
5715 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5716 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5717 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5718 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5719 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5720 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5721 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5722 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5723 if (Node->getOpcode() == ISD::BR_CC ||
5724 Node->getOpcode() == ISD::SELECT_CC)
5725 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5726 // Preserve fast math flags
5727 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5728 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5729 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5730 SDLoc dl(Node);
5731 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5732 switch (Node->getOpcode()) {
5733 case ISD::CTTZ:
5734 case ISD::CTTZ_ZERO_UNDEF:
5735 case ISD::CTLZ:
5736 case ISD::CTPOP: {
5737 // Zero extend the argument unless its cttz, then use any_extend.
5738 if (Node->getOpcode() == ISD::CTTZ ||
5739 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
5740 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5741 else
5742 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5743
5744 unsigned NewOpc = Node->getOpcode();
5745 if (NewOpc == ISD::CTTZ) {
5746 // The count is the same in the promoted type except if the original
5747 // value was zero. This can be handled by setting the bit just off
5748 // the top of the original type.
5749 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5750 BitNo: OVT.getSizeInBits());
5751 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5752 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5753 NewOpc = ISD::CTTZ_ZERO_UNDEF;
5754 }
5755 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
5756 // already the correct result.
5757 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5758 if (NewOpc == ISD::CTLZ) {
5759 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5760 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5761 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5762 OVT.getSizeInBits(), DL: dl, VT: NVT));
5763 }
5764 Results.push_back(
5765 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5766 break;
5767 }
5768 case ISD::CTLZ_ZERO_UNDEF: {
5769 // We know that the argument is unlikely to be zero, hence we can take a
5770 // different approach as compared to ISD::CTLZ
5771
5772 // Any Extend the argument
5773 auto AnyExtendedNode =
5774 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5775
5776 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5777 auto ShiftConstant = DAG.getShiftAmountConstant(
5778 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5779 auto LeftShiftResult =
5780 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5781
5782 // Perform the larger operation
5783 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5784 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5785 break;
5786 }
5787 case ISD::BITREVERSE:
5788 case ISD::BSWAP: {
5789 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5790 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5791 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5792 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5793 N2: DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl));
5794
5795 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5796 break;
5797 }
5798 case ISD::FP_TO_UINT:
5799 case ISD::STRICT_FP_TO_UINT:
5800 case ISD::FP_TO_SINT:
5801 case ISD::STRICT_FP_TO_SINT:
5802 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5803 break;
5804 case ISD::FP_TO_UINT_SAT:
5805 case ISD::FP_TO_SINT_SAT:
5806 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5807 break;
5808 case ISD::UINT_TO_FP:
5809 case ISD::STRICT_UINT_TO_FP:
5810 case ISD::SINT_TO_FP:
5811 case ISD::STRICT_SINT_TO_FP:
5812 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5813 break;
5814 case ISD::VAARG: {
5815 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5816 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5817
5818 unsigned TruncOp;
5819 if (OVT.isVector()) {
5820 TruncOp = ISD::BITCAST;
5821 } else {
5822 assert(OVT.isInteger()
5823 && "VAARG promotion is supported only for vectors or integer types");
5824 TruncOp = ISD::TRUNCATE;
5825 }
5826
5827 // Perform the larger operation, then convert back
5828 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5829 Align: Node->getConstantOperandVal(Num: 3));
5830 Chain = Tmp1.getValue(R: 1);
5831
5832 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5833
5834 // Modified the chain result - switch anything that used the old chain to
5835 // use the new one.
5836 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5837 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5838 if (UpdatedNodes) {
5839 UpdatedNodes->insert(X: Tmp2.getNode());
5840 UpdatedNodes->insert(X: Chain.getNode());
5841 }
5842 ReplacedNode(N: Node);
5843 break;
5844 }
5845 case ISD::MUL:
5846 case ISD::SDIV:
5847 case ISD::SREM:
5848 case ISD::UDIV:
5849 case ISD::UREM:
5850 case ISD::SMIN:
5851 case ISD::SMAX:
5852 case ISD::UMIN:
5853 case ISD::UMAX:
5854 case ISD::AND:
5855 case ISD::OR:
5856 case ISD::XOR: {
5857 unsigned ExtOp, TruncOp;
5858 if (OVT.isVector()) {
5859 ExtOp = ISD::BITCAST;
5860 TruncOp = ISD::BITCAST;
5861 } else {
5862 assert(OVT.isInteger() && "Cannot promote logic operation");
5863
5864 switch (Node->getOpcode()) {
5865 default:
5866 ExtOp = ISD::ANY_EXTEND;
5867 break;
5868 case ISD::SDIV:
5869 case ISD::SREM:
5870 case ISD::SMIN:
5871 case ISD::SMAX:
5872 ExtOp = ISD::SIGN_EXTEND;
5873 break;
5874 case ISD::UDIV:
5875 case ISD::UREM:
5876 ExtOp = ISD::ZERO_EXTEND;
5877 break;
5878 case ISD::UMIN:
5879 case ISD::UMAX:
5880 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5881 ExtOp = ISD::SIGN_EXTEND;
5882 else
5883 ExtOp = ISD::ZERO_EXTEND;
5884 break;
5885 }
5886 TruncOp = ISD::TRUNCATE;
5887 }
5888 // Promote each of the values to the new type.
5889 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5890 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5891 // Perform the larger operation, then convert back
5892 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5893 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5894 break;
5895 }
5896 case ISD::UMUL_LOHI:
5897 case ISD::SMUL_LOHI: {
5898 // Promote to a multiply in a wider integer type.
5899 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5900 : ISD::SIGN_EXTEND;
5901 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5902 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5903 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5904
5905 unsigned OriginalSize = OVT.getScalarSizeInBits();
5906 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5907 N2: DAG.getShiftAmountConstant(Val: OriginalSize, VT: NVT, DL: dl));
5908 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5909 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5910 break;
5911 }
5912 case ISD::SELECT: {
5913 unsigned ExtOp, TruncOp;
5914 if (Node->getValueType(ResNo: 0).isVector() ||
5915 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5916 ExtOp = ISD::BITCAST;
5917 TruncOp = ISD::BITCAST;
5918 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5919 ExtOp = ISD::ANY_EXTEND;
5920 TruncOp = ISD::TRUNCATE;
5921 } else {
5922 ExtOp = ISD::FP_EXTEND;
5923 TruncOp = ISD::FP_ROUND;
5924 }
5925 Tmp1 = Node->getOperand(Num: 0);
5926 // Promote each of the values to the new type.
5927 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5928 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5929 // Perform the larger operation, then round down.
5930 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5931 if (TruncOp != ISD::FP_ROUND)
5932 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5933 else
5934 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5935 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5936 Results.push_back(Elt: Tmp1);
5937 break;
5938 }
5939 case ISD::VECTOR_SHUFFLE: {
5940 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5941
5942 // Cast the two input vectors.
5943 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5944 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5945
5946 // Convert the shuffle mask to the right # elements.
5947 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5948 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5949 Results.push_back(Elt: Tmp1);
5950 break;
5951 }
5952 case ISD::VECTOR_SPLICE_LEFT:
5953 case ISD::VECTOR_SPLICE_RIGHT: {
5954 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5955 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5956 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5957 N3: Node->getOperand(Num: 2));
5958 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5959 break;
5960 }
5961 case ISD::SELECT_CC: {
5962 SDValue Cond = Node->getOperand(Num: 4);
5963 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5964 // Type of the comparison operands.
5965 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5966 assert(CVT == OVT && "not handled");
5967
5968 unsigned ExtOp = ISD::FP_EXTEND;
5969 if (NVT.isInteger()) {
5970 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5971 }
5972
5973 // Promote the comparison operands, if needed.
5974 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5975 Tmp1 = Node->getOperand(Num: 0);
5976 Tmp2 = Node->getOperand(Num: 1);
5977 } else {
5978 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5979 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5980 }
5981 // Cast the true/false operands.
5982 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5983 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5984
5985 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5986 Flags: Node->getFlags());
5987
5988 // Cast the result back to the original type.
5989 if (ExtOp != ISD::FP_EXTEND)
5990 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5991 else
5992 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5993 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5994
5995 Results.push_back(Elt: Tmp1);
5996 break;
5997 }
5998 case ISD::SETCC:
5999 case ISD::STRICT_FSETCC:
6000 case ISD::STRICT_FSETCCS: {
6001 unsigned ExtOp = ISD::FP_EXTEND;
6002 if (NVT.isInteger()) {
6003 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
6004 if (isSignedIntSetCC(Code: CCCode) ||
6005 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
6006 ExtOp = ISD::SIGN_EXTEND;
6007 else
6008 ExtOp = ISD::ZERO_EXTEND;
6009 }
6010 if (Node->isStrictFPOpcode()) {
6011 SDValue InChain = Node->getOperand(Num: 0);
6012 std::tie(args&: Tmp1, args: std::ignore) =
6013 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
6014 std::tie(args&: Tmp2, args: std::ignore) =
6015 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
6016 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
6017 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
6018 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
6019 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
6020 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
6021 Flags: Node->getFlags()));
6022 Results.push_back(Elt: Results.back().getValue(R: 1));
6023 break;
6024 }
6025 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6026 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
6027 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
6028 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
6029 break;
6030 }
6031 case ISD::BR_CC: {
6032 unsigned ExtOp = ISD::FP_EXTEND;
6033 if (NVT.isInteger()) {
6034 ISD::CondCode CCCode =
6035 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
6036 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
6037 }
6038 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
6039 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
6040 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
6041 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
6042 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
6043 break;
6044 }
6045 case ISD::FADD:
6046 case ISD::FSUB:
6047 case ISD::FMUL:
6048 case ISD::FDIV:
6049 case ISD::FREM:
6050 case ISD::FMINNUM:
6051 case ISD::FMAXNUM:
6052 case ISD::FMINIMUM:
6053 case ISD::FMAXIMUM:
6054 case ISD::FMINIMUMNUM:
6055 case ISD::FMAXIMUMNUM:
6056 case ISD::FPOW:
6057 case ISD::FATAN2:
6058 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6059 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
6060 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
6061 Results.push_back(
6062 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
6063 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6064 break;
6065
6066 case ISD::STRICT_FMINIMUM:
6067 case ISD::STRICT_FMAXIMUM: {
6068 SDValue InChain = Node->getOperand(Num: 0);
6069 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
6070 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
6071 N2: Node->getOperand(Num: 1));
6072 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
6073 N2: Node->getOperand(Num: 2));
6074 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
6075 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
6076 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
6077 N1: InChain, N2: Tmp3,
6078 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
6079 Results.push_back(Elt: Tmp4);
6080 Results.push_back(Elt: Tmp4.getValue(R: 1));
6081 break;
6082 }
6083
6084 case ISD::STRICT_FADD:
6085 case ISD::STRICT_FSUB:
6086 case ISD::STRICT_FMUL:
6087 case ISD::STRICT_FDIV:
6088 case ISD::STRICT_FMINNUM:
6089 case ISD::STRICT_FMAXNUM:
6090 case ISD::STRICT_FREM:
6091 case ISD::STRICT_FPOW:
6092 case ISD::STRICT_FATAN2:
6093 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6094 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6095 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6096 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
6097 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
6098 N2: Tmp2.getValue(R: 1));
6099 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6100 Ops: {Tmp3, Tmp1, Tmp2});
6101 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6102 Ops: {Tmp1.getValue(R: 1), Tmp1,
6103 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6104 Results.push_back(Elt: Tmp1);
6105 Results.push_back(Elt: Tmp1.getValue(R: 1));
6106 break;
6107 case ISD::FMA:
6108 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6109 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
6110 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
6111 Results.push_back(
6112 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
6113 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
6114 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6115 break;
6116 case ISD::STRICT_FMA:
6117 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6118 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6119 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6120 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
6121 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6122 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
6123 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
6124 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
6125 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6126 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
6127 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6128 Ops: {Tmp4.getValue(R: 1), Tmp4,
6129 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6130 Results.push_back(Elt: Tmp4);
6131 Results.push_back(Elt: Tmp4.getValue(R: 1));
6132 break;
6133 case ISD::FCOPYSIGN:
6134 case ISD::FLDEXP:
6135 case ISD::FPOWI: {
6136 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6137 Tmp2 = Node->getOperand(Num: 1);
6138 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
6139
6140 // fcopysign doesn't change anything but the sign bit, so
6141 // (fp_round (fcopysign (fpext a), b))
6142 // is as precise as
6143 // (fp_round (fpext a))
6144 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
6145 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
6146 Results.push_back(
6147 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
6148 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
6149 break;
6150 }
6151 case ISD::STRICT_FLDEXP: {
6152 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6153 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6154 Tmp2 = Node->getOperand(Num: 2);
6155 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
6156 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
6157 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6158 Ops: {Tmp3.getValue(R: 1), Tmp3,
6159 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6160 Results.push_back(Elt: Tmp4);
6161 Results.push_back(Elt: Tmp4.getValue(R: 1));
6162 break;
6163 }
6164 case ISD::STRICT_FPOWI:
6165 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6166 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6167 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6168 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
6169 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6170 Ops: {Tmp2.getValue(R: 1), Tmp2,
6171 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6172 Results.push_back(Elt: Tmp3);
6173 Results.push_back(Elt: Tmp3.getValue(R: 1));
6174 break;
6175 case ISD::FFREXP: {
6176 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6177 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
6178
6179 Results.push_back(
6180 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6181 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6182
6183 Results.push_back(Elt: Tmp2.getValue(R: 1));
6184 break;
6185 }
6186 case ISD::FMODF:
6187 case ISD::FSINCOS:
6188 case ISD::FSINCOSPI: {
6189 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6190 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
6191 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
6192 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
6193 Results.push_back(
6194 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
6195 break;
6196 }
6197 case ISD::FFLOOR:
6198 case ISD::FCEIL:
6199 case ISD::FRINT:
6200 case ISD::FNEARBYINT:
6201 case ISD::FROUND:
6202 case ISD::FROUNDEVEN:
6203 case ISD::FTRUNC:
6204 case ISD::FNEG:
6205 case ISD::FSQRT:
6206 case ISD::FSIN:
6207 case ISD::FCOS:
6208 case ISD::FTAN:
6209 case ISD::FASIN:
6210 case ISD::FACOS:
6211 case ISD::FATAN:
6212 case ISD::FSINH:
6213 case ISD::FCOSH:
6214 case ISD::FTANH:
6215 case ISD::FLOG:
6216 case ISD::FLOG2:
6217 case ISD::FLOG10:
6218 case ISD::FABS:
6219 case ISD::FEXP:
6220 case ISD::FEXP2:
6221 case ISD::FEXP10:
6222 case ISD::FCANONICALIZE:
6223 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6224 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6225 Results.push_back(
6226 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6227 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6228 break;
6229 case ISD::STRICT_FFLOOR:
6230 case ISD::STRICT_FCEIL:
6231 case ISD::STRICT_FRINT:
6232 case ISD::STRICT_FNEARBYINT:
6233 case ISD::STRICT_FROUND:
6234 case ISD::STRICT_FROUNDEVEN:
6235 case ISD::STRICT_FTRUNC:
6236 case ISD::STRICT_FSQRT:
6237 case ISD::STRICT_FSIN:
6238 case ISD::STRICT_FCOS:
6239 case ISD::STRICT_FTAN:
6240 case ISD::STRICT_FASIN:
6241 case ISD::STRICT_FACOS:
6242 case ISD::STRICT_FATAN:
6243 case ISD::STRICT_FSINH:
6244 case ISD::STRICT_FCOSH:
6245 case ISD::STRICT_FTANH:
6246 case ISD::STRICT_FLOG:
6247 case ISD::STRICT_FLOG2:
6248 case ISD::STRICT_FLOG10:
6249 case ISD::STRICT_FEXP:
6250 case ISD::STRICT_FEXP2:
6251 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6252 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6253 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6254 Ops: {Tmp1.getValue(R: 1), Tmp1});
6255 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6256 Ops: {Tmp2.getValue(R: 1), Tmp2,
6257 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6258 Results.push_back(Elt: Tmp3);
6259 Results.push_back(Elt: Tmp3.getValue(R: 1));
6260 break;
6261 case ISD::LLROUND:
6262 case ISD::LROUND:
6263 case ISD::LRINT:
6264 case ISD::LLRINT:
6265 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6266 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
6267 Results.push_back(Elt: Tmp2);
6268 break;
6269 case ISD::STRICT_LLROUND:
6270 case ISD::STRICT_LROUND:
6271 case ISD::STRICT_LRINT:
6272 case ISD::STRICT_LLRINT:
6273 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6274 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6275 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6276 Ops: {Tmp1.getValue(R: 1), Tmp1});
6277 Results.push_back(Elt: Tmp2);
6278 Results.push_back(Elt: Tmp2.getValue(R: 1));
6279 break;
6280 case ISD::BUILD_VECTOR: {
6281 MVT EltVT = OVT.getVectorElementType();
6282 MVT NewEltVT = NVT.getVectorElementType();
6283
6284 // Handle bitcasts to a different vector type with the same total bit size
6285 //
6286 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
6287 // =>
6288 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
6289
6290 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6291 "Invalid promote type for build_vector");
6292 assert(NewEltVT.bitsLE(EltVT) && "not handled");
6293
6294 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6295
6296 SmallVector<SDValue, 8> NewOps;
6297 for (const SDValue &Op : Node->op_values())
6298 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
6299
6300 SDLoc SL(Node);
6301 SDValue Concat =
6302 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
6303 DL: SL, VT: NVT, Ops: NewOps);
6304 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6305 Results.push_back(Elt: CvtVec);
6306 break;
6307 }
6308 case ISD::EXTRACT_VECTOR_ELT: {
6309 MVT EltVT = OVT.getVectorElementType();
6310 MVT NewEltVT = NVT.getVectorElementType();
6311
6312 // Handle bitcasts to a different vector type with the same total bit size.
6313 //
6314 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
6315 // =>
6316 // v4i32:castx = bitcast x:v2i64
6317 //
6318 // i64 = bitcast
6319 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
6320 // (i32 (extract_vector_elt castx, (2 * y + 1)))
6321 //
6322
6323 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6324 "Invalid promote type for extract_vector_elt");
6325 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6326
6327 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6328 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6329
6330 SDValue Idx = Node->getOperand(Num: 1);
6331 EVT IdxVT = Idx.getValueType();
6332 SDLoc SL(Node);
6333 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
6334 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6335
6336 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6337
6338 SmallVector<SDValue, 8> NewOps;
6339 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6340 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6341 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6342
6343 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6344 N1: CastVec, N2: TmpIdx);
6345 NewOps.push_back(Elt);
6346 }
6347
6348 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
6349 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
6350 break;
6351 }
6352 case ISD::INSERT_VECTOR_ELT: {
6353 MVT EltVT = OVT.getVectorElementType();
6354 MVT NewEltVT = NVT.getVectorElementType();
6355
6356 // Handle bitcasts to a different vector type with the same total bit size
6357 //
6358 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
6359 // =>
6360 // v4i32:castx = bitcast x:v2i64
6361 // v2i32:casty = bitcast y:i64
6362 //
6363 // v2i64 = bitcast
6364 // (v4i32 insert_vector_elt
6365 // (v4i32 insert_vector_elt v4i32:castx,
6366 // (extract_vector_elt casty, 0), 2 * z),
6367 // (extract_vector_elt casty, 1), (2 * z + 1))
6368
6369 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6370 "Invalid promote type for insert_vector_elt");
6371 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6372
6373 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6374 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6375
6376 SDValue Val = Node->getOperand(Num: 1);
6377 SDValue Idx = Node->getOperand(Num: 2);
6378 EVT IdxVT = Idx.getValueType();
6379 SDLoc SL(Node);
6380
6381 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
6382 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6383
6384 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6385 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6386
6387 SDValue NewVec = CastVec;
6388 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6389 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6390 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6391
6392 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6393 N1: CastVal, N2: IdxOffset);
6394
6395 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
6396 N1: NewVec, N2: Elt, N3: InEltIdx);
6397 }
6398
6399 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
6400 break;
6401 }
6402 case ISD::SCALAR_TO_VECTOR: {
6403 MVT EltVT = OVT.getVectorElementType();
6404 MVT NewEltVT = NVT.getVectorElementType();
6405
6406 // Handle bitcasts to different vector type with the same total bit size.
6407 //
6408 // e.g. v2i64 = scalar_to_vector x:i64
6409 // =>
6410 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
6411 //
6412
6413 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6414 SDValue Val = Node->getOperand(Num: 0);
6415 SDLoc SL(Node);
6416
6417 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6418 SDValue Undef = DAG.getUNDEF(VT: MidVT);
6419
6420 SmallVector<SDValue, 8> NewElts;
6421 NewElts.push_back(Elt: CastVal);
6422 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
6423 NewElts.push_back(Elt: Undef);
6424
6425 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
6426 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6427 Results.push_back(Elt: CvtVec);
6428 break;
6429 }
6430 case ISD::ATOMIC_SWAP:
6431 case ISD::ATOMIC_STORE: {
6432 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6433 SDLoc SL(Node);
6434 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
6435 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6436 "unexpected promotion type");
6437 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6438 "unexpected atomic_swap with illegal type");
6439
6440 SDValue Op0 = AM->getBasePtr();
6441 SDValue Op1 = CastVal;
6442
6443 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
6444 // but really it should merge with ISD::STORE.
6445 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6446 std::swap(a&: Op0, b&: Op1);
6447
6448 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
6449 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
6450
6451 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6452 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6453 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6454 } else
6455 Results.push_back(Elt: NewAtomic);
6456 break;
6457 }
6458 case ISD::ATOMIC_LOAD: {
6459 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6460 SDLoc SL(Node);
6461 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6462 "unexpected promotion type");
6463 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6464 "unexpected atomic_load with illegal type");
6465
6466 SDValue NewAtomic =
6467 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
6468 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
6469 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6470 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6471 break;
6472 }
6473 case ISD::SPLAT_VECTOR: {
6474 SDValue Scalar = Node->getOperand(Num: 0);
6475 MVT ScalarType = Scalar.getSimpleValueType();
6476 MVT NewScalarType = NVT.getVectorElementType();
6477 if (ScalarType.isInteger()) {
6478 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6479 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6480 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
6481 break;
6482 }
6483 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6484 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6485 Results.push_back(
6486 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6487 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6488 break;
6489 }
6490 case ISD::VECREDUCE_FMAX:
6491 case ISD::VECREDUCE_FMIN:
6492 case ISD::VECREDUCE_FMAXIMUM:
6493 case ISD::VECREDUCE_FMINIMUM:
6494 case ISD::VP_REDUCE_FMAX:
6495 case ISD::VP_REDUCE_FMIN:
6496 case ISD::VP_REDUCE_FMAXIMUM:
6497 case ISD::VP_REDUCE_FMINIMUM:
6498 Results.push_back(Elt: PromoteReduction(Node));
6499 break;
6500 }
6501
6502 // Replace the original node with the legalized result.
6503 if (!Results.empty()) {
6504 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6505 ReplaceNode(Old: Node, New: Results.data());
6506 } else
6507 LLVM_DEBUG(dbgs() << "Could not promote node\n");
6508}
6509
6510/// This is the entry point for the file.
6511void SelectionDAG::Legalize() {
6512 AssignTopologicalOrder();
6513
6514 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6515 // Use a delete listener to remove nodes which were deleted during
6516 // legalization from LegalizeNodes. This is needed to handle the situation
6517 // where a new node is allocated by the object pool to the same address of a
6518 // previously deleted node.
6519 DAGNodeDeletedListener DeleteListener(
6520 *this,
6521 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
6522
6523 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6524
6525 // Visit all the nodes. We start in topological order, so that we see
6526 // nodes with their original operands intact. Legalization can produce
6527 // new nodes which may themselves need to be legalized. Iterate until all
6528 // nodes have been legalized.
6529 while (true) {
6530 bool AnyLegalized = false;
6531 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6532 --NI;
6533
6534 SDNode *N = &*NI;
6535 if (N->use_empty() && N != getRoot().getNode()) {
6536 ++NI;
6537 DeleteNode(N);
6538 continue;
6539 }
6540
6541 if (LegalizedNodes.insert(Ptr: N).second) {
6542 AnyLegalized = true;
6543 Legalizer.LegalizeOp(Node: N);
6544
6545 if (N->use_empty() && N != getRoot().getNode()) {
6546 ++NI;
6547 DeleteNode(N);
6548 }
6549 }
6550 }
6551 if (!AnyLegalized)
6552 break;
6553
6554 }
6555
6556 // Remove dead nodes now.
6557 RemoveDeadNodes();
6558}
6559
6560bool SelectionDAG::LegalizeOp(SDNode *N,
6561 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6562 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6563 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6564
6565 // Directly insert the node in question, and legalize it. This will recurse
6566 // as needed through operands.
6567 LegalizedNodes.insert(Ptr: N);
6568 Legalizer.LegalizeOp(Node: N);
6569
6570 return LegalizedNodes.count(Ptr: N);
6571}
6572