1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SelectionDAGNodes.h"
32#include "llvm/CodeGen/TargetFrameLowering.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/CodeGenTypes/MachineValueType.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Compiler.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/MathExtras.h"
49#include "llvm/Support/raw_ostream.h"
50#include "llvm/Target/TargetMachine.h"
51#include "llvm/Target/TargetOptions.h"
52#include <cassert>
53#include <cstdint>
54#include <tuple>
55#include <utility>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "legalizedag"
60
61namespace {
62
63/// Keeps track of state when getting the sign of a floating-point value as an
64/// integer.
65struct FloatSignAsInt {
66 EVT FloatVT;
67 SDValue Chain;
68 SDValue FloatPtr;
69 SDValue IntPtr;
70 MachinePointerInfo IntPointerInfo;
71 MachinePointerInfo FloatPointerInfo;
72 SDValue IntValue;
73 APInt SignMask;
74 uint8_t SignBit;
75};
76
77//===----------------------------------------------------------------------===//
78/// This takes an arbitrary SelectionDAG as input and
79/// hacks on it until the target machine can handle it. This involves
80/// eliminating value sizes the machine cannot handle (promoting small sizes to
81/// large sizes or splitting up large values into small values) as well as
82/// eliminating operations the machine cannot handle.
83///
84/// This code also does a small amount of optimization and recognition of idioms
85/// as part of its processing. For example, if a target does not support a
86/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
87/// will attempt merge setcc and brc instructions into brcc's.
88class SelectionDAGLegalize {
89 const TargetMachine &TM;
90 const TargetLowering &TLI;
91 SelectionDAG &DAG;
92
93 /// The set of nodes which have already been legalized. We hold a
94 /// reference to it in order to update as necessary on node deletion.
95 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
96
97 /// A set of all the nodes updated during legalization.
98 SmallSetVector<SDNode *, 16> *UpdatedNodes;
99
100 EVT getSetCCResultType(EVT VT) const {
101 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
102 }
103
104 // Libcall insertion helpers.
105
106public:
107 SelectionDAGLegalize(SelectionDAG &DAG,
108 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
109 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
110 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
112
113 /// Legalizes the given operation.
114 void LegalizeOp(SDNode *Node);
115
116private:
117 SDValue OptimizeFloatStore(StoreSDNode *ST);
118
119 void LegalizeLoadOps(SDNode *Node);
120 void LegalizeStoreOps(SDNode *Node);
121
122 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
123
124 /// Return a vector shuffle operation which
125 /// performs the same shuffe in terms of order or result bytes, but on a type
126 /// whose vector element type is narrower than the original shuffle type.
127 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
128 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
129 SDValue N1, SDValue N2,
130 ArrayRef<int> Mask) const;
131
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 TargetLowering::ArgListTy &&Args,
134 bool IsSigned, EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
136
137 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 SmallVectorImpl<SDValue> &Results);
139 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
140 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
141 RTLIB::Libcall Call_F128,
142 RTLIB::Libcall Call_PPCF128,
143 SmallVectorImpl<SDValue> &Results);
144
145 void
146 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
147 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
148 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
149 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
150 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
151 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
152 SmallVectorImpl<SDValue> &Results);
153
154 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
155 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
156 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
157 void ExpandArgFPLibCall(SDNode *Node,
158 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
159 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
160 RTLIB::Libcall Call_PPCF128,
161 SmallVectorImpl<SDValue> &Results);
162 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
163 RTLIB::Libcall CallI64,
164 RTLIB::Libcall CallI128);
165 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
166
167 SDValue ExpandSincosStretLibCall(SDNode *Node) const;
168
169 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
170 const SDLoc &dl);
171 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
172 const SDLoc &dl, SDValue ChainIn);
173 SDValue ExpandBUILD_VECTOR(SDNode *Node);
174 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
175 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
176 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
177 SmallVectorImpl<SDValue> &Results);
178 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
179 SDValue Value) const;
180 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
181 SDValue NewIntValue) const;
182 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
183 SDValue ExpandFABS(SDNode *Node) const;
184 SDValue ExpandFNEG(SDNode *Node) const;
185 SDValue expandLdexp(SDNode *Node) const;
186 SDValue expandFrexp(SDNode *Node) const;
187
188 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
189 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
190 SmallVectorImpl<SDValue> &Results);
191 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
192 SmallVectorImpl<SDValue> &Results);
193 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
194
195 /// Implements vector reduce operation promotion.
196 ///
197 /// All vector operands are promoted to a vector type with larger element
198 /// type, and the start value is promoted to a larger scalar type. Then the
199 /// result is truncated back to the original scalar type.
200 SDValue PromoteReduction(SDNode *Node);
201
202 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
203
204 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
205 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
206 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
207 SDValue ExpandConcatVectors(SDNode *Node);
208
209 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
210 SDValue ExpandConstant(ConstantSDNode *CP);
211
212 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
213 bool ExpandNode(SDNode *Node);
214 void ConvertNodeToLibcall(SDNode *Node);
215 void PromoteNode(SDNode *Node);
216
217public:
218 // Node replacement helpers
219
220 void ReplacedNode(SDNode *N) {
221 LegalizedNodes.erase(Ptr: N);
222 if (UpdatedNodes)
223 UpdatedNodes->insert(X: N);
224 }
225
226 void ReplaceNode(SDNode *Old, SDNode *New) {
227 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
228 dbgs() << " with: "; New->dump(&DAG));
229
230 assert(Old->getNumValues() == New->getNumValues() &&
231 "Replacing one node with another that produces a different number "
232 "of values!");
233 DAG.ReplaceAllUsesWith(From: Old, To: New);
234 if (UpdatedNodes)
235 UpdatedNodes->insert(X: New);
236 ReplacedNode(N: Old);
237 }
238
239 void ReplaceNode(SDValue Old, SDValue New) {
240 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
241 dbgs() << " with: "; New->dump(&DAG));
242
243 DAG.ReplaceAllUsesWith(From: Old, To: New);
244 if (UpdatedNodes)
245 UpdatedNodes->insert(X: New.getNode());
246 ReplacedNode(N: Old.getNode());
247 }
248
249 void ReplaceNode(SDNode *Old, const SDValue *New) {
250 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
251
252 DAG.ReplaceAllUsesWith(From: Old, To: New);
253 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
254 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
255 New[i]->dump(&DAG));
256 if (UpdatedNodes)
257 UpdatedNodes->insert(X: New[i].getNode());
258 }
259 ReplacedNode(N: Old);
260 }
261
262 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
263 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
264 dbgs() << " with: "; New->dump(&DAG));
265
266 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
267 if (UpdatedNodes)
268 UpdatedNodes->insert(X: New.getNode());
269 ReplacedNode(N: Old.getNode());
270 }
271};
272
273} // end anonymous namespace
274
275// Helper function that generates an MMO that considers the alignment of the
276// stack, and the size of the stack object
277static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
278 MachineFunction &MF,
279 bool isObjectScalable) {
280 auto &MFI = MF.getFrameInfo();
281 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
282 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
283 LocationSize ObjectSize = isObjectScalable
284 ? LocationSize::beforeOrAfterPointer()
285 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
286 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
287 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
288}
289
290/// Return a vector shuffle operation which
291/// performs the same shuffle in terms of order or result bytes, but on a type
292/// whose vector element type is narrower than the original shuffle type.
293/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
294SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
295 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
296 ArrayRef<int> Mask) const {
297 unsigned NumMaskElts = VT.getVectorNumElements();
298 unsigned NumDestElts = NVT.getVectorNumElements();
299 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
300
301 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
302
303 if (NumEltsGrowth == 1)
304 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
305
306 SmallVector<int, 8> NewMask;
307 for (unsigned i = 0; i != NumMaskElts; ++i) {
308 int Idx = Mask[i];
309 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
310 if (Idx < 0)
311 NewMask.push_back(Elt: -1);
312 else
313 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
314 }
315 }
316 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
317 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
318 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
319}
320
321/// Expands the ConstantFP node to an integer constant or
322/// a load from the constant pool.
323SDValue
324SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
325 bool Extend = false;
326 SDLoc dl(CFP);
327
328 // If a FP immediate is precise when represented as a float and if the
329 // target can do an extending load from float to double, we put it into
330 // the constant pool as a float, even if it's is statically typed as a
331 // double. This shrinks FP constants and canonicalizes them for targets where
332 // an FP extending load is the same cost as a normal load (such as on the x87
333 // fp stack or PPC FP unit).
334 EVT VT = CFP->getValueType(ResNo: 0);
335 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
336 if (!UseCP) {
337 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
338 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
339 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
340 }
341
342 APFloat APF = CFP->getValueAPF();
343 EVT OrigVT = VT;
344 EVT SVT = VT;
345
346 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
347 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
348 if (!APF.isSignaling()) {
349 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
350 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
351 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
352 // Only do this if the target has a native EXTLOAD instruction from
353 // smaller type.
354 TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: OrigVT, MemVT: SVT) &&
355 TLI.ShouldShrinkFPConstant(OrigVT)) {
356 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
357 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
358 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
359 VT = SVT;
360 Extend = true;
361 }
362 }
363 }
364
365 SDValue CPIdx =
366 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
367 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
368 if (Extend) {
369 SDValue Result = DAG.getExtLoad(
370 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
371 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
372 Alignment);
373 return Result;
374 }
375 SDValue Result = DAG.getLoad(
376 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
377 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
378 return Result;
379}
380
381/// Expands the Constant node to a load from the constant pool.
382SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
383 SDLoc dl(CP);
384 EVT VT = CP->getValueType(ResNo: 0);
385 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
386 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
387 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
388 SDValue Result = DAG.getLoad(
389 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
390 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
391 return Result;
392}
393
394SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
395 SDValue Vec = Op.getOperand(i: 0);
396 SDValue Val = Op.getOperand(i: 1);
397 SDValue Idx = Op.getOperand(i: 2);
398 SDLoc dl(Op);
399
400 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
401 // SCALAR_TO_VECTOR requires that the type of the value being inserted
402 // match the element type of the vector being created, except for
403 // integers in which case the inserted value can be over width.
404 EVT EltVT = Vec.getValueType().getVectorElementType();
405 if (Val.getValueType() == EltVT ||
406 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
407 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
408 VT: Vec.getValueType(), Operand: Val);
409
410 unsigned NumElts = Vec.getValueType().getVectorNumElements();
411 // We generate a shuffle of InVec and ScVec, so the shuffle mask
412 // should be 0,1,2,3,4,5... with the appropriate element replaced with
413 // elt 0 of the RHS.
414 SmallVector<int, 8> ShufOps;
415 for (unsigned i = 0; i != NumElts; ++i)
416 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
417
418 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
419 }
420 }
421 return ExpandInsertToVectorThroughStack(Op);
422}
423
424SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
425 if (!ISD::isNormalStore(N: ST))
426 return SDValue();
427
428 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
429 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
430 // FIXME: move this to the DAG Combiner! Note that we can't regress due
431 // to phase ordering between legalized code and the dag combiner. This
432 // probably means that we need to integrate dag combiner and legalizer
433 // together.
434 // We generally can't do this one for long doubles.
435 SDValue Chain = ST->getChain();
436 SDValue Ptr = ST->getBasePtr();
437 SDValue Value = ST->getValue();
438 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
439 AAMDNodes AAInfo = ST->getAAInfo();
440 SDLoc dl(ST);
441
442 // Don't optimise TargetConstantFP
443 if (Value.getOpcode() == ISD::TargetConstantFP)
444 return SDValue();
445
446 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
447 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
448 TLI.isTypeLegal(VT: MVT::i32)) {
449 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
450 bitcastToAPInt().zextOrTrunc(width: 32),
451 DL: SDLoc(CFP), VT: MVT::i32);
452 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
453 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
454 }
455
456 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
457 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
458 // If this target supports 64-bit registers, do a single 64-bit store.
459 if (TLI.isTypeLegal(VT: MVT::i64)) {
460 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
461 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
462 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
463 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
464 }
465
466 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
467 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
468 // stores. If the target supports neither 32- nor 64-bits, this
469 // xform is certainly not worth it.
470 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
471 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
472 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
473 if (DAG.getDataLayout().isBigEndian())
474 std::swap(a&: Lo, b&: Hi);
475
476 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
477 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
478 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
479 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
480 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
481 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
482
483 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
484 }
485 }
486 }
487 return SDValue();
488}
489
490void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
491 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
492 SDValue Chain = ST->getChain();
493 SDValue Ptr = ST->getBasePtr();
494 SDLoc dl(Node);
495
496 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
497 AAMDNodes AAInfo = ST->getAAInfo();
498
499 if (!ST->isTruncatingStore()) {
500 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
501 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
502 ReplaceNode(Old: ST, New: OptStore);
503 return;
504 }
505
506 SDValue Value = ST->getValue();
507 MVT VT = Value.getSimpleValueType();
508 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
509 default: llvm_unreachable("This action is not supported yet!");
510 case TargetLowering::Legal: {
511 // If this is an unaligned store and the target doesn't support it,
512 // expand it.
513 EVT MemVT = ST->getMemoryVT();
514 const DataLayout &DL = DAG.getDataLayout();
515 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
516 MMO: *ST->getMemOperand())) {
517 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
518 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
519 ReplaceNode(Old: SDValue(ST, 0), New: Result);
520 } else
521 LLVM_DEBUG(dbgs() << "Legal store\n");
522 break;
523 }
524 case TargetLowering::Custom: {
525 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
526 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
527 if (Res && Res != SDValue(Node, 0))
528 ReplaceNode(Old: SDValue(Node, 0), New: Res);
529 return;
530 }
531 case TargetLowering::Promote: {
532 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
533 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
534 "Can only promote stores to same size type");
535 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
536 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
537 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
538 ReplaceNode(Old: SDValue(Node, 0), New: Result);
539 break;
540 }
541 }
542 return;
543 }
544
545 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
546 SDValue Value = ST->getValue();
547 EVT StVT = ST->getMemoryVT();
548 TypeSize StWidth = StVT.getSizeInBits();
549 TypeSize StSize = StVT.getStoreSizeInBits();
550 auto &DL = DAG.getDataLayout();
551
552 if (StWidth != StSize) {
553 // Promote to a byte-sized store with upper bits zero if not
554 // storing an integral number of bytes. For example, promote
555 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
556 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
557 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
558 SDValue Result =
559 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
560 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
561 ReplaceNode(Old: SDValue(Node, 0), New: Result);
562 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
563 // If not storing a power-of-2 number of bits, expand as two stores.
564 assert(!StVT.isVector() && "Unsupported truncstore!");
565 unsigned StWidthBits = StWidth.getFixedValue();
566 unsigned LogStWidth = Log2_32(Value: StWidthBits);
567 assert(LogStWidth < 32);
568 unsigned RoundWidth = 1 << LogStWidth;
569 assert(RoundWidth < StWidthBits);
570 unsigned ExtraWidth = StWidthBits - RoundWidth;
571 assert(ExtraWidth < RoundWidth);
572 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
573 "Store size not an integral number of bytes!");
574 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
575 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
576 SDValue Lo, Hi;
577 unsigned IncrementSize;
578
579 if (DL.isLittleEndian()) {
580 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
581 // Store the bottom RoundWidth bits.
582 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
583 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
584
585 // Store the remaining ExtraWidth bits.
586 IncrementSize = RoundWidth / 8;
587 Ptr =
588 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
589 Hi = DAG.getNode(
590 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
591 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Value.getValueType(), DL: dl));
592 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
593 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
594 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
595 } else {
596 // Big endian - avoid unaligned stores.
597 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
598 // Store the top RoundWidth bits.
599 Hi = DAG.getNode(
600 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
601 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Value.getValueType(), DL: dl));
602 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
603 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
604
605 // Store the remaining ExtraWidth bits.
606 IncrementSize = RoundWidth / 8;
607 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
608 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
609 VT: Ptr.getValueType()));
610 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
611 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
612 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
613 }
614
615 // The order of the stores doesn't matter.
616 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
617 ReplaceNode(Old: SDValue(Node, 0), New: Result);
618 } else {
619 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT)) {
620 default: llvm_unreachable("This action is not supported yet!");
621 case TargetLowering::Legal: {
622 EVT MemVT = ST->getMemoryVT();
623 // If this is an unaligned store and the target doesn't support it,
624 // expand it.
625 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
626 MMO: *ST->getMemOperand())) {
627 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
628 ReplaceNode(Old: SDValue(ST, 0), New: Result);
629 }
630 break;
631 }
632 case TargetLowering::Custom: {
633 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
634 if (Res && Res != SDValue(Node, 0))
635 ReplaceNode(Old: SDValue(Node, 0), New: Res);
636 return;
637 }
638 case TargetLowering::Expand:
639 assert(!StVT.isVector() &&
640 "Vector Stores are handled in LegalizeVectorOps");
641
642 SDValue Result;
643
644 // TRUNCSTORE:i16 i32 -> STORE i16
645 if (TLI.isTypeLegal(VT: StVT)) {
646 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
647 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
648 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
649 } else {
650 // The in-memory type isn't legal. Truncate to the type it would promote
651 // to, and then do a truncstore.
652 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
653 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
654 Operand: Value);
655 Result = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
656 SVT: StVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
657 }
658
659 ReplaceNode(Old: SDValue(Node, 0), New: Result);
660 break;
661 }
662 }
663}
664
665void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
666 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
667 SDValue Chain = LD->getChain(); // The chain.
668 SDValue Ptr = LD->getBasePtr(); // The base pointer.
669 SDValue Value; // The value returned by the load op.
670 SDLoc dl(Node);
671
672 ISD::LoadExtType ExtType = LD->getExtensionType();
673 if (ExtType == ISD::NON_EXTLOAD) {
674 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
675 MVT VT = Node->getSimpleValueType(ResNo: 0);
676 SDValue RVal = SDValue(Node, 0);
677 SDValue RChain = SDValue(Node, 1);
678
679 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
680 default: llvm_unreachable("This action is not supported yet!");
681 case TargetLowering::Legal: {
682 EVT MemVT = LD->getMemoryVT();
683 const DataLayout &DL = DAG.getDataLayout();
684 // If this is an unaligned load and the target doesn't support it,
685 // expand it.
686 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
687 MMO: *LD->getMemOperand())) {
688 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
689 }
690 break;
691 }
692 case TargetLowering::Custom:
693 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
694 RVal = Res;
695 RChain = Res.getValue(R: 1);
696 }
697 break;
698
699 case TargetLowering::Promote: {
700 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
701 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
702 "Can only promote loads to same size type");
703
704 // If the range metadata type does not match the legalized memory
705 // operation type, remove the range metadata.
706 if (const MDNode *MD = LD->getRanges()) {
707 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
708 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
709 !NVT.isInteger())
710 LD->getMemOperand()->clearRanges();
711 }
712 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
713 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
714 RChain = Res.getValue(R: 1);
715 break;
716 }
717 }
718 if (RChain.getNode() != Node) {
719 assert(RVal.getNode() != Node && "Load must be completely replaced");
720 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
721 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
722 if (UpdatedNodes) {
723 UpdatedNodes->insert(X: RVal.getNode());
724 UpdatedNodes->insert(X: RChain.getNode());
725 }
726 ReplacedNode(N: Node);
727 }
728 return;
729 }
730
731 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
732 EVT SrcVT = LD->getMemoryVT();
733 TypeSize SrcWidth = SrcVT.getSizeInBits();
734 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
735 AAMDNodes AAInfo = LD->getAAInfo();
736
737 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
738 // Some targets pretend to have an i1 loading operation, and actually
739 // load an i8. This trick is correct for ZEXTLOAD because the top 7
740 // bits are guaranteed to be zero; it helps the optimizers understand
741 // that these bits are zero. It is also useful for EXTLOAD, since it
742 // tells the optimizers that those bits are undefined. It would be
743 // nice to have an effective generic way of getting these benefits...
744 // Until such a way is found, don't insist on promoting i1 here.
745 (SrcVT != MVT::i1 ||
746 TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1) ==
747 TargetLowering::Promote)) {
748 // Promote to a byte-sized load if not loading an integral number of
749 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
750 unsigned NewWidth = SrcVT.getStoreSizeInBits();
751 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
752 SDValue Ch;
753
754 // The extra bits are guaranteed to be zero, since we stored them that
755 // way. A zext load from NVT thus automatically gives zext from SrcVT.
756
757 ISD::LoadExtType NewExtType =
758 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
759
760 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
761 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
762 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
763
764 Ch = Result.getValue(R: 1); // The chain.
765
766 if (ExtType == ISD::SEXTLOAD)
767 // Having the top bits zero doesn't help when sign extending.
768 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
769 VT: Result.getValueType(),
770 N1: Result, N2: DAG.getValueType(SrcVT));
771 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
772 // All the top bits are guaranteed to be zero - inform the optimizers.
773 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
774 VT: Result.getValueType(), N1: Result,
775 N2: DAG.getValueType(SrcVT));
776
777 Value = Result;
778 Chain = Ch;
779 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
780 // If not loading a power-of-2 number of bits, expand as two loads.
781 assert(!SrcVT.isVector() && "Unsupported extload!");
782 unsigned SrcWidthBits = SrcWidth.getFixedValue();
783 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
784 assert(LogSrcWidth < 32);
785 unsigned RoundWidth = 1 << LogSrcWidth;
786 assert(RoundWidth < SrcWidthBits);
787 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
788 assert(ExtraWidth < RoundWidth);
789 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
790 "Load size not an integral number of bytes!");
791 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
792 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
793 SDValue Lo, Hi, Ch;
794 unsigned IncrementSize;
795 auto &DL = DAG.getDataLayout();
796
797 if (DL.isLittleEndian()) {
798 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
799 // Load the bottom RoundWidth bits.
800 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
801 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
802 MMOFlags, AAInfo);
803
804 // Load the remaining ExtraWidth bits.
805 IncrementSize = RoundWidth / 8;
806 Ptr =
807 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
808 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
809 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
810 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
811
812 // Build a factor node to remember that this load is independent of
813 // the other one.
814 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
815 N2: Hi.getValue(R: 1));
816
817 // Move the top bits to the right place.
818 Hi = DAG.getNode(
819 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
820 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Hi.getValueType(), DL: dl));
821
822 // Join the hi and lo parts.
823 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
824 } else {
825 // Big endian - avoid unaligned loads.
826 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
827 // Load the top RoundWidth bits.
828 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
829 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
830 MMOFlags, AAInfo);
831
832 // Load the remaining ExtraWidth bits.
833 IncrementSize = RoundWidth / 8;
834 Ptr =
835 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
836 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
837 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
838 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
839
840 // Build a factor node to remember that this load is independent of
841 // the other one.
842 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
843 N2: Hi.getValue(R: 1));
844
845 // Move the top bits to the right place.
846 Hi = DAG.getNode(
847 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
848 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Hi.getValueType(), DL: dl));
849
850 // Join the hi and lo parts.
851 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
852 }
853
854 Chain = Ch;
855 } else {
856 bool isCustom = false;
857 switch (TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0),
858 MemVT: SrcVT.getSimpleVT())) {
859 default: llvm_unreachable("This action is not supported yet!");
860 case TargetLowering::Custom:
861 isCustom = true;
862 [[fallthrough]];
863 case TargetLowering::Legal:
864 Value = SDValue(Node, 0);
865 Chain = SDValue(Node, 1);
866
867 if (isCustom) {
868 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
869 Value = Res;
870 Chain = Res.getValue(R: 1);
871 }
872 } else {
873 // If this is an unaligned load and the target doesn't support it,
874 // expand it.
875 EVT MemVT = LD->getMemoryVT();
876 const DataLayout &DL = DAG.getDataLayout();
877 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
878 MMO: *LD->getMemOperand())) {
879 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
880 }
881 }
882 break;
883
884 case TargetLowering::Expand: {
885 EVT DestVT = Node->getValueType(ResNo: 0);
886 if (!TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SrcVT)) {
887 // If the source type is not legal, see if there is a legal extload to
888 // an intermediate type that we can then extend further.
889 EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT());
890 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
891 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
892 TLI.isLoadExtLegal(ExtType, ValVT: LoadVT, MemVT: SrcVT))) {
893 // If we are loading a legal type, this is a non-extload followed by a
894 // full extend.
895 ISD::LoadExtType MidExtType =
896 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
897
898 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
899 MemVT: SrcVT, MMO: LD->getMemOperand());
900 unsigned ExtendOp =
901 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
902 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
903 Chain = Load.getValue(R: 1);
904 break;
905 }
906
907 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
908 // normal undefined upper bits behavior to allow using an in-reg extend
909 // with the illegal FP type, so load as an integer and do the
910 // from-integer conversion.
911 EVT SVT = SrcVT.getScalarType();
912 if (SVT == MVT::f16 || SVT == MVT::bf16) {
913 EVT ISrcVT = SrcVT.changeTypeToInteger();
914 EVT IDestVT = DestVT.changeTypeToInteger();
915 EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT());
916
917 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
918 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
919 Value =
920 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
921 DL: dl, VT: DestVT, Operand: Result);
922 Chain = Result.getValue(R: 1);
923 break;
924 }
925 }
926
927 assert(!SrcVT.isVector() &&
928 "Vector Loads are handled in LegalizeVectorOps");
929
930 // FIXME: This does not work for vectors on most targets. Sign-
931 // and zero-extend operations are currently folded into extending
932 // loads, whether they are legal or not, and then we end up here
933 // without any support for legalizing them.
934 assert(ExtType != ISD::EXTLOAD &&
935 "EXTLOAD should always be supported!");
936 // Turn the unsupported load into an EXTLOAD followed by an
937 // explicit zero/sign extend inreg.
938 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
939 VT: Node->getValueType(ResNo: 0),
940 Chain, Ptr, MemVT: SrcVT,
941 MMO: LD->getMemOperand());
942 SDValue ValRes;
943 if (ExtType == ISD::SEXTLOAD)
944 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
945 VT: Result.getValueType(),
946 N1: Result, N2: DAG.getValueType(SrcVT));
947 else
948 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
949 Value = ValRes;
950 Chain = Result.getValue(R: 1);
951 break;
952 }
953 }
954 }
955
956 // Since loads produce two values, make sure to remember that we legalized
957 // both of them.
958 if (Chain.getNode() != Node) {
959 assert(Value.getNode() != Node && "Load must be completely replaced");
960 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
961 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
962 if (UpdatedNodes) {
963 UpdatedNodes->insert(X: Value.getNode());
964 UpdatedNodes->insert(X: Chain.getNode());
965 }
966 ReplacedNode(N: Node);
967 }
968}
969
970/// Return a legal replacement for the given operation, with all legal operands.
971void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
972 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
973
974 // Allow illegal target nodes and illegal registers.
975 if (Node->getOpcode() == ISD::TargetConstant ||
976 Node->getOpcode() == ISD::Register)
977 return;
978
979#ifndef NDEBUG
980 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
981 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
982 TargetLowering::TypeLegal &&
983 "Unexpected illegal type!");
984
985 for (const SDValue &Op : Node->op_values())
986 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
987 TargetLowering::TypeLegal ||
988 Op.getOpcode() == ISD::TargetConstant ||
989 Op.getOpcode() == ISD::Register) &&
990 "Unexpected illegal type!");
991#endif
992
993 // Figure out the correct action; the way to query this varies by opcode
994 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
995 bool SimpleFinishLegalizing = true;
996 switch (Node->getOpcode()) {
997 case ISD::POISON: {
998 // TODO: Currently, POISON is being lowered to UNDEF here. However, there is
999 // an open concern that this transformation may not be ideal, as targets
1000 // should ideally handle POISON directly. Changing this behavior would
1001 // require adding support for POISON in TableGen, which is a large change.
1002 // Additionally, many existing test cases rely on the current behavior
1003 // (e.g., llvm/test/CodeGen/PowerPC/vec_shuffle.ll). A broader discussion
1004 // and incremental changes might be needed to properly support POISON
1005 // without breaking existing targets and tests.
1006 SDValue UndefNode = DAG.getUNDEF(VT: Node->getValueType(ResNo: 0));
1007 ReplaceNode(Old: Node, New: UndefNode.getNode());
1008 return;
1009 }
1010 case ISD::INTRINSIC_W_CHAIN:
1011 case ISD::INTRINSIC_WO_CHAIN:
1012 case ISD::INTRINSIC_VOID:
1013 case ISD::STACKSAVE:
1014 case ISD::STACKADDRESS:
1015 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1016 break;
1017 case ISD::GET_DYNAMIC_AREA_OFFSET:
1018 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1019 VT: Node->getValueType(ResNo: 0));
1020 break;
1021 case ISD::VAARG:
1022 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1023 VT: Node->getValueType(ResNo: 0));
1024 if (Action != TargetLowering::Promote)
1025 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1026 break;
1027 case ISD::SET_FPENV:
1028 case ISD::SET_FPMODE:
1029 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1030 VT: Node->getOperand(Num: 1).getValueType());
1031 break;
1032 case ISD::FP_TO_FP16:
1033 case ISD::FP_TO_BF16:
1034 case ISD::SINT_TO_FP:
1035 case ISD::UINT_TO_FP:
1036 case ISD::EXTRACT_VECTOR_ELT:
1037 case ISD::LROUND:
1038 case ISD::LLROUND:
1039 case ISD::LRINT:
1040 case ISD::LLRINT:
1041 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1042 VT: Node->getOperand(Num: 0).getValueType());
1043 break;
1044 case ISD::STRICT_FP_TO_FP16:
1045 case ISD::STRICT_FP_TO_BF16:
1046 case ISD::STRICT_SINT_TO_FP:
1047 case ISD::STRICT_UINT_TO_FP:
1048 case ISD::STRICT_LRINT:
1049 case ISD::STRICT_LLRINT:
1050 case ISD::STRICT_LROUND:
1051 case ISD::STRICT_LLROUND:
1052 // These pseudo-ops are the same as the other STRICT_ ops except
1053 // they are registered with setOperationAction() using the input type
1054 // instead of the output type.
1055 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1056 VT: Node->getOperand(Num: 1).getValueType());
1057 break;
1058 case ISD::SIGN_EXTEND_INREG: {
1059 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1060 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1061 break;
1062 }
1063 case ISD::ATOMIC_STORE:
1064 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1065 VT: Node->getOperand(Num: 1).getValueType());
1066 break;
1067 case ISD::SELECT_CC:
1068 case ISD::STRICT_FSETCC:
1069 case ISD::STRICT_FSETCCS:
1070 case ISD::SETCC:
1071 case ISD::SETCCCARRY:
1072 case ISD::VP_SETCC:
1073 case ISD::BR_CC: {
1074 unsigned Opc = Node->getOpcode();
1075 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1076 : Opc == ISD::STRICT_FSETCC ? 3
1077 : Opc == ISD::STRICT_FSETCCS ? 3
1078 : Opc == ISD::SETCCCARRY ? 3
1079 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1080 : 1;
1081 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1082 : Opc == ISD::STRICT_FSETCC ? 1
1083 : Opc == ISD::STRICT_FSETCCS ? 1
1084 : 0;
1085 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1086 ISD::CondCode CCCode =
1087 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1088 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1089 if (Action == TargetLowering::Legal) {
1090 if (Node->getOpcode() == ISD::SELECT_CC)
1091 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1092 VT: Node->getValueType(ResNo: 0));
1093 else
1094 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1095 }
1096 break;
1097 }
1098 case ISD::LOAD:
1099 case ISD::STORE:
1100 // FIXME: Model these properly. LOAD and STORE are complicated, and
1101 // STORE expects the unlegalized operand in some cases.
1102 SimpleFinishLegalizing = false;
1103 break;
1104 case ISD::CALLSEQ_START:
1105 case ISD::CALLSEQ_END:
1106 // FIXME: This shouldn't be necessary. These nodes have special properties
1107 // dealing with the recursive nature of legalization. Removing this
1108 // special case should be done as part of making LegalizeDAG non-recursive.
1109 SimpleFinishLegalizing = false;
1110 break;
1111 case ISD::EXTRACT_ELEMENT:
1112 case ISD::GET_ROUNDING:
1113 case ISD::MERGE_VALUES:
1114 case ISD::EH_RETURN:
1115 case ISD::FRAME_TO_ARGS_OFFSET:
1116 case ISD::EH_DWARF_CFA:
1117 case ISD::EH_SJLJ_SETJMP:
1118 case ISD::EH_SJLJ_LONGJMP:
1119 case ISD::EH_SJLJ_SETUP_DISPATCH:
1120 // These operations lie about being legal: when they claim to be legal,
1121 // they should actually be expanded.
1122 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1123 if (Action == TargetLowering::Legal)
1124 Action = TargetLowering::Expand;
1125 break;
1126 case ISD::INIT_TRAMPOLINE:
1127 case ISD::ADJUST_TRAMPOLINE:
1128 case ISD::FRAMEADDR:
1129 case ISD::RETURNADDR:
1130 case ISD::ADDROFRETURNADDR:
1131 case ISD::SPONENTRY:
1132 // These operations lie about being legal: when they claim to be legal,
1133 // they should actually be custom-lowered.
1134 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1135 if (Action == TargetLowering::Legal)
1136 Action = TargetLowering::Custom;
1137 break;
1138 case ISD::CLEAR_CACHE:
1139 // This operation is typically going to be LibCall unless the target wants
1140 // something differrent.
1141 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1142 break;
1143 case ISD::READCYCLECOUNTER:
1144 case ISD::READSTEADYCOUNTER:
1145 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1146 // legalization might have expanded that to several smaller types.
1147 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1148 break;
1149 case ISD::READ_REGISTER:
1150 case ISD::WRITE_REGISTER:
1151 // Named register is legal in the DAG, but blocked by register name
1152 // selection if not implemented by target (to chose the correct register)
1153 // They'll be converted to Copy(To/From)Reg.
1154 Action = TargetLowering::Legal;
1155 break;
1156 case ISD::UBSANTRAP:
1157 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1158 if (Action == TargetLowering::Expand) {
1159 // replace ISD::UBSANTRAP with ISD::TRAP
1160 SDValue NewVal;
1161 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1162 N: Node->getOperand(Num: 0));
1163 ReplaceNode(Old: Node, New: NewVal.getNode());
1164 LegalizeOp(Node: NewVal.getNode());
1165 return;
1166 }
1167 break;
1168 case ISD::DEBUGTRAP:
1169 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1170 if (Action == TargetLowering::Expand) {
1171 // replace ISD::DEBUGTRAP with ISD::TRAP
1172 SDValue NewVal;
1173 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1174 N: Node->getOperand(Num: 0));
1175 ReplaceNode(Old: Node, New: NewVal.getNode());
1176 LegalizeOp(Node: NewVal.getNode());
1177 return;
1178 }
1179 break;
1180 case ISD::SADDSAT:
1181 case ISD::UADDSAT:
1182 case ISD::SSUBSAT:
1183 case ISD::USUBSAT:
1184 case ISD::SSHLSAT:
1185 case ISD::USHLSAT:
1186 case ISD::SCMP:
1187 case ISD::UCMP:
1188 case ISD::FP_TO_SINT_SAT:
1189 case ISD::FP_TO_UINT_SAT:
1190 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1191 break;
1192 case ISD::SMULFIX:
1193 case ISD::SMULFIXSAT:
1194 case ISD::UMULFIX:
1195 case ISD::UMULFIXSAT:
1196 case ISD::SDIVFIX:
1197 case ISD::SDIVFIXSAT:
1198 case ISD::UDIVFIX:
1199 case ISD::UDIVFIXSAT: {
1200 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1201 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1202 VT: Node->getValueType(ResNo: 0), Scale);
1203 break;
1204 }
1205 case ISD::MSCATTER:
1206 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1207 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1208 break;
1209 case ISD::MSTORE:
1210 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1211 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1212 break;
1213 case ISD::VP_SCATTER:
1214 Action = TLI.getOperationAction(
1215 Op: Node->getOpcode(),
1216 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1217 break;
1218 case ISD::VP_STORE:
1219 Action = TLI.getOperationAction(
1220 Op: Node->getOpcode(),
1221 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1222 break;
1223 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1224 Action = TLI.getOperationAction(
1225 Op: Node->getOpcode(),
1226 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1227 break;
1228 case ISD::VECREDUCE_FADD:
1229 case ISD::VECREDUCE_FMUL:
1230 case ISD::VECREDUCE_ADD:
1231 case ISD::VECREDUCE_MUL:
1232 case ISD::VECREDUCE_AND:
1233 case ISD::VECREDUCE_OR:
1234 case ISD::VECREDUCE_XOR:
1235 case ISD::VECREDUCE_SMAX:
1236 case ISD::VECREDUCE_SMIN:
1237 case ISD::VECREDUCE_UMAX:
1238 case ISD::VECREDUCE_UMIN:
1239 case ISD::VECREDUCE_FMAX:
1240 case ISD::VECREDUCE_FMIN:
1241 case ISD::VECREDUCE_FMAXIMUM:
1242 case ISD::VECREDUCE_FMINIMUM:
1243 case ISD::IS_FPCLASS:
1244 Action = TLI.getOperationAction(
1245 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1246 break;
1247 case ISD::VECREDUCE_SEQ_FADD:
1248 case ISD::VECREDUCE_SEQ_FMUL:
1249 case ISD::VP_REDUCE_FADD:
1250 case ISD::VP_REDUCE_FMUL:
1251 case ISD::VP_REDUCE_ADD:
1252 case ISD::VP_REDUCE_MUL:
1253 case ISD::VP_REDUCE_AND:
1254 case ISD::VP_REDUCE_OR:
1255 case ISD::VP_REDUCE_XOR:
1256 case ISD::VP_REDUCE_SMAX:
1257 case ISD::VP_REDUCE_SMIN:
1258 case ISD::VP_REDUCE_UMAX:
1259 case ISD::VP_REDUCE_UMIN:
1260 case ISD::VP_REDUCE_FMAX:
1261 case ISD::VP_REDUCE_FMIN:
1262 case ISD::VP_REDUCE_FMAXIMUM:
1263 case ISD::VP_REDUCE_FMINIMUM:
1264 case ISD::VP_REDUCE_SEQ_FADD:
1265 case ISD::VP_REDUCE_SEQ_FMUL:
1266 Action = TLI.getOperationAction(
1267 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1268 break;
1269 case ISD::VP_CTTZ_ELTS:
1270 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1271 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1272 VT: Node->getOperand(Num: 0).getValueType());
1273 break;
1274 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1275 Action = TLI.getOperationAction(
1276 Op: Node->getOpcode(),
1277 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1278 break;
1279 default:
1280 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1281 Action = TLI.getCustomOperationAction(Op&: *Node);
1282 } else {
1283 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1284 }
1285 break;
1286 }
1287
1288 if (SimpleFinishLegalizing) {
1289 SDNode *NewNode = Node;
1290 switch (Node->getOpcode()) {
1291 default: break;
1292 case ISD::SHL:
1293 case ISD::SRL:
1294 case ISD::SRA:
1295 case ISD::ROTL:
1296 case ISD::ROTR:
1297 case ISD::SSHLSAT:
1298 case ISD::USHLSAT: {
1299 // Legalizing shifts/rotates requires adjusting the shift amount
1300 // to the appropriate width.
1301 SDValue Op0 = Node->getOperand(Num: 0);
1302 SDValue Op1 = Node->getOperand(Num: 1);
1303 if (!Op1.getValueType().isVector()) {
1304 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1305 // The getShiftAmountOperand() may create a new operand node or
1306 // return the existing one. If new operand is created we need
1307 // to update the parent node.
1308 // Do not try to legalize SAO here! It will be automatically legalized
1309 // in the next round.
1310 if (SAO != Op1)
1311 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1312 }
1313 break;
1314 }
1315 case ISD::FSHL:
1316 case ISD::FSHR:
1317 case ISD::SRL_PARTS:
1318 case ISD::SRA_PARTS:
1319 case ISD::SHL_PARTS: {
1320 // Legalizing shifts/rotates requires adjusting the shift amount
1321 // to the appropriate width.
1322 SDValue Op0 = Node->getOperand(Num: 0);
1323 SDValue Op1 = Node->getOperand(Num: 1);
1324 SDValue Op2 = Node->getOperand(Num: 2);
1325 if (!Op2.getValueType().isVector()) {
1326 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1327 // The getShiftAmountOperand() may create a new operand node or
1328 // return the existing one. If new operand is created we need
1329 // to update the parent node.
1330 if (SAO != Op2)
1331 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1332 }
1333 break;
1334 }
1335 }
1336
1337 if (NewNode != Node) {
1338 ReplaceNode(Old: Node, New: NewNode);
1339 Node = NewNode;
1340 }
1341 switch (Action) {
1342 case TargetLowering::Legal:
1343 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1344 return;
1345 case TargetLowering::Custom:
1346 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1347 // FIXME: The handling for custom lowering with multiple results is
1348 // a complete mess.
1349 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1350 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1351 return;
1352
1353 if (Node->getNumValues() == 1) {
1354 // Verify the new types match the original. Glue is waived because
1355 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1356 assert((Res.getValueType() == Node->getValueType(0) ||
1357 Node->getValueType(0) == MVT::Glue) &&
1358 "Type mismatch for custom legalized operation");
1359 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1360 // We can just directly replace this node with the lowered value.
1361 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1362 return;
1363 }
1364
1365 SmallVector<SDValue, 8> ResultVals;
1366 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1367 // Verify the new types match the original. Glue is waived because
1368 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1369 assert((Res->getValueType(i) == Node->getValueType(i) ||
1370 Node->getValueType(i) == MVT::Glue) &&
1371 "Type mismatch for custom legalized operation");
1372 ResultVals.push_back(Elt: Res.getValue(R: i));
1373 }
1374 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1375 ReplaceNode(Old: Node, New: ResultVals.data());
1376 return;
1377 }
1378 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1379 [[fallthrough]];
1380 case TargetLowering::Expand:
1381 if (ExpandNode(Node))
1382 return;
1383 [[fallthrough]];
1384 case TargetLowering::LibCall:
1385 ConvertNodeToLibcall(Node);
1386 return;
1387 case TargetLowering::Promote:
1388 PromoteNode(Node);
1389 return;
1390 }
1391 }
1392
1393 switch (Node->getOpcode()) {
1394 default:
1395#ifndef NDEBUG
1396 dbgs() << "NODE: ";
1397 Node->dump( &DAG);
1398 dbgs() << "\n";
1399#endif
1400 llvm_unreachable("Do not know how to legalize this operator!");
1401
1402 case ISD::CALLSEQ_START:
1403 case ISD::CALLSEQ_END:
1404 break;
1405 case ISD::LOAD:
1406 return LegalizeLoadOps(Node);
1407 case ISD::STORE:
1408 return LegalizeStoreOps(Node);
1409 }
1410}
1411
1412SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1413 SDValue Vec = Op.getOperand(i: 0);
1414 SDValue Idx = Op.getOperand(i: 1);
1415 SDLoc dl(Op);
1416
1417 // Before we generate a new store to a temporary stack slot, see if there is
1418 // already one that we can use. There often is because when we scalarize
1419 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1420 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1421 // the vector. If all are expanded here, we don't want one store per vector
1422 // element.
1423
1424 // Caches for hasPredecessorHelper
1425 SmallPtrSet<const SDNode *, 32> Visited;
1426 SmallVector<const SDNode *, 16> Worklist;
1427 Visited.insert(Ptr: Op.getNode());
1428 Worklist.push_back(Elt: Idx.getNode());
1429 SDValue StackPtr, Ch;
1430 for (SDNode *User : Vec.getNode()->users()) {
1431 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1432 if (ST->isIndexed() || ST->isTruncatingStore() ||
1433 ST->getValue() != Vec)
1434 continue;
1435
1436 // Make sure that nothing else could have stored into the destination of
1437 // this store.
1438 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1439 continue;
1440
1441 // If the index is dependent on the store we will introduce a cycle when
1442 // creating the load (the load uses the index, and by replacing the chain
1443 // we will make the index dependent on the load). Also, the store might be
1444 // dependent on the extractelement and introduce a cycle when creating
1445 // the load.
1446 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1447 ST->hasPredecessor(N: Op.getNode()))
1448 continue;
1449
1450 StackPtr = ST->getBasePtr();
1451 Ch = SDValue(ST, 0);
1452 break;
1453 }
1454 }
1455
1456 EVT VecVT = Vec.getValueType();
1457
1458 if (!Ch.getNode()) {
1459 // Store the value to a temporary stack slot, then LOAD the returned part.
1460 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1461 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1462 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1463 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1464 }
1465
1466 SDValue NewLoad;
1467 Align ElementAlignment =
1468 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1469 b: DAG.getDataLayout().getPrefTypeAlign(
1470 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1471
1472 if (Op.getValueType().isVector()) {
1473 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1474 SubVecVT: Op.getValueType(), Index: Idx);
1475 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1476 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1477 } else {
1478 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1479 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1480 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1481 Alignment: ElementAlignment);
1482 }
1483
1484 // Replace the chain going out of the store, by the one out of the load.
1485 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1486
1487 // We introduced a cycle though, so update the loads operands, making sure
1488 // to use the original store's chain as an incoming chain.
1489 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1490 NewLoadOperands[0] = Ch;
1491 NewLoad =
1492 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1493 return NewLoad;
1494}
1495
1496SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1497 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1498
1499 SDValue Vec = Op.getOperand(i: 0);
1500 SDValue Part = Op.getOperand(i: 1);
1501 SDValue Idx = Op.getOperand(i: 2);
1502 SDLoc dl(Op);
1503
1504 // Store the value to a temporary stack slot, then LOAD the returned part.
1505 EVT VecVT = Vec.getValueType();
1506 EVT PartVT = Part.getValueType();
1507 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1508 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1509 MachinePointerInfo PtrInfo =
1510 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1511
1512 // First store the whole vector.
1513 Align BaseVecAlignment =
1514 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1515 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1516 Alignment: BaseVecAlignment);
1517
1518 // Freeze the index so we don't poison the clamping code we're about to emit.
1519 Idx = DAG.getFreeze(V: Idx);
1520
1521 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1522 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1523
1524 // Then store the inserted part.
1525 if (PartVT.isVector()) {
1526 SDValue SubStackPtr =
1527 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1528
1529 // Store the subvector.
1530 Ch = DAG.getStore(
1531 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1532 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1533 Alignment: PartAlignment);
1534 } else {
1535 SDValue SubStackPtr =
1536 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1537
1538 // Store the scalar value.
1539 Ch = DAG.getTruncStore(
1540 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1541 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1542 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1543 }
1544
1545 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1546 "ElementAlignment does not match!");
1547
1548 // Finally, load the updated vector.
1549 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1550 Alignment: BaseVecAlignment);
1551}
1552
1553SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1554 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1555 SDLoc DL(Node);
1556 SmallVector<SDValue, 16> Ops;
1557 unsigned NumOperands = Node->getNumOperands();
1558 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1559 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1560 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1561 EVT ElementValueType = TLI.getTypeToTransformTo(
1562 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1563 for (unsigned I = 0; I < NumOperands; ++I) {
1564 SDValue SubOp = Node->getOperand(Num: I);
1565 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1566 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1567 N1: SubOp,
1568 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1569 }
1570 }
1571 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1572}
1573
1574SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1575 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1576 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1577 "Unexpected opcode!");
1578
1579 // We can't handle this case efficiently. Allocate a sufficiently
1580 // aligned object on the stack, store each operand into it, then load
1581 // the result as a vector.
1582 // Create the stack frame object.
1583 EVT VT = Node->getValueType(ResNo: 0);
1584 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1585 : Node->getOperand(Num: 0).getValueType();
1586 SDLoc dl(Node);
1587 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1588 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1589 MachinePointerInfo PtrInfo =
1590 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1591
1592 // Emit a store of each element to the stack slot.
1593 SmallVector<SDValue, 8> Stores;
1594 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1595 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1596
1597 // If the destination vector element type of a BUILD_VECTOR is narrower than
1598 // the source element type, only store the bits necessary.
1599 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1600 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1601
1602 // Store (in the right endianness) the elements to memory.
1603 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1604 // Ignore undef elements.
1605 if (Node->getOperand(Num: i).isUndef()) continue;
1606
1607 unsigned Offset = TypeByteSize*i;
1608
1609 SDValue Idx =
1610 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1611
1612 if (Truncate)
1613 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1614 Val: Node->getOperand(Num: i), Ptr: Idx,
1615 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1616 else
1617 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1618 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1619 }
1620
1621 SDValue StoreChain;
1622 if (!Stores.empty()) // Not all undef elements?
1623 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1624 else
1625 StoreChain = DAG.getEntryNode();
1626
1627 // Result is a load from the stack slot.
1628 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1629}
1630
1631/// Bitcast a floating-point value to an integer value. Only bitcast the part
1632/// containing the sign bit if the target has no integer value capable of
1633/// holding all bits of the floating-point value.
1634void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1635 const SDLoc &DL,
1636 SDValue Value) const {
1637 EVT FloatVT = Value.getValueType();
1638 unsigned NumBits = FloatVT.getScalarSizeInBits();
1639 State.FloatVT = FloatVT;
1640 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1641 // Convert to an integer of the same size.
1642 if (TLI.isTypeLegal(VT: IVT)) {
1643 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1644 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1645 State.SignBit = NumBits - 1;
1646 return;
1647 }
1648
1649 auto &DataLayout = DAG.getDataLayout();
1650 // Store the float to memory, then load the sign part out as an integer.
1651 MVT LoadTy = TLI.getRegisterType(VT: MVT::i8);
1652 // First create a temporary that is aligned for both the load and store.
1653 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1654 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1655 // Then store the float to it.
1656 State.FloatPtr = StackPtr;
1657 MachineFunction &MF = DAG.getMachineFunction();
1658 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1659 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1660 PtrInfo: State.FloatPointerInfo);
1661
1662 SDValue IntPtr;
1663 if (DataLayout.isBigEndian()) {
1664 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1665 // Load out a legal integer with the same sign bit as the float.
1666 IntPtr = StackPtr;
1667 State.IntPointerInfo = State.FloatPointerInfo;
1668 } else {
1669 // Advance the pointer so that the loaded byte will contain the sign bit.
1670 unsigned ByteOffset = (NumBits / 8) - 1;
1671 IntPtr =
1672 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1673 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1674 Offset: ByteOffset);
1675 }
1676
1677 State.IntPtr = IntPtr;
1678 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1679 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1680 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1681 State.SignBit = 7;
1682}
1683
1684/// Replace the integer value produced by getSignAsIntValue() with a new value
1685/// and cast the result back to a floating-point type.
1686SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1687 const SDLoc &DL,
1688 SDValue NewIntValue) const {
1689 if (!State.Chain)
1690 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1691
1692 // Override the part containing the sign bit in the value stored on the stack.
1693 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1694 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1695 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1696 PtrInfo: State.FloatPointerInfo);
1697}
1698
1699SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1700 SDLoc DL(Node);
1701 SDValue Mag = Node->getOperand(Num: 0);
1702 SDValue Sign = Node->getOperand(Num: 1);
1703
1704 // Get sign bit into an integer value.
1705 FloatSignAsInt SignAsInt;
1706 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1707
1708 EVT IntVT = SignAsInt.IntValue.getValueType();
1709 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1710 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1711 N2: SignMask);
1712
1713 // If FABS is legal transform
1714 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1715 EVT FloatVT = Mag.getValueType();
1716 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1717 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1718 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1719 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1720 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1721 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1722 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1723 }
1724
1725 // Transform Mag value to integer, and clear the sign bit.
1726 FloatSignAsInt MagAsInt;
1727 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1728 EVT MagVT = MagAsInt.IntValue.getValueType();
1729 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1730 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1731 N2: ClearSignMask);
1732
1733 // Get the signbit at the right position for MagAsInt.
1734 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1735 EVT ShiftVT = IntVT;
1736 if (SignBit.getScalarValueSizeInBits() <
1737 ClearedSign.getScalarValueSizeInBits()) {
1738 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1739 ShiftVT = MagVT;
1740 }
1741 if (ShiftAmount > 0) {
1742 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1743 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1744 } else if (ShiftAmount < 0) {
1745 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1746 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1747 }
1748 if (SignBit.getScalarValueSizeInBits() >
1749 ClearedSign.getScalarValueSizeInBits()) {
1750 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1751 }
1752
1753 // Store the part with the modified sign and convert back to float.
1754 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1755 Flags: SDNodeFlags::Disjoint);
1756
1757 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1758}
1759
1760SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1761 // Get the sign bit as an integer.
1762 SDLoc DL(Node);
1763 FloatSignAsInt SignAsInt;
1764 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1765 EVT IntVT = SignAsInt.IntValue.getValueType();
1766
1767 // Flip the sign.
1768 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1769 SDValue SignFlip =
1770 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1771
1772 // Convert back to float.
1773 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1774}
1775
1776SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1777 SDLoc DL(Node);
1778 SDValue Value = Node->getOperand(Num: 0);
1779
1780 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1781 EVT FloatVT = Value.getValueType();
1782 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1783 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1784 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1785 }
1786
1787 // Transform value to integer, clear the sign bit and transform back.
1788 FloatSignAsInt ValueAsInt;
1789 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1790 EVT IntVT = ValueAsInt.IntValue.getValueType();
1791 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1792 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1793 N2: ClearSignMask);
1794 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1795}
1796
1797void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1798 SmallVectorImpl<SDValue> &Results) {
1799 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1800 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1801 " not tell us which reg is the stack pointer!");
1802 SDLoc dl(Node);
1803 EVT VT = Node->getValueType(ResNo: 0);
1804 SDValue Tmp1 = SDValue(Node, 0);
1805 SDValue Tmp2 = SDValue(Node, 1);
1806 SDValue Tmp3 = Node->getOperand(Num: 2);
1807 SDValue Chain = Tmp1.getOperand(i: 0);
1808
1809 // Chain the dynamic stack allocation so that it doesn't modify the stack
1810 // pointer when other instructions are using the stack.
1811 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1812
1813 SDValue Size = Tmp2.getOperand(i: 1);
1814 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1815 Chain = SP.getValue(R: 1);
1816 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1817 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1818 unsigned Opc =
1819 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1820 ISD::ADD : ISD::SUB;
1821
1822 Align StackAlign = TFL->getStackAlign();
1823 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1824 if (Alignment > StackAlign)
1825 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1826 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1827 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1828
1829 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1830
1831 Results.push_back(Elt: Tmp1);
1832 Results.push_back(Elt: Tmp2);
1833}
1834
1835/// Emit a store/load combination to the stack. This stores
1836/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1837/// a load from the stack slot to DestVT, extending it if needed.
1838/// The resultant code need not be legal.
1839SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1840 EVT DestVT, const SDLoc &dl) {
1841 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1842}
1843
1844SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1845 EVT DestVT, const SDLoc &dl,
1846 SDValue Chain) {
1847 EVT SrcVT = SrcOp.getValueType();
1848 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1849 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1850
1851 // Don't convert with stack if the load/store is expensive.
1852 if ((SrcVT.bitsGT(VT: SlotVT) &&
1853 !TLI.isTruncStoreLegalOrCustom(ValVT: SrcOp.getValueType(), MemVT: SlotVT)) ||
1854 (SlotVT.bitsLT(VT: DestVT) &&
1855 !TLI.isLoadExtLegalOrCustom(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SlotVT)))
1856 return SDValue();
1857
1858 // Create the stack frame object.
1859 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1860 Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1861 SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1862
1863 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1864 int SPFI = StackPtrFI->getIndex();
1865 MachinePointerInfo PtrInfo =
1866 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1867
1868 // Emit a store to the stack slot. Use a truncstore if the input value is
1869 // later than DestVT.
1870 SDValue Store;
1871
1872 if (SrcVT.bitsGT(VT: SlotVT))
1873 Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1874 SVT: SlotVT, Alignment: SrcAlign);
1875 else {
1876 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1877 Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1878 }
1879
1880 // Result is a load from the stack slot.
1881 if (SlotVT.bitsEq(VT: DestVT))
1882 return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1883
1884 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1885 return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1886 Alignment: DestAlign);
1887}
1888
1889SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1890 SDLoc dl(Node);
1891 // Create a vector sized/aligned stack slot, store the value to element #0,
1892 // then load the whole vector back out.
1893 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1894
1895 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1896 int SPFI = StackPtrFI->getIndex();
1897
1898 SDValue Ch = DAG.getTruncStore(
1899 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1900 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1901 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1902 return DAG.getLoad(
1903 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1904 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1905}
1906
1907static bool
1908ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1909 const TargetLowering &TLI, SDValue &Res) {
1910 unsigned NumElems = Node->getNumOperands();
1911 SDLoc dl(Node);
1912 EVT VT = Node->getValueType(ResNo: 0);
1913
1914 // Try to group the scalars into pairs, shuffle the pairs together, then
1915 // shuffle the pairs of pairs together, etc. until the vector has
1916 // been built. This will work only if all of the necessary shuffle masks
1917 // are legal.
1918
1919 // We do this in two phases; first to check the legality of the shuffles,
1920 // and next, assuming that all shuffles are legal, to create the new nodes.
1921 for (int Phase = 0; Phase < 2; ++Phase) {
1922 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1923 NewIntermedVals;
1924 for (unsigned i = 0; i < NumElems; ++i) {
1925 SDValue V = Node->getOperand(Num: i);
1926 if (V.isUndef())
1927 continue;
1928
1929 SDValue Vec;
1930 if (Phase)
1931 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1932 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1933 }
1934
1935 while (IntermedVals.size() > 2) {
1936 NewIntermedVals.clear();
1937 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1938 // This vector and the next vector are shuffled together (simply to
1939 // append the one to the other).
1940 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1941
1942 SmallVector<int, 16> FinalIndices;
1943 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1944 IntermedVals[i+1].second.size());
1945
1946 int k = 0;
1947 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1948 ++j, ++k) {
1949 ShuffleVec[k] = j;
1950 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1951 }
1952 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1953 ++j, ++k) {
1954 ShuffleVec[k] = NumElems + j;
1955 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1956 }
1957
1958 SDValue Shuffle;
1959 if (Phase)
1960 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1961 N2: IntermedVals[i+1].first,
1962 Mask: ShuffleVec);
1963 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1964 return false;
1965 NewIntermedVals.push_back(
1966 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1967 }
1968
1969 // If we had an odd number of defined values, then append the last
1970 // element to the array of new vectors.
1971 if ((IntermedVals.size() & 1) != 0)
1972 NewIntermedVals.push_back(Elt: IntermedVals.back());
1973
1974 IntermedVals.swap(RHS&: NewIntermedVals);
1975 }
1976
1977 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1978 "Invalid number of intermediate vectors");
1979 SDValue Vec1 = IntermedVals[0].first;
1980 SDValue Vec2;
1981 if (IntermedVals.size() > 1)
1982 Vec2 = IntermedVals[1].first;
1983 else if (Phase)
1984 Vec2 = DAG.getPOISON(VT);
1985
1986 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1987 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1988 ShuffleVec[IntermedVals[0].second[i]] = i;
1989 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1990 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1991
1992 if (Phase)
1993 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1994 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1995 return false;
1996 }
1997
1998 return true;
1999}
2000
2001/// Expand a BUILD_VECTOR node on targets that don't
2002/// support the operation, but do support the resultant vector type.
2003SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2004 unsigned NumElems = Node->getNumOperands();
2005 SDValue Value1, Value2;
2006 SDLoc dl(Node);
2007 EVT VT = Node->getValueType(ResNo: 0);
2008 EVT OpVT = Node->getOperand(Num: 0).getValueType();
2009 EVT EltVT = VT.getVectorElementType();
2010
2011 // If the only non-undef value is the low element, turn this into a
2012 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
2013 bool isOnlyLowElement = true;
2014 bool MoreThanTwoValues = false;
2015 bool isConstant = true;
2016 for (unsigned i = 0; i < NumElems; ++i) {
2017 SDValue V = Node->getOperand(Num: i);
2018 if (V.isUndef())
2019 continue;
2020 if (i > 0)
2021 isOnlyLowElement = false;
2022 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2023 isConstant = false;
2024
2025 if (!Value1.getNode()) {
2026 Value1 = V;
2027 } else if (!Value2.getNode()) {
2028 if (V != Value1)
2029 Value2 = V;
2030 } else if (V != Value1 && V != Value2) {
2031 MoreThanTwoValues = true;
2032 }
2033 }
2034
2035 if (!Value1.getNode())
2036 return DAG.getUNDEF(VT);
2037
2038 if (isOnlyLowElement)
2039 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2040
2041 // If all elements are constants, create a load from the constant pool.
2042 if (isConstant) {
2043 SmallVector<Constant*, 16> CV;
2044 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2045 if (ConstantFPSDNode *V =
2046 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2047 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2048 } else if (ConstantSDNode *V =
2049 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2050 if (OpVT==EltVT)
2051 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2052 else {
2053 // If OpVT and EltVT don't match, EltVT is not legal and the
2054 // element values have been promoted/truncated earlier. Undo this;
2055 // we don't want a v16i8 to become a v16i32 for example.
2056 const ConstantInt *CI = V->getConstantIntValue();
2057 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2058 V: CI->getZExtValue(), /*IsSigned=*/false,
2059 /*ImplicitTrunc=*/true));
2060 }
2061 } else {
2062 assert(Node->getOperand(i).isUndef());
2063 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2064 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2065 }
2066 }
2067 Constant *CP = ConstantVector::get(V: CV);
2068 SDValue CPIdx =
2069 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2070 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2071 return DAG.getLoad(
2072 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2073 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2074 Alignment);
2075 }
2076
2077 SmallSet<SDValue, 16> DefinedValues;
2078 for (unsigned i = 0; i < NumElems; ++i) {
2079 if (Node->getOperand(Num: i).isUndef())
2080 continue;
2081 DefinedValues.insert(V: Node->getOperand(Num: i));
2082 }
2083
2084 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2085 if (!MoreThanTwoValues) {
2086 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2087 for (unsigned i = 0; i < NumElems; ++i) {
2088 SDValue V = Node->getOperand(Num: i);
2089 if (V.isUndef())
2090 continue;
2091 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2092 }
2093 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2094 // Get the splatted value into the low element of a vector register.
2095 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2096 SDValue Vec2;
2097 if (Value2.getNode())
2098 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2099 else
2100 Vec2 = DAG.getPOISON(VT);
2101
2102 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2103 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2104 }
2105 } else {
2106 SDValue Res;
2107 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2108 return Res;
2109 }
2110 }
2111
2112 // Otherwise, we can't handle this case efficiently.
2113 return ExpandVectorBuildThroughStack(Node);
2114}
2115
2116SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2117 SDLoc DL(Node);
2118 EVT VT = Node->getValueType(ResNo: 0);
2119 SDValue SplatVal = Node->getOperand(Num: 0);
2120
2121 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2122}
2123
2124// Expand a node into a call to a libcall, returning the value as the first
2125// result and the chain as the second. If the result value does not fit into a
2126// register, return the lo part and set the hi part to the by-reg argument in
2127// the first. If it does fit into a single register, return the result and
2128// leave the Hi part unset.
2129std::pair<SDValue, SDValue>
2130SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2131 TargetLowering::ArgListTy &&Args,
2132 bool IsSigned, EVT RetVT) {
2133 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2134 SDValue Callee;
2135 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2136 if (LCImpl != RTLIB::Unsupported)
2137 Callee = DAG.getExternalSymbol(LCImpl, VT: CodePtrTy);
2138 else {
2139 Callee = DAG.getPOISON(VT: CodePtrTy);
2140 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2141 Node->getOperationName(G: &DAG));
2142 }
2143
2144 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2145
2146 // By default, the input chain to this libcall is the entry node of the
2147 // function. If the libcall is going to be emitted as a tail call then
2148 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2149 // node which is being folded has a non-entry input chain.
2150 SDValue InChain = DAG.getEntryNode();
2151
2152 // isTailCall may be true since the callee does not reference caller stack
2153 // frame. Check if it's in the right position and that the return types match.
2154 SDValue TCChain = InChain;
2155 const Function &F = DAG.getMachineFunction().getFunction();
2156 bool isTailCall =
2157 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2158 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2159 if (isTailCall)
2160 InChain = TCChain;
2161
2162 TargetLowering::CallLoweringInfo CLI(DAG);
2163 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2164 CLI.setDebugLoc(SDLoc(Node))
2165 .setChain(InChain)
2166 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
2167 Target: Callee, ArgsList: std::move(Args))
2168 .setTailCall(isTailCall)
2169 .setSExtResult(signExtend)
2170 .setZExtResult(!signExtend)
2171 .setIsPostTypeLegalization(true);
2172
2173 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2174
2175 if (!CallInfo.second.getNode()) {
2176 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2177 // It's a tailcall, return the chain (which is the DAG root).
2178 return {DAG.getRoot(), DAG.getRoot()};
2179 }
2180
2181 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2182 return CallInfo;
2183}
2184
2185std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2186 bool isSigned) {
2187 TargetLowering::ArgListTy Args;
2188 for (const SDValue &Op : Node->op_values()) {
2189 EVT ArgVT = Op.getValueType();
2190 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2191 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2192 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2193 Entry.IsZExt = !Entry.IsSExt;
2194 Args.push_back(x: Entry);
2195 }
2196
2197 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2198 RetVT: Node->getValueType(ResNo: 0));
2199}
2200
2201void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2202 RTLIB::Libcall LC,
2203 SmallVectorImpl<SDValue> &Results) {
2204 if (LC == RTLIB::UNKNOWN_LIBCALL)
2205 llvm_unreachable("Can't create an unknown libcall!");
2206
2207 if (Node->isStrictFPOpcode()) {
2208 EVT RetVT = Node->getValueType(ResNo: 0);
2209 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2210 TargetLowering::MakeLibCallOptions CallOptions;
2211 CallOptions.IsPostTypeLegalization = true;
2212 // FIXME: This doesn't support tail calls.
2213 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2214 Ops, CallOptions,
2215 dl: SDLoc(Node),
2216 Chain: Node->getOperand(Num: 0));
2217 Results.push_back(Elt: Tmp.first);
2218 Results.push_back(Elt: Tmp.second);
2219 } else {
2220 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2221 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2222 Results.push_back(Elt: Tmp);
2223 }
2224}
2225
2226/// Expand the node to a libcall based on the result type.
2227void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2228 RTLIB::Libcall Call_F32,
2229 RTLIB::Libcall Call_F64,
2230 RTLIB::Libcall Call_F80,
2231 RTLIB::Libcall Call_F128,
2232 RTLIB::Libcall Call_PPCF128,
2233 SmallVectorImpl<SDValue> &Results) {
2234 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0),
2235 Call_F32, Call_F64, Call_F80,
2236 Call_F128, Call_PPCF128);
2237 ExpandFPLibCall(Node, LC, Results);
2238}
2239
2240void SelectionDAGLegalize::ExpandFastFPLibCall(
2241 SDNode *Node, bool IsFast,
2242 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2243 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2244 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2245 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2246 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2247 SmallVectorImpl<SDValue> &Results) {
2248
2249 EVT VT = Node->getSimpleValueType(ResNo: 0);
2250
2251 RTLIB::Libcall LC;
2252
2253 // FIXME: Probably should define fast to respect nan/inf and only be
2254 // approximate functions.
2255
2256 if (IsFast) {
2257 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.first, Call_F64: Call_F64.first, Call_F80: Call_F80.first,
2258 Call_F128: Call_F128.first, Call_PPCF128: Call_PPCF128.first);
2259 }
2260
2261 if (!IsFast || DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
2262 // Fall back if we don't have a fast implementation.
2263 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.second, Call_F64: Call_F64.second,
2264 Call_F80: Call_F80.second, Call_F128: Call_F128.second,
2265 Call_PPCF128: Call_PPCF128.second);
2266 }
2267
2268 ExpandFPLibCall(Node, LC, Results);
2269}
2270
2271SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2272 RTLIB::Libcall Call_I8,
2273 RTLIB::Libcall Call_I16,
2274 RTLIB::Libcall Call_I32,
2275 RTLIB::Libcall Call_I64,
2276 RTLIB::Libcall Call_I128) {
2277 RTLIB::Libcall LC;
2278 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2279 default: llvm_unreachable("Unexpected request for libcall!");
2280 case MVT::i8: LC = Call_I8; break;
2281 case MVT::i16: LC = Call_I16; break;
2282 case MVT::i32: LC = Call_I32; break;
2283 case MVT::i64: LC = Call_I64; break;
2284 case MVT::i128: LC = Call_I128; break;
2285 }
2286 return ExpandLibCall(LC, Node, isSigned).first;
2287}
2288
2289/// Expand the node to a libcall based on first argument type (for instance
2290/// lround and its variant).
2291void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2292 RTLIB::Libcall Call_F32,
2293 RTLIB::Libcall Call_F64,
2294 RTLIB::Libcall Call_F80,
2295 RTLIB::Libcall Call_F128,
2296 RTLIB::Libcall Call_PPCF128,
2297 SmallVectorImpl<SDValue> &Results) {
2298 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2299 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2300 Call_F32, Call_F64, Call_F80,
2301 Call_F128, Call_PPCF128);
2302 ExpandFPLibCall(Node, LC, Results);
2303}
2304
2305SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2306 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2307 RTLIB::Libcall CallI128) {
2308 RTLIB::Libcall LC;
2309 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2310 default:
2311 llvm_unreachable("Unexpected request for libcall!");
2312 case MVT::i32:
2313 LC = CallI32;
2314 break;
2315 case MVT::i64:
2316 LC = CallI64;
2317 break;
2318 case MVT::i128:
2319 LC = CallI128;
2320 break;
2321 }
2322
2323 // Bit-counting libcalls have one unsigned argument and return `int`.
2324 // Note that `int` may be illegal on this target; ExpandLibCall will
2325 // take care of promoting it to a legal type.
2326 SDValue Op = Node->getOperand(Num: 0);
2327 EVT IntVT =
2328 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2329
2330 EVT ArgVT = Op.getValueType();
2331 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2332 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2333 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2334 Arg.IsZExt = !Arg.IsSExt;
2335
2336 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2337 /*IsSigned=*/true, RetVT: IntVT)
2338 .first;
2339
2340 // If ExpandLibCall created a tail call, the result was already
2341 // of the correct type. Otherwise, we need to sign extend it.
2342 if (Res.getValueType() != MVT::Other)
2343 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2344 return Res;
2345}
2346
2347/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2348void
2349SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2350 SmallVectorImpl<SDValue> &Results) {
2351 unsigned Opcode = Node->getOpcode();
2352 bool isSigned = Opcode == ISD::SDIVREM;
2353
2354 RTLIB::Libcall LC;
2355 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2356 default: llvm_unreachable("Unexpected request for libcall!");
2357 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2358 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2359 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2360 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2361 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2362 }
2363
2364 // The input chain to this libcall is the entry node of the function.
2365 // Legalizing the call will automatically add the previous call to the
2366 // dependence.
2367 SDValue InChain = DAG.getEntryNode();
2368
2369 EVT RetVT = Node->getValueType(ResNo: 0);
2370 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2371
2372 TargetLowering::ArgListTy Args;
2373 for (const SDValue &Op : Node->op_values()) {
2374 EVT ArgVT = Op.getValueType();
2375 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2376 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2377 Entry.IsSExt = isSigned;
2378 Entry.IsZExt = !isSigned;
2379 Args.push_back(x: Entry);
2380 }
2381
2382 // Also pass the return address of the remainder.
2383 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2384 TargetLowering::ArgListEntry Entry(
2385 FIPtr, PointerType::getUnqual(C&: RetTy->getContext()));
2386 Entry.IsSExt = isSigned;
2387 Entry.IsZExt = !isSigned;
2388 Args.push_back(x: Entry);
2389
2390 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2391 if (LibcallImpl == RTLIB::Unsupported) {
2392 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2393 Node->getOperationName(G: &DAG));
2394 SDValue Poison = DAG.getPOISON(VT: RetVT);
2395 Results.push_back(Elt: Poison);
2396 Results.push_back(Elt: Poison);
2397 return;
2398 }
2399
2400 SDValue Callee =
2401 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2402
2403 SDLoc dl(Node);
2404 TargetLowering::CallLoweringInfo CLI(DAG);
2405 CLI.setDebugLoc(dl)
2406 .setChain(InChain)
2407 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
2408 ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2409 .setSExtResult(isSigned)
2410 .setZExtResult(!isSigned);
2411
2412 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2413
2414 // Remainder is loaded back from the stack frame.
2415 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
2416 MachinePointerInfo PtrInfo =
2417 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2418
2419 SDValue Rem = DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo);
2420 Results.push_back(Elt: CallInfo.first);
2421 Results.push_back(Elt: Rem);
2422}
2423
2424/// Return true if sincos or __sincos_stret libcall is available.
2425static bool isSinCosLibcallAvailable(SDNode *Node,
2426 const LibcallLoweringInfo &Libcalls) {
2427 MVT::SimpleValueType VT = Node->getSimpleValueType(ResNo: 0).SimpleTy;
2428 return Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS(RetVT: VT)) != RTLIB::Unsupported ||
2429 Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS_STRET(RetVT: VT)) !=
2430 RTLIB::Unsupported;
2431}
2432
2433/// Only issue sincos libcall if both sin and cos are needed.
2434static bool useSinCos(SDNode *Node) {
2435 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2436 ? ISD::FCOS : ISD::FSIN;
2437
2438 SDValue Op0 = Node->getOperand(Num: 0);
2439 for (const SDNode *User : Op0.getNode()->users()) {
2440 if (User == Node)
2441 continue;
2442 // The other user might have been turned into sincos already.
2443 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2444 return true;
2445 }
2446 return false;
2447}
2448
2449SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2450 // For iOS, we want to call an alternative entry point: __sincos_stret,
2451 // which returns the values in two S / D registers.
2452 SDLoc dl(Node);
2453 SDValue Arg = Node->getOperand(Num: 0);
2454 EVT ArgVT = Arg.getValueType();
2455 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(RetVT: ArgVT);
2456 RTLIB::LibcallImpl SincosStret = DAG.getLibcalls().getLibcallImpl(Call: LC);
2457 if (SincosStret == RTLIB::Unsupported)
2458 return SDValue();
2459
2460 /// There are 3 different ABI cases to handle:
2461 /// - Direct return of separate fields in registers
2462 /// - Single return as vector elements
2463 /// - sret struct
2464
2465 const RTLIB::RuntimeLibcallsInfo &CallsInfo = TLI.getRuntimeLibcallsInfo();
2466
2467 const DataLayout &DL = DAG.getDataLayout();
2468
2469 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2470 Ctx&: *DAG.getContext(), TT: TM.getTargetTriple(), DL, LibcallImpl: SincosStret);
2471
2472 Type *SincosStretRetTy = FuncTy->getReturnType();
2473 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(Call: SincosStret);
2474
2475 SDValue Callee =
2476 DAG.getExternalSymbol(LCImpl: SincosStret, VT: TLI.getProgramPointerTy(DL));
2477
2478 TargetLowering::ArgListTy Args;
2479 SDValue SRet;
2480
2481 int FrameIdx;
2482 if (FuncTy->getParamType(i: 0)->isPointerTy()) {
2483 // Uses sret
2484 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2485
2486 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(ArgNo: 0);
2487 Type *StructTy = PtrAttrs.getStructRetType();
2488 const uint64_t ByteSize = DL.getTypeAllocSize(Ty: StructTy);
2489 const Align StackAlign = DL.getPrefTypeAlign(Ty: StructTy);
2490
2491 FrameIdx = MFI.CreateStackObject(Size: ByteSize, Alignment: StackAlign, isSpillSlot: false);
2492 SRet = DAG.getFrameIndex(FI: FrameIdx, VT: TLI.getFrameIndexTy(DL));
2493
2494 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(i: 0));
2495 Entry.IsSRet = true;
2496 Entry.IndirectType = StructTy;
2497 Entry.Alignment = StackAlign;
2498
2499 Args.push_back(x: Entry);
2500 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 1));
2501 } else {
2502 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 0));
2503 }
2504
2505 TargetLowering::CallLoweringInfo CLI(DAG);
2506 CLI.setDebugLoc(dl)
2507 .setChain(DAG.getEntryNode())
2508 .setLibCallee(CC: CallConv, ResultType: SincosStretRetTy, Target: Callee, ArgsList: std::move(Args))
2509 .setIsPostTypeLegalization();
2510
2511 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2512
2513 if (SRet) {
2514 MachinePointerInfo PtrInfo =
2515 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2516 SDValue LoadSin = DAG.getLoad(VT: ArgVT, dl, Chain: CallResult.second, Ptr: SRet, PtrInfo);
2517
2518 TypeSize StoreSize = ArgVT.getStoreSize();
2519
2520 // Address of cos field.
2521 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr: SRet, Offset: StoreSize);
2522 SDValue LoadCos = DAG.getLoad(VT: ArgVT, dl, Chain: LoadSin.getValue(R: 1), Ptr: Add,
2523 PtrInfo: PtrInfo.getWithOffset(O: StoreSize));
2524
2525 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2526 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: LoadSin.getValue(R: 0),
2527 N2: LoadCos.getValue(R: 0));
2528 }
2529
2530 if (!CallResult.first.getValueType().isVector())
2531 return CallResult.first;
2532
2533 SDValue SinVal =
2534 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2535 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2536 SDValue CosVal =
2537 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2538 N2: DAG.getVectorIdxConstant(Val: 1, DL: dl));
2539 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2540 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: SinVal, N2: CosVal);
2541}
2542
2543SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2544 SDLoc dl(Node);
2545 EVT VT = Node->getValueType(ResNo: 0);
2546 SDValue X = Node->getOperand(Num: 0);
2547 SDValue N = Node->getOperand(Num: 1);
2548 EVT ExpVT = N.getValueType();
2549 EVT AsIntVT = VT.changeTypeToInteger();
2550 if (AsIntVT == EVT()) // TODO: How to handle f80?
2551 return SDValue();
2552
2553 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2554 return SDValue();
2555
2556 SDNodeFlags NSW;
2557 NSW.setNoSignedWrap(true);
2558 SDNodeFlags NUW_NSW;
2559 NUW_NSW.setNoUnsignedWrap(true);
2560 NUW_NSW.setNoSignedWrap(true);
2561
2562 EVT SetCCVT =
2563 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2564 const fltSemantics &FltSem = VT.getFltSemantics();
2565
2566 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2567 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2568 const int Precision = APFloat::semanticsPrecision(FltSem);
2569
2570 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2571 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2572
2573 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2574
2575 const APFloat One(FltSem, "1.0");
2576 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2577
2578 // Offset by precision to avoid denormal range.
2579 APFloat ScaleDownK =
2580 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2581
2582 // TODO: Should really introduce control flow and use a block for the >
2583 // MaxExp, < MinExp cases
2584
2585 // First, handle exponents Exp > MaxExp and scale down.
2586 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2587
2588 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2589 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2590 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2591 SDValue DecN1 =
2592 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2593
2594 SDValue ScaleUpTwice =
2595 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2596
2597 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2598 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2599 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2600
2601 SDValue SelectN_Big =
2602 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2603 SDValue SelectX_Big =
2604 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2605
2606 // Now handle exponents Exp < MinExp
2607 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2608
2609 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2610 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2611
2612 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2613
2614 SDValue ClampMinVal =
2615 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2616 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2617 SDValue IncN1 =
2618 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2619
2620 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2621 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2622 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2623
2624 SDValue ScaleDownTwice = DAG.getSetCC(
2625 DL: dl, VT: SetCCVT, LHS: N,
2626 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2627
2628 SDValue SelectN_Small =
2629 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2630 SDValue SelectX_Small =
2631 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2632
2633 // Now combine the two out of range exponent handling cases with the base
2634 // case.
2635 SDValue NewX = DAG.getNode(
2636 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2637 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2638
2639 SDValue NewN = DAG.getNode(
2640 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2641 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2642
2643 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2644
2645 SDValue ExponentShiftAmt =
2646 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2647 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2648
2649 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2650 N2: ExponentShiftAmt, Flags: NUW_NSW);
2651 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2652 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2653}
2654
2655SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2656 SDLoc dl(Node);
2657 SDValue Val = Node->getOperand(Num: 0);
2658 EVT VT = Val.getValueType();
2659 EVT ExpVT = Node->getValueType(ResNo: 1);
2660 EVT AsIntVT = VT.changeTypeToInteger();
2661 if (AsIntVT == EVT()) // TODO: How to handle f80?
2662 return SDValue();
2663
2664 const fltSemantics &FltSem = VT.getFltSemantics();
2665 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2666 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2667 const unsigned BitSize = VT.getScalarSizeInBits();
2668
2669 // TODO: Could introduce control flow and skip over the denormal handling.
2670
2671 // scale_up = fmul value, scalbn(1.0, precision + 1)
2672 // extracted_exp = (bitcast value to uint) >> precision - 1
2673 // biased_exp = extracted_exp + min_exp
2674 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2675 //
2676 // is_denormal = val < smallest_normalized
2677 // computed_fract = is_denormal ? scale_up : extracted_fract
2678 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2679 //
2680 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2681 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2682
2683 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2684 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2685 VT: AsIntVT);
2686
2687 SDValue SmallestNormalizedInt = DAG.getConstant(
2688 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2689 VT: AsIntVT);
2690
2691 // Masks out the exponent bits.
2692 SDValue ExpMask =
2693 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2694
2695 // Mask out the exponent part of the value.
2696 //
2697 // e.g, for f32 FractSignMaskVal = 0x807fffff
2698 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2699 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2700
2701 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2702 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2703
2704 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2705
2706 const APFloat One(FltSem, "1.0");
2707 // Scale a possible denormal input.
2708 // e.g., for f64, 0x1p+54
2709 APFloat ScaleUpKVal =
2710 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2711
2712 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2713 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2714
2715 EVT SetCCVT =
2716 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2717
2718 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2719
2720 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2721
2722 SDValue AddNegSmallestNormal =
2723 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2724 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2725 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2726
2727 SDValue IsDenormal =
2728 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2729
2730 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2731 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2732
2733 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2734 SDValue ScaledSelect =
2735 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2736
2737 SDValue ExpMaskScaled =
2738 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2739
2740 SDValue ScaledValue =
2741 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2742
2743 // Extract the exponent bits.
2744 SDValue ExponentShiftAmt =
2745 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2746 SDValue ShiftedExp =
2747 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2748 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2749
2750 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2751 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2752 SDValue DenormalExpBias =
2753 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2754
2755 SDValue MaskedFractAsInt =
2756 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2757 const APFloat Half(FltSem, "0.5");
2758 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2759 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2760 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2761
2762 SDValue ComputedExp =
2763 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2764
2765 SDValue Result0 =
2766 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2767
2768 SDValue Result1 =
2769 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2770
2771 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2772}
2773
2774/// This function is responsible for legalizing a
2775/// INT_TO_FP operation of the specified operand when the target requests that
2776/// we expand it. At this point, we know that the result and operand types are
2777/// legal for the target.
2778SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2779 SDValue &Chain) {
2780 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2781 Node->getOpcode() == ISD::SINT_TO_FP);
2782 EVT DestVT = Node->getValueType(ResNo: 0);
2783 SDLoc dl(Node);
2784 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2785 SDValue Op0 = Node->getOperand(Num: OpNo);
2786 EVT SrcVT = Op0.getValueType();
2787
2788 // TODO: Should any fast-math-flags be set for the created nodes?
2789 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2790 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2791 (DestVT.bitsLE(VT: MVT::f64) ||
2792 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2793 : ISD::FP_EXTEND,
2794 VT: DestVT))) {
2795 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2796 "expansion\n");
2797
2798 // Get the stack frame index of a 8 byte buffer.
2799 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2800
2801 SDValue Lo = Op0;
2802 // if signed map to unsigned space
2803 if (isSigned) {
2804 // Invert sign bit (signed to unsigned mapping).
2805 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2806 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2807 }
2808 // Initial hi portion of constructed double.
2809 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2810
2811 // If this a big endian target, swap the lo and high data.
2812 if (DAG.getDataLayout().isBigEndian())
2813 std::swap(a&: Lo, b&: Hi);
2814
2815 SDValue MemChain = DAG.getEntryNode();
2816
2817 // Store the lo of the constructed double.
2818 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2819 PtrInfo: MachinePointerInfo());
2820 // Store the hi of the constructed double.
2821 SDValue HiPtr =
2822 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2823 SDValue Store2 =
2824 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2825 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2826
2827 // load the constructed double
2828 SDValue Load =
2829 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2830 // FP constant to bias correct the final result
2831 SDValue Bias = DAG.getConstantFP(
2832 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2833 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2834 DL: dl, VT: MVT::f64);
2835 // Subtract the bias and get the final result.
2836 SDValue Sub;
2837 SDValue Result;
2838 if (Node->isStrictFPOpcode()) {
2839 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2840 Ops: {Node->getOperand(Num: 0), Load, Bias});
2841 Chain = Sub.getValue(R: 1);
2842 if (DestVT != Sub.getValueType()) {
2843 std::pair<SDValue, SDValue> ResultPair;
2844 ResultPair =
2845 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2846 Result = ResultPair.first;
2847 Chain = ResultPair.second;
2848 }
2849 else
2850 Result = Sub;
2851 } else {
2852 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2853 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2854 }
2855 return Result;
2856 }
2857
2858 if (isSigned)
2859 return SDValue();
2860
2861 // TODO: Generalize this for use with other types.
2862 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2863 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2864 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2865 // For unsigned conversions, convert them to signed conversions using the
2866 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2867 // should be valid for i32->f32 as well.
2868
2869 // More generally this transform should be valid if there are 3 more bits
2870 // in the integer type than the significand. Rounding uses the first bit
2871 // after the width of the significand and the OR of all bits after that. So
2872 // we need to be able to OR the shifted out bit into one of the bits that
2873 // participate in the OR.
2874
2875 // TODO: This really should be implemented using a branch rather than a
2876 // select. We happen to get lucky and machinesink does the right
2877 // thing most of the time. This would be a good candidate for a
2878 // pseudo-op, or, even better, for whole-function isel.
2879 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2880
2881 SDValue SignBitTest = DAG.getSetCC(
2882 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2883
2884 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL: dl);
2885 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2886 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2887 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2888 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2889
2890 SDValue Slow, Fast;
2891 if (Node->isStrictFPOpcode()) {
2892 // In strict mode, we must avoid spurious exceptions, and therefore
2893 // must make sure to only emit a single STRICT_SINT_TO_FP.
2894 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2895 // The STRICT_SINT_TO_FP inherits the exception mode from the
2896 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2897 // never raise any exception.
2898 SDNodeFlags Flags;
2899 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2900 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: {DestVT, MVT::Other},
2901 Ops: {Node->getOperand(Num: 0), InCvt}, Flags);
2902 Flags.setNoFPExcept(true);
2903 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: {DestVT, MVT::Other},
2904 Ops: {Fast.getValue(R: 1), Fast, Fast}, Flags);
2905 Chain = Slow.getValue(R: 1);
2906 } else {
2907 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2908 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2909 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2910 }
2911
2912 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2913 }
2914
2915 // Don't expand it if there isn't cheap fadd.
2916 if (!TLI.isOperationLegalOrCustom(
2917 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2918 return SDValue();
2919
2920 // The following optimization is valid only if every value in SrcVT (when
2921 // treated as signed) is representable in DestVT. Check that the mantissa
2922 // size of DestVT is >= than the number of bits in SrcVT -1.
2923 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2924 SrcVT.getSizeInBits() - 1 &&
2925 "Cannot perform lossless SINT_TO_FP!");
2926
2927 SDValue Tmp1;
2928 if (Node->isStrictFPOpcode()) {
2929 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2930 Ops: { Node->getOperand(Num: 0), Op0 });
2931 } else
2932 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2933
2934 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2935 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2936 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2937 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2938 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2939 Cond: SignSet, LHS: Four, RHS: Zero);
2940
2941 // If the sign bit of the integer is set, the large number will be treated
2942 // as a negative number. To counteract this, the dynamic code adds an
2943 // offset depending on the data type.
2944 uint64_t FF;
2945 switch (SrcVT.getSimpleVT().SimpleTy) {
2946 default:
2947 return SDValue();
2948 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2949 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2950 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2951 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2952 }
2953 if (DAG.getDataLayout().isLittleEndian())
2954 FF <<= 32;
2955 Constant *FudgeFactor = ConstantInt::get(
2956 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2957
2958 SDValue CPIdx =
2959 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2960 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2961 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2962 Alignment = commonAlignment(A: Alignment, Offset: 4);
2963 SDValue FudgeInReg;
2964 if (DestVT == MVT::f32)
2965 FudgeInReg = DAG.getLoad(
2966 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2967 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2968 Alignment);
2969 else {
2970 SDValue Load = DAG.getExtLoad(
2971 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2972 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
2973 Alignment);
2974 HandleSDNode Handle(Load);
2975 LegalizeOp(Node: Load.getNode());
2976 FudgeInReg = Handle.getValue();
2977 }
2978
2979 if (Node->isStrictFPOpcode()) {
2980 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
2981 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
2982 Chain = Result.getValue(R: 1);
2983 return Result;
2984 }
2985
2986 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
2987}
2988
2989/// This function is responsible for legalizing a
2990/// *INT_TO_FP operation of the specified operand when the target requests that
2991/// we promote it. At this point, we know that the result and operand types are
2992/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2993/// operation that takes a larger input.
2994void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2995 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2996 bool IsStrict = N->isStrictFPOpcode();
2997 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2998 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2999 EVT DestVT = N->getValueType(ResNo: 0);
3000 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3001 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
3002 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
3003
3004 // First step, figure out the appropriate *INT_TO_FP operation to use.
3005 EVT NewInTy = LegalOp.getValueType();
3006
3007 unsigned OpToUse = 0;
3008
3009 // Scan for the appropriate larger type to use.
3010 while (true) {
3011 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
3012 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3013
3014 // If the target supports SINT_TO_FP of this type, use it.
3015 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
3016 OpToUse = SIntOp;
3017 break;
3018 }
3019 if (IsSigned)
3020 continue;
3021
3022 // If the target supports UINT_TO_FP of this type, use it.
3023 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
3024 OpToUse = UIntOp;
3025 break;
3026 }
3027
3028 // Otherwise, try a larger type.
3029 }
3030
3031 // Okay, we found the operation and type to use. Zero extend our input to the
3032 // desired type then run the operation on it.
3033 if (IsStrict) {
3034 SDValue Res =
3035 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
3036 Ops: {N->getOperand(Num: 0),
3037 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3038 DL: dl, VT: NewInTy, Operand: LegalOp)});
3039 Results.push_back(Elt: Res);
3040 Results.push_back(Elt: Res.getValue(R: 1));
3041 return;
3042 }
3043
3044 Results.push_back(
3045 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
3046 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3047 DL: dl, VT: NewInTy, Operand: LegalOp)));
3048}
3049
3050/// This function is responsible for legalizing a
3051/// FP_TO_*INT operation of the specified operand when the target requests that
3052/// we promote it. At this point, we know that the result and operand types are
3053/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
3054/// operation that returns a larger result.
3055void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3056 SmallVectorImpl<SDValue> &Results) {
3057 bool IsStrict = N->isStrictFPOpcode();
3058 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3059 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3060 EVT DestVT = N->getValueType(ResNo: 0);
3061 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3062 // First step, figure out the appropriate FP_TO*INT operation to use.
3063 EVT NewOutTy = DestVT;
3064
3065 unsigned OpToUse = 0;
3066
3067 // Scan for the appropriate larger type to use.
3068 while (true) {
3069 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
3070 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3071
3072 // A larger signed type can hold all unsigned values of the requested type,
3073 // so using FP_TO_SINT is valid
3074 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
3075 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3076 break;
3077
3078 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
3079 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
3080 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3081 break;
3082
3083 // Otherwise, try a larger type.
3084 }
3085
3086 // Okay, we found the operation and type to use.
3087 SDValue Operation;
3088 if (IsStrict) {
3089 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
3090 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
3091 } else
3092 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
3093
3094 // Truncate the result of the extended FP_TO_*INT operation to the desired
3095 // size.
3096 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
3097 Results.push_back(Elt: Trunc);
3098 if (IsStrict)
3099 Results.push_back(Elt: Operation.getValue(R: 1));
3100}
3101
3102/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
3103/// the result and operand types are legal and there must be a legal
3104/// FP_TO_*INT_SAT operation for a larger result type.
3105SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3106 const SDLoc &dl) {
3107 unsigned Opcode = Node->getOpcode();
3108
3109 // Scan for the appropriate larger type to use.
3110 EVT NewOutTy = Node->getValueType(ResNo: 0);
3111 while (true) {
3112 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
3113 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3114
3115 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
3116 break;
3117 }
3118
3119 // Saturation width is determined by second operand, so we don't have to
3120 // perform any fixup and can directly truncate the result.
3121 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
3122 N2: Node->getOperand(Num: 1));
3123 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
3124}
3125
3126/// Open code the operations for PARITY of the specified operation.
3127SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3128 EVT VT = Op.getValueType();
3129 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3130 unsigned Sz = VT.getScalarSizeInBits();
3131
3132 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3133 SDValue Result;
3134 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3135 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3136 } else {
3137 Result = Op;
3138 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3139 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3140 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3141 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3142 }
3143 }
3144
3145 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3146}
3147
3148SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3149 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3150 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3151 : Node->getOperand(Num: 0).getSimpleValueType();
3152 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3153 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3154 MVT NewScalarVT = NewVecVT.getVectorElementType();
3155
3156 SDLoc DL(Node);
3157 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3158
3159 // FIXME: Support integer.
3160 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3161 "Only FP promotion is supported");
3162
3163 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3164 if (Node->getOperand(Num: j).getValueType().isVector() &&
3165 !(IsVPOpcode &&
3166 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3167 // promote the vector operand.
3168 // FIXME: Support integer.
3169 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3170 "Only FP promotion is supported");
3171 Operands[j] =
3172 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3173 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3174 // promote the initial value.
3175 Operands[j] =
3176 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3177 } else {
3178 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3179 }
3180
3181 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3182 Flags: Node->getFlags());
3183
3184 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3185 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3186 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3187}
3188
3189bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3190 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3191 SmallVector<SDValue, 8> Results;
3192 SDLoc dl(Node);
3193 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3194 bool NeedInvert;
3195 switch (Node->getOpcode()) {
3196 case ISD::ABS:
3197 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3198 Results.push_back(Elt: Tmp1);
3199 break;
3200 case ISD::ABDS:
3201 case ISD::ABDU:
3202 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3203 Results.push_back(Elt: Tmp1);
3204 break;
3205 case ISD::AVGCEILS:
3206 case ISD::AVGCEILU:
3207 case ISD::AVGFLOORS:
3208 case ISD::AVGFLOORU:
3209 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3210 Results.push_back(Elt: Tmp1);
3211 break;
3212 case ISD::CTPOP:
3213 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3214 Results.push_back(Elt: Tmp1);
3215 break;
3216 case ISD::CTLZ:
3217 case ISD::CTLZ_ZERO_UNDEF:
3218 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3219 Results.push_back(Elt: Tmp1);
3220 break;
3221 case ISD::CTTZ:
3222 case ISD::CTTZ_ZERO_UNDEF:
3223 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3224 Results.push_back(Elt: Tmp1);
3225 break;
3226 case ISD::BITREVERSE:
3227 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3228 Results.push_back(Elt: Tmp1);
3229 break;
3230 case ISD::BSWAP:
3231 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3232 Results.push_back(Elt: Tmp1);
3233 break;
3234 case ISD::PARITY:
3235 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3236 break;
3237 case ISD::FRAMEADDR:
3238 case ISD::RETURNADDR:
3239 case ISD::FRAME_TO_ARGS_OFFSET:
3240 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3241 break;
3242 case ISD::EH_DWARF_CFA: {
3243 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3244 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3245 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3246 VT: CfaArg.getValueType(),
3247 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3248 VT: CfaArg.getValueType()),
3249 N2: CfaArg);
3250 SDValue FA = DAG.getNode(
3251 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3252 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3253 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3254 N1: FA, N2: Offset));
3255 break;
3256 }
3257 case ISD::GET_ROUNDING:
3258 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3259 Results.push_back(Elt: Node->getOperand(Num: 0));
3260 break;
3261 case ISD::EH_RETURN:
3262 case ISD::EH_LABEL:
3263 case ISD::PREFETCH:
3264 case ISD::VAEND:
3265 case ISD::EH_SJLJ_LONGJMP:
3266 // If the target didn't expand these, there's nothing to do, so just
3267 // preserve the chain and be done.
3268 Results.push_back(Elt: Node->getOperand(Num: 0));
3269 break;
3270 case ISD::READCYCLECOUNTER:
3271 case ISD::READSTEADYCOUNTER:
3272 // If the target didn't expand this, just return 'zero' and preserve the
3273 // chain.
3274 Results.append(NumInputs: Node->getNumValues() - 1,
3275 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3276 Results.push_back(Elt: Node->getOperand(Num: 0));
3277 break;
3278 case ISD::EH_SJLJ_SETJMP:
3279 // If the target didn't expand this, just return 'zero' and preserve the
3280 // chain.
3281 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3282 Results.push_back(Elt: Node->getOperand(Num: 0));
3283 break;
3284 case ISD::ATOMIC_LOAD: {
3285 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3286 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3287 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3288 SDValue Swap = DAG.getAtomicCmpSwap(
3289 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3290 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3291 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3292 Results.push_back(Elt: Swap.getValue(R: 0));
3293 Results.push_back(Elt: Swap.getValue(R: 1));
3294 break;
3295 }
3296 case ISD::ATOMIC_STORE: {
3297 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3298 SDValue Swap = DAG.getAtomic(
3299 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3300 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3301 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3302 Results.push_back(Elt: Swap.getValue(R: 1));
3303 break;
3304 }
3305 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3306 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3307 // splits out the success value as a comparison. Expanding the resulting
3308 // ATOMIC_CMP_SWAP will produce a libcall.
3309 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3310 SDValue Res = DAG.getAtomicCmpSwap(
3311 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3312 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3313 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3314
3315 SDValue ExtRes = Res;
3316 SDValue LHS = Res;
3317 SDValue RHS = Node->getOperand(Num: 1);
3318
3319 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3320 EVT OuterType = Node->getValueType(ResNo: 0);
3321 switch (TLI.getExtendForAtomicOps()) {
3322 case ISD::SIGN_EXTEND:
3323 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3324 N2: DAG.getValueType(AtomicType));
3325 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3326 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3327 ExtRes = LHS;
3328 break;
3329 case ISD::ZERO_EXTEND:
3330 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3331 N2: DAG.getValueType(AtomicType));
3332 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3333 ExtRes = LHS;
3334 break;
3335 case ISD::ANY_EXTEND:
3336 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3337 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3338 break;
3339 default:
3340 llvm_unreachable("Invalid atomic op extension");
3341 }
3342
3343 SDValue Success =
3344 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3345
3346 Results.push_back(Elt: ExtRes.getValue(R: 0));
3347 Results.push_back(Elt: Success);
3348 Results.push_back(Elt: Res.getValue(R: 1));
3349 break;
3350 }
3351 case ISD::ATOMIC_LOAD_SUB: {
3352 SDLoc DL(Node);
3353 EVT VT = Node->getValueType(ResNo: 0);
3354 SDValue RHS = Node->getOperand(Num: 2);
3355 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3356 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3357 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3358 RHS = RHS->getOperand(Num: 0);
3359 SDValue NewRHS =
3360 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3361 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3362 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3363 Val: NewRHS, MMO: AN->getMemOperand());
3364 Results.push_back(Elt: Res);
3365 Results.push_back(Elt: Res.getValue(R: 1));
3366 break;
3367 }
3368 case ISD::DYNAMIC_STACKALLOC:
3369 ExpandDYNAMIC_STACKALLOC(Node, Results);
3370 break;
3371 case ISD::MERGE_VALUES:
3372 for (unsigned i = 0; i < Node->getNumValues(); i++)
3373 Results.push_back(Elt: Node->getOperand(Num: i));
3374 break;
3375 case ISD::POISON:
3376 case ISD::UNDEF: {
3377 EVT VT = Node->getValueType(ResNo: 0);
3378 if (VT.isInteger())
3379 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3380 else {
3381 assert(VT.isFloatingPoint() && "Unknown value type!");
3382 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3383 }
3384 break;
3385 }
3386 case ISD::STRICT_FP_ROUND:
3387 // When strict mode is enforced we can't do expansion because it
3388 // does not honor the "strict" properties. Only libcall is allowed.
3389 if (TLI.isStrictFPEnabled())
3390 break;
3391 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3392 // since this operation is more efficient than stack operation.
3393 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3394 VT: Node->getValueType(ResNo: 0))
3395 == TargetLowering::Legal)
3396 break;
3397 // We fall back to use stack operation when the FP_ROUND operation
3398 // isn't available.
3399 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3400 DestVT: Node->getValueType(ResNo: 0), dl,
3401 Chain: Node->getOperand(Num: 0)))) {
3402 ReplaceNode(Old: Node, New: Tmp1.getNode());
3403 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3404 return true;
3405 }
3406 break;
3407 case ISD::FP_ROUND: {
3408 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3409 Results.push_back(Elt: Tmp1);
3410 break;
3411 }
3412
3413 [[fallthrough]];
3414 }
3415 case ISD::BITCAST:
3416 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3417 DestVT: Node->getValueType(ResNo: 0), dl)))
3418 Results.push_back(Elt: Tmp1);
3419 break;
3420 case ISD::STRICT_FP_EXTEND:
3421 // When strict mode is enforced we can't do expansion because it
3422 // does not honor the "strict" properties. Only libcall is allowed.
3423 if (TLI.isStrictFPEnabled())
3424 break;
3425 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3426 // since this operation is more efficient than stack operation.
3427 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3428 VT: Node->getValueType(ResNo: 0))
3429 == TargetLowering::Legal)
3430 break;
3431 // We fall back to use stack operation when the FP_EXTEND operation
3432 // isn't available.
3433 if ((Tmp1 = EmitStackConvert(
3434 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3435 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3436 ReplaceNode(Old: Node, New: Tmp1.getNode());
3437 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3438 return true;
3439 }
3440 break;
3441 case ISD::FP_EXTEND: {
3442 SDValue Op = Node->getOperand(Num: 0);
3443 EVT SrcVT = Op.getValueType();
3444 EVT DstVT = Node->getValueType(ResNo: 0);
3445 if (SrcVT.getScalarType() == MVT::bf16) {
3446 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3447 break;
3448 }
3449
3450 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3451 Results.push_back(Elt: Tmp1);
3452 break;
3453 }
3454 case ISD::BF16_TO_FP: {
3455 // Always expand bf16 to f32 casts, they lower to ext + shift.
3456 //
3457 // Note that the operand of this code can be bf16 or an integer type in case
3458 // bf16 is not supported on the target and was softened.
3459 SDValue Op = Node->getOperand(Num: 0);
3460 if (Op.getValueType() == MVT::bf16) {
3461 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3462 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3463 } else {
3464 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3465 }
3466 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3467 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3468 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3469 // Add fp_extend in case the output is bigger than f32.
3470 if (Node->getValueType(ResNo: 0) != MVT::f32)
3471 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3472 Results.push_back(Elt: Op);
3473 break;
3474 }
3475 case ISD::FP_TO_BF16: {
3476 SDValue Op = Node->getOperand(Num: 0);
3477 if (Op.getValueType() != MVT::f32)
3478 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3479 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3480 // Certain SNaNs will turn into infinities if we do a simple shift right.
3481 if (!DAG.isKnownNeverSNaN(Op)) {
3482 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3483 }
3484 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
3485 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3486 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3487 // The result of this node can be bf16 or an integer type in case bf16 is
3488 // not supported on the target and was softened to i16 for storage.
3489 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3490 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3491 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3492 } else {
3493 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3494 }
3495 Results.push_back(Elt: Op);
3496 break;
3497 }
3498 case ISD::FCANONICALIZE: {
3499 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
3500 // suggested in
3501 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
3502 // It uses strict_fp operations even outside a strict_fp context in order
3503 // to guarantee that the canonicalization is not optimized away by later
3504 // passes. The result chain introduced by that is intentionally ignored
3505 // since no ordering requirement is intended here.
3506
3507 // Create strict multiplication by 1.0.
3508 SDValue Operand = Node->getOperand(Num: 0);
3509 EVT VT = Operand.getValueType();
3510 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
3511 SDValue Chain = DAG.getEntryNode();
3512 // Propagate existing flags on canonicalize, and additionally set
3513 // NoFPExcept.
3514 SDNodeFlags CanonicalizeFlags = Node->getFlags();
3515 CanonicalizeFlags.setNoFPExcept(true);
3516 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
3517 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
3518
3519 Results.push_back(Elt: Mul);
3520 break;
3521 }
3522 case ISD::SIGN_EXTEND_INREG: {
3523 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3524 EVT VT = Node->getValueType(ResNo: 0);
3525
3526 // An in-register sign-extend of a boolean is a negation:
3527 // 'true' (1) sign-extended is -1.
3528 // 'false' (0) sign-extended is 0.
3529 // However, we must mask the high bits of the source operand because the
3530 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3531
3532 // TODO: Do this for vectors too?
3533 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3534 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3535 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3536 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3537 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3538 Results.push_back(Elt: Neg);
3539 break;
3540 }
3541
3542 // NOTE: we could fall back on load/store here too for targets without
3543 // SRA. However, it is doubtful that any exist.
3544 unsigned BitsDiff = VT.getScalarSizeInBits() -
3545 ExtraVT.getScalarSizeInBits();
3546 SDValue ShiftCst = DAG.getShiftAmountConstant(Val: BitsDiff, VT, DL: dl);
3547 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: ShiftCst);
3548 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Tmp1, N2: ShiftCst);
3549 Results.push_back(Elt: Tmp1);
3550 break;
3551 }
3552 case ISD::UINT_TO_FP:
3553 case ISD::STRICT_UINT_TO_FP:
3554 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3555 Results.push_back(Elt: Tmp1);
3556 if (Node->isStrictFPOpcode())
3557 Results.push_back(Elt: Tmp2);
3558 break;
3559 }
3560 [[fallthrough]];
3561 case ISD::SINT_TO_FP:
3562 case ISD::STRICT_SINT_TO_FP:
3563 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3564 Results.push_back(Elt: Tmp1);
3565 if (Node->isStrictFPOpcode())
3566 Results.push_back(Elt: Tmp2);
3567 }
3568 break;
3569 case ISD::FP_TO_SINT:
3570 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3571 Results.push_back(Elt: Tmp1);
3572 break;
3573 case ISD::STRICT_FP_TO_SINT:
3574 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3575 ReplaceNode(Old: Node, New: Tmp1.getNode());
3576 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3577 return true;
3578 }
3579 break;
3580 case ISD::FP_TO_UINT:
3581 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3582 Results.push_back(Elt: Tmp1);
3583 break;
3584 case ISD::STRICT_FP_TO_UINT:
3585 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3586 // Relink the chain.
3587 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3588 // Replace the new UINT result.
3589 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3590 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3591 return true;
3592 }
3593 break;
3594 case ISD::FP_TO_SINT_SAT:
3595 case ISD::FP_TO_UINT_SAT:
3596 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3597 break;
3598 case ISD::LROUND:
3599 case ISD::LLROUND: {
3600 SDValue Arg = Node->getOperand(Num: 0);
3601 EVT ArgVT = Arg.getValueType();
3602 EVT ResVT = Node->getValueType(ResNo: 0);
3603 SDLoc dl(Node);
3604 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3605 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3606 break;
3607 }
3608 case ISD::VAARG:
3609 Results.push_back(Elt: DAG.expandVAArg(Node));
3610 Results.push_back(Elt: Results[0].getValue(R: 1));
3611 break;
3612 case ISD::VACOPY:
3613 Results.push_back(Elt: DAG.expandVACopy(Node));
3614 break;
3615 case ISD::EXTRACT_VECTOR_ELT:
3616 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3617 // This must be an access of the only element. Return it.
3618 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3619 Operand: Node->getOperand(Num: 0));
3620 else
3621 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3622 Results.push_back(Elt: Tmp1);
3623 break;
3624 case ISD::EXTRACT_SUBVECTOR:
3625 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3626 break;
3627 case ISD::INSERT_SUBVECTOR:
3628 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3629 break;
3630 case ISD::CONCAT_VECTORS:
3631 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3632 VectorValueType.isScalableVector() ||
3633 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3634 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3635 else
3636 Results.push_back(Elt: ExpandConcatVectors(Node));
3637 break;
3638 case ISD::SCALAR_TO_VECTOR:
3639 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3640 break;
3641 case ISD::INSERT_VECTOR_ELT:
3642 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3643 break;
3644 case ISD::VECTOR_SHUFFLE: {
3645 SmallVector<int, 32> NewMask;
3646 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3647
3648 EVT VT = Node->getValueType(ResNo: 0);
3649 EVT EltVT = VT.getVectorElementType();
3650 SDValue Op0 = Node->getOperand(Num: 0);
3651 SDValue Op1 = Node->getOperand(Num: 1);
3652 if (!TLI.isTypeLegal(VT: EltVT)) {
3653 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3654
3655 // BUILD_VECTOR operands are allowed to be wider than the element type.
3656 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3657 // it.
3658 if (NewEltVT.bitsLT(VT: EltVT)) {
3659 // Convert shuffle node.
3660 // If original node was v4i64 and the new EltVT is i32,
3661 // cast operands to v8i32 and re-build the mask.
3662
3663 // Calculate new VT, the size of the new VT should be equal to original.
3664 EVT NewVT =
3665 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3666 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3667 assert(NewVT.bitsEq(VT));
3668
3669 // cast operands to new VT
3670 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3671 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3672
3673 // Convert the shuffle mask
3674 unsigned int factor =
3675 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3676
3677 // EltVT gets smaller
3678 assert(factor > 0);
3679
3680 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3681 if (Mask[i] < 0) {
3682 for (unsigned fi = 0; fi < factor; ++fi)
3683 NewMask.push_back(Elt: Mask[i]);
3684 }
3685 else {
3686 for (unsigned fi = 0; fi < factor; ++fi)
3687 NewMask.push_back(Elt: Mask[i]*factor+fi);
3688 }
3689 }
3690 Mask = NewMask;
3691 VT = NewVT;
3692 }
3693 EltVT = NewEltVT;
3694 }
3695 unsigned NumElems = VT.getVectorNumElements();
3696 SmallVector<SDValue, 16> Ops;
3697 for (unsigned i = 0; i != NumElems; ++i) {
3698 if (Mask[i] < 0) {
3699 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3700 continue;
3701 }
3702 unsigned Idx = Mask[i];
3703 if (Idx < NumElems)
3704 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3705 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3706 else
3707 Ops.push_back(
3708 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3709 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3710 }
3711
3712 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3713 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3714 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3715 Results.push_back(Elt: Tmp1);
3716 break;
3717 }
3718 case ISD::VECTOR_SPLICE_LEFT:
3719 case ISD::VECTOR_SPLICE_RIGHT: {
3720 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3721 break;
3722 }
3723 case ISD::VECTOR_DEINTERLEAVE: {
3724 unsigned Factor = Node->getNumOperands();
3725 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
3726 break;
3727 SmallVector<SDValue, 8> Ops(Node->ops());
3728 EVT VecVT = Node->getValueType(ResNo: 0);
3729 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3730 // Deinterleave at Factor/2 so each result contains two factors interleaved:
3731 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
3732 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3733 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
3734 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3735 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
3736 Results.resize(N: Factor);
3737 // Deinterleave the 2 factors out:
3738 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
3739 for (unsigned I = 0; I < Factor / 2; I++) {
3740 SDValue Deinterleave =
3741 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3742 Ops: {L.getValue(R: I), R.getValue(R: I)});
3743 Results[I] = Deinterleave.getValue(R: 0);
3744 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
3745 }
3746 break;
3747 }
3748 case ISD::VECTOR_INTERLEAVE: {
3749 unsigned Factor = Node->getNumOperands();
3750 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
3751 break;
3752 EVT VecVT = Node->getValueType(ResNo: 0);
3753 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3754 SmallVector<SDValue, 8> LOps, ROps;
3755 // Interleave so we have 2 factors per result:
3756 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
3757 for (unsigned I = 0; I < Factor / 2; I++) {
3758 SDValue Interleave =
3759 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3760 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
3761 LOps.push_back(Elt: Interleave.getValue(R: 0));
3762 ROps.push_back(Elt: Interleave.getValue(R: 1));
3763 }
3764 // Interleave at Factor/2:
3765 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
3766 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: LOps);
3767 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: ROps);
3768 for (unsigned I = 0; I < Factor / 2; I++)
3769 Results.push_back(Elt: L.getValue(R: I));
3770 for (unsigned I = 0; I < Factor / 2; I++)
3771 Results.push_back(Elt: R.getValue(R: I));
3772 break;
3773 }
3774 case ISD::EXTRACT_ELEMENT: {
3775 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3776 if (Node->getConstantOperandVal(Num: 1)) {
3777 // 1 -> Hi
3778 Tmp1 = DAG.getNode(
3779 Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3780 N2: DAG.getShiftAmountConstant(Val: OpTy.getSizeInBits() / 2, VT: OpTy, DL: dl));
3781 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3782 } else {
3783 // 0 -> Lo
3784 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3785 Operand: Node->getOperand(Num: 0));
3786 }
3787 Results.push_back(Elt: Tmp1);
3788 break;
3789 }
3790 case ISD::STACKADDRESS:
3791 case ISD::STACKSAVE:
3792 // Expand to CopyFromReg if the target set
3793 // StackPointerRegisterToSaveRestore.
3794 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3795 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3796 VT: Node->getValueType(ResNo: 0)));
3797 Results.push_back(Elt: Results[0].getValue(R: 1));
3798 } else {
3799 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3800 Results.push_back(Elt: Node->getOperand(Num: 0));
3801
3802 StringRef IntrinsicName = Node->getOpcode() == ISD::STACKADDRESS
3803 ? "llvm.stackaddress"
3804 : "llvm.stacksave";
3805 DAG.getContext()->diagnose(DI: DiagnosticInfoLegalizationFailure(
3806 Twine(IntrinsicName) + " is not supported on this target.",
3807 DAG.getMachineFunction().getFunction(), dl.getDebugLoc()));
3808 }
3809 break;
3810 case ISD::STACKRESTORE:
3811 // Expand to CopyToReg if the target set
3812 // StackPointerRegisterToSaveRestore.
3813 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3814 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3815 N: Node->getOperand(Num: 1)));
3816 } else {
3817 Results.push_back(Elt: Node->getOperand(Num: 0));
3818 }
3819 break;
3820 case ISD::GET_DYNAMIC_AREA_OFFSET:
3821 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3822 Results.push_back(Elt: Results[0].getValue(R: 0));
3823 break;
3824 case ISD::FCOPYSIGN:
3825 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3826 break;
3827 case ISD::FNEG:
3828 Results.push_back(Elt: ExpandFNEG(Node));
3829 break;
3830 case ISD::FABS:
3831 Results.push_back(Elt: ExpandFABS(Node));
3832 break;
3833 case ISD::IS_FPCLASS: {
3834 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3835 if (SDValue Expanded =
3836 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3837 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3838 Results.push_back(Elt: Expanded);
3839 break;
3840 }
3841 case ISD::SMIN:
3842 case ISD::SMAX:
3843 case ISD::UMIN:
3844 case ISD::UMAX: {
3845 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3846 ISD::CondCode Pred;
3847 switch (Node->getOpcode()) {
3848 default: llvm_unreachable("How did we get here?");
3849 case ISD::SMAX: Pred = ISD::SETGT; break;
3850 case ISD::SMIN: Pred = ISD::SETLT; break;
3851 case ISD::UMAX: Pred = ISD::SETUGT; break;
3852 case ISD::UMIN: Pred = ISD::SETULT; break;
3853 }
3854 Tmp1 = Node->getOperand(Num: 0);
3855 Tmp2 = Node->getOperand(Num: 1);
3856 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3857 Results.push_back(Elt: Tmp1);
3858 break;
3859 }
3860 case ISD::FMINNUM:
3861 case ISD::FMAXNUM: {
3862 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3863 Results.push_back(Elt: Expanded);
3864 break;
3865 }
3866 case ISD::FMINIMUM:
3867 case ISD::FMAXIMUM: {
3868 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
3869 Results.push_back(Elt: Expanded);
3870 break;
3871 }
3872 case ISD::FMINIMUMNUM:
3873 case ISD::FMAXIMUMNUM: {
3874 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
3875 break;
3876 }
3877 case ISD::FSIN:
3878 case ISD::FCOS: {
3879 EVT VT = Node->getValueType(ResNo: 0);
3880 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3881 // fcos which share the same operand and both are used.
3882 if ((TLI.isOperationLegal(Op: ISD::FSINCOS, VT) ||
3883 isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls())) &&
3884 useSinCos(Node)) {
3885 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3886 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3887 if (Node->getOpcode() == ISD::FCOS)
3888 Tmp1 = Tmp1.getValue(R: 1);
3889 Results.push_back(Elt: Tmp1);
3890 }
3891 break;
3892 }
3893 case ISD::FLDEXP:
3894 case ISD::STRICT_FLDEXP: {
3895 EVT VT = Node->getValueType(ResNo: 0);
3896 RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT);
3897 // Use the LibCall instead, it is very likely faster
3898 // FIXME: Use separate LibCall action.
3899 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3900 break;
3901
3902 if (SDValue Expanded = expandLdexp(Node)) {
3903 Results.push_back(Elt: Expanded);
3904 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3905 Results.push_back(Elt: Expanded.getValue(R: 1));
3906 }
3907
3908 break;
3909 }
3910 case ISD::FFREXP: {
3911 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0));
3912 // Use the LibCall instead, it is very likely faster
3913 // FIXME: Use separate LibCall action.
3914 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3915 break;
3916
3917 if (SDValue Expanded = expandFrexp(Node)) {
3918 Results.push_back(Elt: Expanded);
3919 Results.push_back(Elt: Expanded.getValue(R: 1));
3920 }
3921 break;
3922 }
3923 case ISD::FSINCOS: {
3924 if (isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls()))
3925 break;
3926 EVT VT = Node->getValueType(ResNo: 0);
3927 SDValue Op = Node->getOperand(Num: 0);
3928 SDNodeFlags Flags = Node->getFlags();
3929 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
3930 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
3931 Results.append(IL: {Tmp1, Tmp2});
3932 break;
3933 }
3934 case ISD::FMAD:
3935 llvm_unreachable("Illegal fmad should never be formed");
3936
3937 case ISD::FP16_TO_FP:
3938 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3939 // We can extend to types bigger than f32 in two steps without changing
3940 // the result. Since "f16 -> f32" is much more commonly available, give
3941 // CodeGen the option of emitting that before resorting to a libcall.
3942 SDValue Res =
3943 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
3944 Results.push_back(
3945 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
3946 }
3947 break;
3948 case ISD::STRICT_BF16_TO_FP:
3949 case ISD::STRICT_FP16_TO_FP:
3950 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3951 // We can extend to types bigger than f32 in two steps without changing
3952 // the result. Since "f16 -> f32" is much more commonly available, give
3953 // CodeGen the option of emitting that before resorting to a libcall.
3954 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
3955 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
3956 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
3957 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
3958 Ops: {Res.getValue(R: 1), Res});
3959 Results.push_back(Elt: Res);
3960 Results.push_back(Elt: Res.getValue(R: 1));
3961 }
3962 break;
3963 case ISD::FP_TO_FP16:
3964 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3965 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
3966 SDValue Op = Node->getOperand(Num: 0);
3967 MVT SVT = Op.getSimpleValueType();
3968 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3969 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
3970 // Under fastmath, we can expand this node into a fround followed by
3971 // a float-half conversion.
3972 SDValue FloatVal =
3973 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3974 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3975 Results.push_back(
3976 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
3977 }
3978 }
3979 break;
3980 case ISD::ConstantFP: {
3981 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
3982 // Check to see if this FP immediate is already legal.
3983 // If this is a legal constant, turn it into a TargetConstantFP node.
3984 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
3985 ForCodeSize: DAG.shouldOptForSize()))
3986 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
3987 break;
3988 }
3989 case ISD::Constant: {
3990 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
3991 Results.push_back(Elt: ExpandConstant(CP));
3992 break;
3993 }
3994 case ISD::FSUB: {
3995 EVT VT = Node->getValueType(ResNo: 0);
3996 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
3997 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
3998 const SDNodeFlags Flags = Node->getFlags();
3999 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
4000 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
4001 Results.push_back(Elt: Tmp1);
4002 }
4003 break;
4004 }
4005 case ISD::SUB: {
4006 EVT VT = Node->getValueType(ResNo: 0);
4007 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
4008 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
4009 "Don't know how to expand this subtraction!");
4010 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
4011 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
4012 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
4013 break;
4014 }
4015 case ISD::UREM:
4016 case ISD::SREM:
4017 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
4018 Results.push_back(Elt: Tmp1);
4019 break;
4020 case ISD::UDIV:
4021 case ISD::SDIV: {
4022 bool isSigned = Node->getOpcode() == ISD::SDIV;
4023 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4024 EVT VT = Node->getValueType(ResNo: 0);
4025 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
4026 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4027 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4028 N2: Node->getOperand(Num: 1));
4029 Results.push_back(Elt: Tmp1);
4030 }
4031 break;
4032 }
4033 case ISD::MULHU:
4034 case ISD::MULHS: {
4035 unsigned ExpandOpcode =
4036 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
4037 EVT VT = Node->getValueType(ResNo: 0);
4038 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4039
4040 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4041 N2: Node->getOperand(Num: 1));
4042 Results.push_back(Elt: Tmp1.getValue(R: 1));
4043 break;
4044 }
4045 case ISD::UMUL_LOHI:
4046 case ISD::SMUL_LOHI: {
4047 SDValue LHS = Node->getOperand(Num: 0);
4048 SDValue RHS = Node->getOperand(Num: 1);
4049 EVT VT = LHS.getValueType();
4050 unsigned MULHOpcode =
4051 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
4052
4053 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
4054 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
4055 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
4056 break;
4057 }
4058
4059 SmallVector<SDValue, 4> Halves;
4060 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4061 assert(TLI.isTypeLegal(HalfType));
4062 if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
4063 HiLoVT: HalfType, DAG,
4064 Kind: TargetLowering::MulExpansionKind::Always)) {
4065 for (unsigned i = 0; i < 2; ++i) {
4066 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
4067 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
4068 SDValue Shift =
4069 DAG.getShiftAmountConstant(Val: HalfType.getScalarSizeInBits(), VT, DL: dl);
4070 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4071 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4072 }
4073 break;
4074 }
4075 break;
4076 }
4077 case ISD::MUL: {
4078 EVT VT = Node->getValueType(ResNo: 0);
4079 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4080 // See if multiply or divide can be lowered using two-result operations.
4081 // We just need the low half of the multiply; try both the signed
4082 // and unsigned forms. If the target supports both SMUL_LOHI and
4083 // UMUL_LOHI, form a preference by checking which forms of plain
4084 // MULH it supports.
4085 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
4086 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
4087 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
4088 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
4089 unsigned OpToUse = 0;
4090 if (HasSMUL_LOHI && !HasMULHS) {
4091 OpToUse = ISD::SMUL_LOHI;
4092 } else if (HasUMUL_LOHI && !HasMULHU) {
4093 OpToUse = ISD::UMUL_LOHI;
4094 } else if (HasSMUL_LOHI) {
4095 OpToUse = ISD::SMUL_LOHI;
4096 } else if (HasUMUL_LOHI) {
4097 OpToUse = ISD::UMUL_LOHI;
4098 }
4099 if (OpToUse) {
4100 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4101 N2: Node->getOperand(Num: 1)));
4102 break;
4103 }
4104
4105 SDValue Lo, Hi;
4106 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4107 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
4108 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
4109 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
4110 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
4111 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
4112 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4113 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
4114 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
4115 SDValue Shift =
4116 DAG.getShiftAmountConstant(Val: HalfType.getSizeInBits(), VT, DL: dl);
4117 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4118 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4119 }
4120 break;
4121 }
4122 case ISD::FSHL:
4123 case ISD::FSHR:
4124 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
4125 Results.push_back(Elt: Expanded);
4126 break;
4127 case ISD::ROTL:
4128 case ISD::ROTR:
4129 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
4130 Results.push_back(Elt: Expanded);
4131 break;
4132 case ISD::CLMUL:
4133 case ISD::CLMULR:
4134 case ISD::CLMULH:
4135 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG))
4136 Results.push_back(Elt: Expanded);
4137 break;
4138 case ISD::SADDSAT:
4139 case ISD::UADDSAT:
4140 case ISD::SSUBSAT:
4141 case ISD::USUBSAT:
4142 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
4143 break;
4144 case ISD::SCMP:
4145 case ISD::UCMP:
4146 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
4147 break;
4148 case ISD::SSHLSAT:
4149 case ISD::USHLSAT:
4150 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
4151 break;
4152 case ISD::SMULFIX:
4153 case ISD::SMULFIXSAT:
4154 case ISD::UMULFIX:
4155 case ISD::UMULFIXSAT:
4156 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
4157 break;
4158 case ISD::SDIVFIX:
4159 case ISD::SDIVFIXSAT:
4160 case ISD::UDIVFIX:
4161 case ISD::UDIVFIXSAT:
4162 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
4163 LHS: Node->getOperand(Num: 0),
4164 RHS: Node->getOperand(Num: 1),
4165 Scale: Node->getConstantOperandVal(Num: 2),
4166 DAG)) {
4167 Results.push_back(Elt: V);
4168 break;
4169 }
4170 // FIXME: We might want to retry here with a wider type if we fail, if that
4171 // type is legal.
4172 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4173 // <= 128 (which is the case for all of the default Embedded-C types),
4174 // we will only get here with types and scales that we could always expand
4175 // if we were allowed to generate libcalls to division functions of illegal
4176 // type. But we cannot do that.
4177 llvm_unreachable("Cannot expand DIVFIX!");
4178 case ISD::UADDO_CARRY:
4179 case ISD::USUBO_CARRY: {
4180 SDValue LHS = Node->getOperand(Num: 0);
4181 SDValue RHS = Node->getOperand(Num: 1);
4182 SDValue Carry = Node->getOperand(Num: 2);
4183
4184 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4185
4186 // Initial add of the 2 operands.
4187 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4188 EVT VT = LHS.getValueType();
4189 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4190
4191 // Initial check for overflow.
4192 EVT CarryType = Node->getValueType(ResNo: 1);
4193 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4194 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4195 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4196
4197 // Add of the sum and the carry.
4198 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4199 SDValue CarryExt =
4200 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4201 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4202
4203 // Second check for overflow. If we are adding, we can only overflow if the
4204 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4205 // If we are subtracting, we can only overflow if the initial sum is 0 and
4206 // the carry is set, resulting in a new sum of all 1s.
4207 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4208 SDValue Overflow2 =
4209 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4210 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4211 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4212 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4213
4214 SDValue ResultCarry =
4215 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4216
4217 Results.push_back(Elt: Sum2);
4218 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4219 break;
4220 }
4221 case ISD::SADDO:
4222 case ISD::SSUBO: {
4223 SDValue Result, Overflow;
4224 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4225 Results.push_back(Elt: Result);
4226 Results.push_back(Elt: Overflow);
4227 break;
4228 }
4229 case ISD::UADDO:
4230 case ISD::USUBO: {
4231 SDValue Result, Overflow;
4232 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4233 Results.push_back(Elt: Result);
4234 Results.push_back(Elt: Overflow);
4235 break;
4236 }
4237 case ISD::UMULO:
4238 case ISD::SMULO: {
4239 SDValue Result, Overflow;
4240 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4241 Results.push_back(Elt: Result);
4242 Results.push_back(Elt: Overflow);
4243 }
4244 break;
4245 }
4246 case ISD::BUILD_PAIR: {
4247 EVT PairTy = Node->getValueType(ResNo: 0);
4248 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4249 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4250 Tmp2 = DAG.getNode(
4251 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4252 N2: DAG.getShiftAmountConstant(Val: PairTy.getSizeInBits() / 2, VT: PairTy, DL: dl));
4253 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4254 break;
4255 }
4256 case ISD::SELECT:
4257 Tmp1 = Node->getOperand(Num: 0);
4258 Tmp2 = Node->getOperand(Num: 1);
4259 Tmp3 = Node->getOperand(Num: 2);
4260 if (Tmp1.getOpcode() == ISD::SETCC) {
4261 Tmp1 = DAG.getSelectCC(
4262 DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), True: Tmp2, False: Tmp3,
4263 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get(), Flags: Node->getFlags());
4264 } else {
4265 Tmp1 =
4266 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4267 True: Tmp2, False: Tmp3, Cond: ISD::SETNE, Flags: Node->getFlags());
4268 }
4269 Results.push_back(Elt: Tmp1);
4270 break;
4271 case ISD::BR_JT: {
4272 SDValue Chain = Node->getOperand(Num: 0);
4273 SDValue Table = Node->getOperand(Num: 1);
4274 SDValue Index = Node->getOperand(Num: 2);
4275 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4276
4277 const DataLayout &TD = DAG.getDataLayout();
4278 EVT PTy = TLI.getPointerTy(DL: TD);
4279
4280 unsigned EntrySize =
4281 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4282
4283 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4284 // This transformation needs to be done here since otherwise the MIPS
4285 // backend will end up emitting a three instruction multiply sequence
4286 // instead of a single shift and MSP430 will call a runtime function.
4287 if (llvm::isPowerOf2_32(Value: EntrySize))
4288 Index = DAG.getNode(
4289 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4290 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4291 else
4292 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4293 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4294 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4295
4296 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4297 SDValue LD = DAG.getExtLoad(
4298 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4299 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4300 Addr = LD;
4301 if (TLI.isJumpTableRelative()) {
4302 // For PIC, the sequence is:
4303 // BRIND(RelocBase + load(Jumptable + index))
4304 // RelocBase can be JumpTable, GOT or some sort of global base.
4305 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4306 Offset: Addr, DL: dl);
4307 }
4308
4309 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4310 Results.push_back(Elt: Tmp1);
4311 break;
4312 }
4313 case ISD::BRCOND:
4314 // Expand brcond's setcc into its constituent parts and create a BR_CC
4315 // Node.
4316 Tmp1 = Node->getOperand(Num: 0);
4317 Tmp2 = Node->getOperand(Num: 1);
4318 if (Tmp2.getOpcode() == ISD::SETCC &&
4319 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4320 VT: Tmp2.getOperand(i: 0).getValueType())) {
4321 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4322 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4323 N5: Node->getOperand(Num: 2));
4324 } else {
4325 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4326 if (Tmp2.isUndef() ||
4327 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4328 Tmp3 = Tmp2;
4329 else
4330 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4331 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4332 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4333 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4334 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4335 N5: Node->getOperand(Num: 2));
4336 }
4337 Results.push_back(Elt: Tmp1);
4338 break;
4339 case ISD::SETCC:
4340 case ISD::VP_SETCC:
4341 case ISD::STRICT_FSETCC:
4342 case ISD::STRICT_FSETCCS: {
4343 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4344 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4345 Node->getOpcode() == ISD::STRICT_FSETCCS;
4346 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4347 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4348 unsigned Offset = IsStrict ? 1 : 0;
4349 Tmp1 = Node->getOperand(Num: 0 + Offset);
4350 Tmp2 = Node->getOperand(Num: 1 + Offset);
4351 Tmp3 = Node->getOperand(Num: 2 + Offset);
4352 SDValue Mask, EVL;
4353 if (IsVP) {
4354 Mask = Node->getOperand(Num: 3 + Offset);
4355 EVL = Node->getOperand(Num: 4 + Offset);
4356 }
4357 bool Legalized = TLI.LegalizeSetCCCondCode(
4358 DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl,
4359 Chain, IsSignaling);
4360
4361 if (Legalized) {
4362 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4363 // condition code, create a new SETCC node.
4364 if (Tmp3.getNode()) {
4365 if (IsStrict) {
4366 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4367 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4368 Chain = Tmp1.getValue(R: 1);
4369 } else if (IsVP) {
4370 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0),
4371 Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags());
4372 } else {
4373 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4374 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4375 }
4376 }
4377
4378 // If we expanded the SETCC by inverting the condition code, then wrap
4379 // the existing SETCC in a NOT to restore the intended condition.
4380 if (NeedInvert) {
4381 if (!IsVP)
4382 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4383 else
4384 Tmp1 =
4385 DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0));
4386 }
4387
4388 Results.push_back(Elt: Tmp1);
4389 if (IsStrict)
4390 Results.push_back(Elt: Chain);
4391
4392 break;
4393 }
4394
4395 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4396 // understand if this code is useful for strict nodes.
4397 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4398
4399 // Otherwise, SETCC for the given comparison type must be completely
4400 // illegal; expand it into a SELECT_CC.
4401 // FIXME: This drops the mask/evl for VP_SETCC.
4402 EVT VT = Node->getValueType(ResNo: 0);
4403 EVT Tmp1VT = Tmp1.getValueType();
4404 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4405 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4406 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3,
4407 Flags: Node->getFlags());
4408 Results.push_back(Elt: Tmp1);
4409 break;
4410 }
4411 case ISD::SELECT_CC: {
4412 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4413 Tmp1 = Node->getOperand(Num: 0); // LHS
4414 Tmp2 = Node->getOperand(Num: 1); // RHS
4415 Tmp3 = Node->getOperand(Num: 2); // True
4416 Tmp4 = Node->getOperand(Num: 3); // False
4417 EVT VT = Node->getValueType(ResNo: 0);
4418 SDValue Chain;
4419 SDValue CC = Node->getOperand(Num: 4);
4420 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4421
4422 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4423 // If the condition code is legal, then we need to expand this
4424 // node using SETCC and SELECT.
4425 EVT CmpVT = Tmp1.getValueType();
4426 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4427 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4428 "expanded.");
4429 EVT CCVT = getSetCCResultType(VT: CmpVT);
4430 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4431 Results.push_back(
4432 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4433 break;
4434 }
4435
4436 // SELECT_CC is legal, so the condition code must not be.
4437 bool Legalized = false;
4438 // Try to legalize by inverting the condition. This is for targets that
4439 // might support an ordered version of a condition, but not the unordered
4440 // version (or vice versa).
4441 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4442 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4443 // Use the new condition code and swap true and false
4444 Legalized = true;
4445 Tmp1 =
4446 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC, Flags: Node->getFlags());
4447 } else {
4448 // If The inverse is not legal, then try to swap the arguments using
4449 // the inverse condition code.
4450 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4451 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4452 // The swapped inverse condition is legal, so swap true and false,
4453 // lhs and rhs.
4454 Legalized = true;
4455 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC,
4456 Flags: Node->getFlags());
4457 }
4458 }
4459
4460 if (!Legalized) {
4461 Legalized = TLI.LegalizeSetCCCondCode(
4462 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4463 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4464
4465 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4466
4467 // If we expanded the SETCC by inverting the condition code, then swap
4468 // the True/False operands to match.
4469 if (NeedInvert)
4470 std::swap(a&: Tmp3, b&: Tmp4);
4471
4472 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4473 // condition code, create a new SELECT_CC node.
4474 if (CC.getNode()) {
4475 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4476 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4477 } else {
4478 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4479 CC = DAG.getCondCode(Cond: ISD::SETNE);
4480 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4481 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4482 }
4483 }
4484 Results.push_back(Elt: Tmp1);
4485 break;
4486 }
4487 case ISD::BR_CC: {
4488 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4489 SDValue Chain;
4490 Tmp1 = Node->getOperand(Num: 0); // Chain
4491 Tmp2 = Node->getOperand(Num: 2); // LHS
4492 Tmp3 = Node->getOperand(Num: 3); // RHS
4493 Tmp4 = Node->getOperand(Num: 1); // CC
4494
4495 bool Legalized = TLI.LegalizeSetCCCondCode(
4496 DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4,
4497 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4498 (void)Legalized;
4499 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4500
4501 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4502 // node.
4503 if (Tmp4.getNode()) {
4504 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4505
4506 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4507 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4508 } else {
4509 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4510 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4511 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4512 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4513 }
4514 Results.push_back(Elt: Tmp1);
4515 break;
4516 }
4517 case ISD::BUILD_VECTOR:
4518 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4519 break;
4520 case ISD::SPLAT_VECTOR:
4521 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4522 break;
4523 case ISD::SRA:
4524 case ISD::SRL:
4525 case ISD::SHL: {
4526 // Scalarize vector SRA/SRL/SHL.
4527 EVT VT = Node->getValueType(ResNo: 0);
4528 assert(VT.isVector() && "Unable to legalize non-vector shift");
4529 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4530 unsigned NumElem = VT.getVectorNumElements();
4531
4532 SmallVector<SDValue, 8> Scalars;
4533 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4534 SDValue Ex =
4535 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4536 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4537 SDValue Sh =
4538 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4539 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4540 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4541 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4542 }
4543
4544 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4545 Results.push_back(Elt: Result);
4546 break;
4547 }
4548 case ISD::VECREDUCE_FADD:
4549 case ISD::VECREDUCE_FMUL:
4550 case ISD::VECREDUCE_ADD:
4551 case ISD::VECREDUCE_MUL:
4552 case ISD::VECREDUCE_AND:
4553 case ISD::VECREDUCE_OR:
4554 case ISD::VECREDUCE_XOR:
4555 case ISD::VECREDUCE_SMAX:
4556 case ISD::VECREDUCE_SMIN:
4557 case ISD::VECREDUCE_UMAX:
4558 case ISD::VECREDUCE_UMIN:
4559 case ISD::VECREDUCE_FMAX:
4560 case ISD::VECREDUCE_FMIN:
4561 case ISD::VECREDUCE_FMAXIMUM:
4562 case ISD::VECREDUCE_FMINIMUM:
4563 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4564 break;
4565 case ISD::VP_CTTZ_ELTS:
4566 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4567 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4568 break;
4569 case ISD::CLEAR_CACHE:
4570 // The default expansion of llvm.clear_cache is simply a no-op for those
4571 // targets where it is not needed.
4572 Results.push_back(Elt: Node->getOperand(Num: 0));
4573 break;
4574 case ISD::LRINT:
4575 case ISD::LLRINT: {
4576 SDValue Arg = Node->getOperand(Num: 0);
4577 EVT ArgVT = Arg.getValueType();
4578 EVT ResVT = Node->getValueType(ResNo: 0);
4579 SDLoc dl(Node);
4580 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL: dl, VT: ArgVT, Operand: Arg);
4581 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
4582 break;
4583 }
4584 case ISD::ADDRSPACECAST:
4585 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4586 break;
4587 case ISD::GLOBAL_OFFSET_TABLE:
4588 case ISD::GlobalAddress:
4589 case ISD::GlobalTLSAddress:
4590 case ISD::ExternalSymbol:
4591 case ISD::ConstantPool:
4592 case ISD::JumpTable:
4593 case ISD::INTRINSIC_W_CHAIN:
4594 case ISD::INTRINSIC_WO_CHAIN:
4595 case ISD::INTRINSIC_VOID:
4596 // FIXME: Custom lowering for these operations shouldn't return null!
4597 // Return true so that we don't call ConvertNodeToLibcall which also won't
4598 // do anything.
4599 return true;
4600 }
4601
4602 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4603 // FIXME: We were asked to expand a strict floating-point operation,
4604 // but there is currently no expansion implemented that would preserve
4605 // the "strict" properties. For now, we just fall back to the non-strict
4606 // version if that is legal on the target. The actual mutation of the
4607 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4608 switch (Node->getOpcode()) {
4609 default:
4610 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4611 VT: Node->getValueType(ResNo: 0))
4612 == TargetLowering::Legal)
4613 return true;
4614 break;
4615 case ISD::STRICT_FSUB: {
4616 if (TLI.getStrictFPOperationAction(
4617 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4618 return true;
4619 if (TLI.getStrictFPOperationAction(
4620 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4621 break;
4622
4623 EVT VT = Node->getValueType(ResNo: 0);
4624 const SDNodeFlags Flags = Node->getFlags();
4625 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4626 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4627 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4628 Flags);
4629
4630 Results.push_back(Elt: Fadd);
4631 Results.push_back(Elt: Fadd.getValue(R: 1));
4632 break;
4633 }
4634 case ISD::STRICT_SINT_TO_FP:
4635 case ISD::STRICT_UINT_TO_FP:
4636 case ISD::STRICT_LRINT:
4637 case ISD::STRICT_LLRINT:
4638 case ISD::STRICT_LROUND:
4639 case ISD::STRICT_LLROUND:
4640 // These are registered by the operand type instead of the value
4641 // type. Reflect that here.
4642 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4643 VT: Node->getOperand(Num: 1).getValueType())
4644 == TargetLowering::Legal)
4645 return true;
4646 break;
4647 }
4648 }
4649
4650 // Replace the original node with the legalized result.
4651 if (Results.empty()) {
4652 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4653 return false;
4654 }
4655
4656 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4657 ReplaceNode(Old: Node, New: Results.data());
4658 return true;
4659}
4660
4661/// Return if we can use the FAST_* variant of a math libcall for the node.
4662/// FIXME: This is just guessing, we probably should have unique specific sets
4663/// flags required per libcall.
4664static bool canUseFastMathLibcall(const SDNode *Node) {
4665 // FIXME: Probably should define fast to respect nan/inf and only be
4666 // approximate functions.
4667
4668 SDNodeFlags Flags = Node->getFlags();
4669 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4670 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4671}
4672
4673void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4674 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4675 SmallVector<SDValue, 8> Results;
4676 SDLoc dl(Node);
4677 TargetLowering::MakeLibCallOptions CallOptions;
4678 CallOptions.IsPostTypeLegalization = true;
4679 // FIXME: Check flags on the node to see if we can use a finite call.
4680 unsigned Opc = Node->getOpcode();
4681 switch (Opc) {
4682 case ISD::ATOMIC_FENCE: {
4683 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4684 // FIXME: handle "fence singlethread" more efficiently.
4685 TargetLowering::ArgListTy Args;
4686
4687 TargetLowering::CallLoweringInfo CLI(DAG);
4688 CLI.setDebugLoc(dl)
4689 .setChain(Node->getOperand(Num: 0))
4690 .setLibCallee(
4691 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4692 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4693 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4694 ArgsList: std::move(Args));
4695
4696 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4697
4698 Results.push_back(Elt: CallResult.second);
4699 break;
4700 }
4701 // By default, atomic intrinsics are marked Legal and lowered. Targets
4702 // which don't support them directly, however, may want libcalls, in which
4703 // case they mark them Expand, and we get here.
4704 case ISD::ATOMIC_SWAP:
4705 case ISD::ATOMIC_LOAD_ADD:
4706 case ISD::ATOMIC_LOAD_SUB:
4707 case ISD::ATOMIC_LOAD_AND:
4708 case ISD::ATOMIC_LOAD_CLR:
4709 case ISD::ATOMIC_LOAD_OR:
4710 case ISD::ATOMIC_LOAD_XOR:
4711 case ISD::ATOMIC_LOAD_NAND:
4712 case ISD::ATOMIC_LOAD_MIN:
4713 case ISD::ATOMIC_LOAD_MAX:
4714 case ISD::ATOMIC_LOAD_UMIN:
4715 case ISD::ATOMIC_LOAD_UMAX:
4716 case ISD::ATOMIC_CMP_SWAP: {
4717 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4718 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4719 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4720 EVT RetVT = Node->getValueType(ResNo: 0);
4721 SmallVector<SDValue, 4> Ops;
4722 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported) {
4723 // If outline atomic available, prepare its arguments and expand.
4724 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4725 Ops.push_back(Elt: Node->getOperand(Num: 1));
4726
4727 } else {
4728 LC = RTLIB::getSYNC(Opc, VT);
4729 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4730 "Unexpected atomic op or value type!");
4731 // Arguments for expansion to sync libcall
4732 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4733 }
4734 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4735 Ops, CallOptions,
4736 dl: SDLoc(Node),
4737 Chain: Node->getOperand(Num: 0));
4738 Results.push_back(Elt: Tmp.first);
4739 Results.push_back(Elt: Tmp.second);
4740 break;
4741 }
4742 case ISD::TRAP: {
4743 // If this operation is not supported, lower it to 'abort()' call
4744 TargetLowering::ArgListTy Args;
4745 TargetLowering::CallLoweringInfo CLI(DAG);
4746 CLI.setDebugLoc(dl)
4747 .setChain(Node->getOperand(Num: 0))
4748 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4749 Target: DAG.getExternalSymbol(
4750 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4751 ArgsList: std::move(Args));
4752 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4753
4754 Results.push_back(Elt: CallResult.second);
4755 break;
4756 }
4757 case ISD::CLEAR_CACHE: {
4758 SDValue InputChain = Node->getOperand(Num: 0);
4759 SDValue StartVal = Node->getOperand(Num: 1);
4760 SDValue EndVal = Node->getOperand(Num: 2);
4761 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4762 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
4763 dl: SDLoc(Node), Chain: InputChain);
4764 Results.push_back(Elt: Tmp.second);
4765 break;
4766 }
4767 case ISD::FMINNUM:
4768 case ISD::STRICT_FMINNUM:
4769 ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
4770 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
4771 Call_PPCF128: RTLIB::FMIN_PPCF128, Results);
4772 break;
4773 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4774 // libcall legalization for these nodes, but there is no default expasion for
4775 // these nodes either (see PR63267 for example).
4776 case ISD::FMAXNUM:
4777 case ISD::STRICT_FMAXNUM:
4778 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
4779 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
4780 Call_PPCF128: RTLIB::FMAX_PPCF128, Results);
4781 break;
4782 case ISD::FMINIMUMNUM:
4783 ExpandFPLibCall(Node, Call_F32: RTLIB::FMINIMUM_NUM_F32, Call_F64: RTLIB::FMINIMUM_NUM_F64,
4784 Call_F80: RTLIB::FMINIMUM_NUM_F80, Call_F128: RTLIB::FMINIMUM_NUM_F128,
4785 Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128, Results);
4786 break;
4787 case ISD::FMAXIMUMNUM:
4788 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAXIMUM_NUM_F32, Call_F64: RTLIB::FMAXIMUM_NUM_F64,
4789 Call_F80: RTLIB::FMAXIMUM_NUM_F80, Call_F128: RTLIB::FMAXIMUM_NUM_F128,
4790 Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128, Results);
4791 break;
4792 case ISD::FSQRT:
4793 case ISD::STRICT_FSQRT: {
4794 // FIXME: Probably should define fast to respect nan/inf and only be
4795 // approximate functions.
4796 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
4797 Call_F32: {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
4798 Call_F64: {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
4799 Call_F80: {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
4800 Call_F128: {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
4801 Call_PPCF128: {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
4802 Results);
4803 break;
4804 }
4805 case ISD::FCBRT:
4806 ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64,
4807 Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128,
4808 Call_PPCF128: RTLIB::CBRT_PPCF128, Results);
4809 break;
4810 case ISD::FSIN:
4811 case ISD::STRICT_FSIN:
4812 ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
4813 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
4814 Call_PPCF128: RTLIB::SIN_PPCF128, Results);
4815 break;
4816 case ISD::FCOS:
4817 case ISD::STRICT_FCOS:
4818 ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
4819 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
4820 Call_PPCF128: RTLIB::COS_PPCF128, Results);
4821 break;
4822 case ISD::FTAN:
4823 case ISD::STRICT_FTAN:
4824 ExpandFPLibCall(Node, Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
4825 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128, Results);
4826 break;
4827 case ISD::FASIN:
4828 case ISD::STRICT_FASIN:
4829 ExpandFPLibCall(Node, Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
4830 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128, Results);
4831 break;
4832 case ISD::FACOS:
4833 case ISD::STRICT_FACOS:
4834 ExpandFPLibCall(Node, Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
4835 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128, Results);
4836 break;
4837 case ISD::FATAN:
4838 case ISD::STRICT_FATAN:
4839 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
4840 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128, Results);
4841 break;
4842 case ISD::FATAN2:
4843 case ISD::STRICT_FATAN2:
4844 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN2_F32, Call_F64: RTLIB::ATAN2_F64, Call_F80: RTLIB::ATAN2_F80,
4845 Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128, Results);
4846 break;
4847 case ISD::FSINH:
4848 case ISD::STRICT_FSINH:
4849 ExpandFPLibCall(Node, Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
4850 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128, Results);
4851 break;
4852 case ISD::FCOSH:
4853 case ISD::STRICT_FCOSH:
4854 ExpandFPLibCall(Node, Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
4855 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128, Results);
4856 break;
4857 case ISD::FTANH:
4858 case ISD::STRICT_FTANH:
4859 ExpandFPLibCall(Node, Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
4860 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128, Results);
4861 break;
4862 case ISD::FSINCOS:
4863 case ISD::FSINCOSPI: {
4864 EVT VT = Node->getValueType(ResNo: 0);
4865
4866 if (Node->getOpcode() == ISD::FSINCOS) {
4867 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(RetVT: VT);
4868 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
4869 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
4870 Results.push_back(Elt: Expanded);
4871 Results.push_back(Elt: Expanded.getValue(R: 1));
4872 break;
4873 }
4874 }
4875 }
4876
4877 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4878 ? RTLIB::getSINCOS(RetVT: VT)
4879 : RTLIB::getSINCOSPI(RetVT: VT);
4880 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);
4881 if (!Expanded) {
4882 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
4883 Node->getOperationName(G: &DAG));
4884 SDValue Poison = DAG.getPOISON(VT);
4885 Results.push_back(Elt: Poison);
4886 Results.push_back(Elt: Poison);
4887 }
4888
4889 break;
4890 }
4891 case ISD::FLOG:
4892 case ISD::STRICT_FLOG:
4893 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80,
4894 Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results);
4895 break;
4896 case ISD::FLOG2:
4897 case ISD::STRICT_FLOG2:
4898 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80,
4899 Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results);
4900 break;
4901 case ISD::FLOG10:
4902 case ISD::STRICT_FLOG10:
4903 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80,
4904 Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results);
4905 break;
4906 case ISD::FEXP:
4907 case ISD::STRICT_FEXP:
4908 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80,
4909 Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results);
4910 break;
4911 case ISD::FEXP2:
4912 case ISD::STRICT_FEXP2:
4913 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80,
4914 Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results);
4915 break;
4916 case ISD::FEXP10:
4917 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
4918 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results);
4919 break;
4920 case ISD::FTRUNC:
4921 case ISD::STRICT_FTRUNC:
4922 ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
4923 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
4924 Call_PPCF128: RTLIB::TRUNC_PPCF128, Results);
4925 break;
4926 case ISD::FFLOOR:
4927 case ISD::STRICT_FFLOOR:
4928 ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
4929 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
4930 Call_PPCF128: RTLIB::FLOOR_PPCF128, Results);
4931 break;
4932 case ISD::FCEIL:
4933 case ISD::STRICT_FCEIL:
4934 ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
4935 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
4936 Call_PPCF128: RTLIB::CEIL_PPCF128, Results);
4937 break;
4938 case ISD::FRINT:
4939 case ISD::STRICT_FRINT:
4940 ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
4941 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
4942 Call_PPCF128: RTLIB::RINT_PPCF128, Results);
4943 break;
4944 case ISD::FNEARBYINT:
4945 case ISD::STRICT_FNEARBYINT:
4946 ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32,
4947 Call_F64: RTLIB::NEARBYINT_F64,
4948 Call_F80: RTLIB::NEARBYINT_F80,
4949 Call_F128: RTLIB::NEARBYINT_F128,
4950 Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results);
4951 break;
4952 case ISD::FROUND:
4953 case ISD::STRICT_FROUND:
4954 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32,
4955 Call_F64: RTLIB::ROUND_F64,
4956 Call_F80: RTLIB::ROUND_F80,
4957 Call_F128: RTLIB::ROUND_F128,
4958 Call_PPCF128: RTLIB::ROUND_PPCF128, Results);
4959 break;
4960 case ISD::FROUNDEVEN:
4961 case ISD::STRICT_FROUNDEVEN:
4962 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32,
4963 Call_F64: RTLIB::ROUNDEVEN_F64,
4964 Call_F80: RTLIB::ROUNDEVEN_F80,
4965 Call_F128: RTLIB::ROUNDEVEN_F128,
4966 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results);
4967 break;
4968 case ISD::FLDEXP:
4969 case ISD::STRICT_FLDEXP:
4970 ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80,
4971 Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results);
4972 break;
4973 case ISD::FMODF:
4974 case ISD::FFREXP: {
4975 EVT VT = Node->getValueType(ResNo: 0);
4976 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
4977 : RTLIB::getFREXP(RetVT: VT);
4978 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
4979 /*CallRetResNo=*/0);
4980 if (!Expanded)
4981 llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");
4982 break;
4983 }
4984 case ISD::FPOWI:
4985 case ISD::STRICT_FPOWI: {
4986 RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0));
4987 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4988 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
4989 // Some targets don't have a powi libcall; use pow instead.
4990 if (Node->isStrictFPOpcode()) {
4991 SDValue Exponent =
4992 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
4993 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4994 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
4995 SDValue FPOW =
4996 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
4997 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4998 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
4999 Results.push_back(Elt: FPOW);
5000 Results.push_back(Elt: FPOW.getValue(R: 1));
5001 } else {
5002 SDValue Exponent =
5003 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
5004 Operand: Node->getOperand(Num: 1));
5005 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
5006 VT: Node->getValueType(ResNo: 0),
5007 N1: Node->getOperand(Num: 0), N2: Exponent));
5008 }
5009 break;
5010 }
5011 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
5012 bool ExponentHasSizeOfInt =
5013 DAG.getLibInfo().getIntSize() ==
5014 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
5015 if (!ExponentHasSizeOfInt) {
5016 // If the exponent does not match with sizeof(int) a libcall to
5017 // RTLIB::POWI would use the wrong type for the argument.
5018 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
5019 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
5020 break;
5021 }
5022 ExpandFPLibCall(Node, LC, Results);
5023 break;
5024 }
5025 case ISD::FPOW:
5026 case ISD::STRICT_FPOW:
5027 ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80,
5028 Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results);
5029 break;
5030 case ISD::LROUND:
5031 case ISD::STRICT_LROUND:
5032 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
5033 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
5034 Call_F128: RTLIB::LROUND_F128,
5035 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
5036 break;
5037 case ISD::LLROUND:
5038 case ISD::STRICT_LLROUND:
5039 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
5040 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
5041 Call_F128: RTLIB::LLROUND_F128,
5042 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
5043 break;
5044 case ISD::LRINT:
5045 case ISD::STRICT_LRINT:
5046 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
5047 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
5048 Call_F128: RTLIB::LRINT_F128,
5049 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
5050 break;
5051 case ISD::LLRINT:
5052 case ISD::STRICT_LLRINT:
5053 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
5054 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
5055 Call_F128: RTLIB::LLRINT_F128,
5056 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
5057 break;
5058 case ISD::FDIV:
5059 case ISD::STRICT_FDIV: {
5060 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5061 Call_F32: {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5062 Call_F64: {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5063 Call_F80: {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5064 Call_F128: {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5065 Call_PPCF128: {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5066 break;
5067 }
5068 case ISD::FREM:
5069 case ISD::STRICT_FREM:
5070 ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
5071 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
5072 Call_PPCF128: RTLIB::REM_PPCF128, Results);
5073 break;
5074 case ISD::FMA:
5075 case ISD::STRICT_FMA:
5076 ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64,
5077 Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128,
5078 Call_PPCF128: RTLIB::FMA_PPCF128, Results);
5079 break;
5080 case ISD::FADD:
5081 case ISD::STRICT_FADD: {
5082 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5083 Call_F32: {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5084 Call_F64: {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5085 Call_F80: {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5086 Call_F128: {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5087 Call_PPCF128: {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5088 break;
5089 }
5090 case ISD::FMUL:
5091 case ISD::STRICT_FMUL: {
5092 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5093 Call_F32: {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5094 Call_F64: {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5095 Call_F80: {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5096 Call_F128: {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5097 Call_PPCF128: {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5098 break;
5099 }
5100 case ISD::FP16_TO_FP:
5101 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5102 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
5103 }
5104 break;
5105 case ISD::STRICT_BF16_TO_FP:
5106 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5107 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5108 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
5109 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5110 Results.push_back(Elt: Tmp.first);
5111 Results.push_back(Elt: Tmp.second);
5112 }
5113 break;
5114 case ISD::STRICT_FP16_TO_FP: {
5115 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5116 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5117 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
5118 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5119 Results.push_back(Elt: Tmp.first);
5120 Results.push_back(Elt: Tmp.second);
5121 }
5122 break;
5123 }
5124 case ISD::FP_TO_FP16: {
5125 RTLIB::Libcall LC =
5126 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
5127 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5128 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5129 break;
5130 }
5131 case ISD::FP_TO_BF16: {
5132 RTLIB::Libcall LC =
5133 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
5134 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5135 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5136 break;
5137 }
5138 case ISD::STRICT_SINT_TO_FP:
5139 case ISD::STRICT_UINT_TO_FP:
5140 case ISD::SINT_TO_FP:
5141 case ISD::UINT_TO_FP: {
5142 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
5143 bool IsStrict = Node->isStrictFPOpcode();
5144 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
5145 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
5146 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
5147 EVT RVT = Node->getValueType(ResNo: 0);
5148 EVT NVT = EVT();
5149 SDLoc dl(Node);
5150
5151 // Even if the input is legal, no libcall may exactly match, eg. we don't
5152 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
5153 // eg: i13 -> fp. Then, look for an appropriate libcall.
5154 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5155 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5156 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5157 ++t) {
5158 NVT = (MVT::SimpleValueType)t;
5159 // The source needs to big enough to hold the operand.
5160 if (NVT.bitsGE(VT: SVT))
5161 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
5162 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
5163 }
5164 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5165
5166 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5167 // Sign/zero extend the argument if the libcall takes a larger type.
5168 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
5169 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
5170 CallOptions.setIsSigned(Signed);
5171 std::pair<SDValue, SDValue> Tmp =
5172 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
5173 Results.push_back(Elt: Tmp.first);
5174 if (IsStrict)
5175 Results.push_back(Elt: Tmp.second);
5176 break;
5177 }
5178 case ISD::FP_TO_SINT:
5179 case ISD::FP_TO_UINT:
5180 case ISD::STRICT_FP_TO_SINT:
5181 case ISD::STRICT_FP_TO_UINT: {
5182 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
5183 bool IsStrict = Node->isStrictFPOpcode();
5184 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
5185 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
5186
5187 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5188 EVT SVT = Op.getValueType();
5189 EVT RVT = Node->getValueType(ResNo: 0);
5190 EVT NVT = EVT();
5191 SDLoc dl(Node);
5192
5193 // Even if the result is legal, no libcall may exactly match, eg. we don't
5194 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
5195 // eg: fp -> i32. Then, look for an appropriate libcall.
5196 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5197 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5198 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5199 ++IntVT) {
5200 NVT = (MVT::SimpleValueType)IntVT;
5201 // The type needs to big enough to hold the result.
5202 if (NVT.bitsGE(VT: RVT))
5203 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
5204 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
5205 }
5206 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5207
5208 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5209 std::pair<SDValue, SDValue> Tmp =
5210 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
5211
5212 // Truncate the result if the libcall returns a larger type.
5213 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
5214 if (IsStrict)
5215 Results.push_back(Elt: Tmp.second);
5216 break;
5217 }
5218
5219 case ISD::FP_ROUND:
5220 case ISD::STRICT_FP_ROUND: {
5221 // X = FP_ROUND(Y, TRUNC)
5222 // TRUNC is a flag, which is always an integer that is zero or one.
5223 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5224 // is known to not change the value of Y.
5225 // We can only expand it into libcall if the TRUNC is 0.
5226 bool IsStrict = Node->isStrictFPOpcode();
5227 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5228 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5229 EVT VT = Node->getValueType(ResNo: 0);
5230 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5231 "Unable to expand as libcall if it is not normal rounding");
5232
5233 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5234 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5235
5236 std::pair<SDValue, SDValue> Tmp =
5237 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5238 Results.push_back(Elt: Tmp.first);
5239 if (IsStrict)
5240 Results.push_back(Elt: Tmp.second);
5241 break;
5242 }
5243 case ISD::FP_EXTEND: {
5244 Results.push_back(
5245 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5246 RetVT: Node->getValueType(ResNo: 0)),
5247 Node, isSigned: false).first);
5248 break;
5249 }
5250 case ISD::STRICT_FP_EXTEND:
5251 case ISD::STRICT_FP_TO_FP16:
5252 case ISD::STRICT_FP_TO_BF16: {
5253 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5254 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5255 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5256 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5257 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5258 else
5259 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5260 RetVT: Node->getValueType(ResNo: 0));
5261
5262 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5263
5264 std::pair<SDValue, SDValue> Tmp =
5265 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5266 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5267 Results.push_back(Elt: Tmp.first);
5268 Results.push_back(Elt: Tmp.second);
5269 break;
5270 }
5271 case ISD::FSUB:
5272 case ISD::STRICT_FSUB: {
5273 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5274 Call_F32: {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5275 Call_F64: {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5276 Call_F80: {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5277 Call_F128: {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5278 Call_PPCF128: {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5279 break;
5280 }
5281 case ISD::SREM:
5282 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5283 Call_I8: RTLIB::SREM_I8,
5284 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5285 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5286 break;
5287 case ISD::UREM:
5288 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5289 Call_I8: RTLIB::UREM_I8,
5290 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5291 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5292 break;
5293 case ISD::SDIV:
5294 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5295 Call_I8: RTLIB::SDIV_I8,
5296 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5297 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5298 break;
5299 case ISD::UDIV:
5300 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5301 Call_I8: RTLIB::UDIV_I8,
5302 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5303 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5304 break;
5305 case ISD::SDIVREM:
5306 case ISD::UDIVREM:
5307 // Expand into divrem libcall
5308 ExpandDivRemLibCall(Node, Results);
5309 break;
5310 case ISD::MUL:
5311 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5312 Call_I8: RTLIB::MUL_I8,
5313 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5314 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5315 break;
5316 case ISD::CTLZ_ZERO_UNDEF:
5317 Results.push_back(Elt: ExpandBitCountingLibCall(
5318 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5319 break;
5320 case ISD::CTPOP:
5321 Results.push_back(Elt: ExpandBitCountingLibCall(
5322 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5323 break;
5324 case ISD::RESET_FPENV: {
5325 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5326 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5327 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5328 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5329 SDValue Chain = Node->getOperand(Num: 0);
5330 Results.push_back(
5331 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
5332 break;
5333 }
5334 case ISD::GET_FPENV_MEM: {
5335 SDValue Chain = Node->getOperand(Num: 0);
5336 SDValue EnvPtr = Node->getOperand(Num: 1);
5337 Results.push_back(
5338 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5339 break;
5340 }
5341 case ISD::SET_FPENV_MEM: {
5342 SDValue Chain = Node->getOperand(Num: 0);
5343 SDValue EnvPtr = Node->getOperand(Num: 1);
5344 Results.push_back(
5345 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5346 break;
5347 }
5348 case ISD::GET_FPMODE: {
5349 // Call fegetmode, which saves control modes into a stack slot. Then load
5350 // the value to return from the stack.
5351 EVT ModeVT = Node->getValueType(ResNo: 0);
5352 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5353 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5354 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5355 InChain: Node->getOperand(Num: 0), DLoc: dl);
5356 SDValue LdInst = DAG.getLoad(
5357 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5358 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5359 Results.push_back(Elt: LdInst);
5360 Results.push_back(Elt: LdInst.getValue(R: 1));
5361 break;
5362 }
5363 case ISD::SET_FPMODE: {
5364 // Move control modes to stack slot and then call fesetmode with the pointer
5365 // to the slot as argument.
5366 SDValue Mode = Node->getOperand(Num: 1);
5367 EVT ModeVT = Mode.getValueType();
5368 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5369 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5370 SDValue StInst = DAG.getStore(
5371 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5372 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5373 Results.push_back(
5374 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
5375 break;
5376 }
5377 case ISD::RESET_FPMODE: {
5378 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5379 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5380 // target must provide custom lowering.
5381 const DataLayout &DL = DAG.getDataLayout();
5382 EVT PtrTy = TLI.getPointerTy(DL);
5383 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5384 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5385 InChain: Node->getOperand(Num: 0), DLoc: dl));
5386 break;
5387 }
5388 }
5389
5390 // Replace the original node with the legalized result.
5391 if (!Results.empty()) {
5392 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5393 ReplaceNode(Old: Node, New: Results.data());
5394 } else
5395 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5396}
5397
5398// Determine the vector type to use in place of an original scalar element when
5399// promoting equally sized vectors.
5400static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5401 MVT EltVT, MVT NewEltVT) {
5402 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5403 MVT MidVT = OldEltsPerNewElt == 1
5404 ? NewEltVT
5405 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5406 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5407 return MidVT;
5408}
5409
5410void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5411 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5412 SmallVector<SDValue, 8> Results;
5413 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5414 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5415 Node->getOpcode() == ISD::SINT_TO_FP ||
5416 Node->getOpcode() == ISD::SETCC ||
5417 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5418 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5419 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5420 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5421 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5422 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM) {
5423 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5424 }
5425 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5426 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5427 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5428 Node->getOpcode() == ISD::STRICT_FSETCC ||
5429 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5430 Node->getOpcode() == ISD::STRICT_LRINT ||
5431 Node->getOpcode() == ISD::STRICT_LLRINT ||
5432 Node->getOpcode() == ISD::STRICT_LROUND ||
5433 Node->getOpcode() == ISD::STRICT_LLROUND ||
5434 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5435 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5436 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5437 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5438 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5439 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5440 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5441 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5442 if (Node->getOpcode() == ISD::BR_CC ||
5443 Node->getOpcode() == ISD::SELECT_CC)
5444 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5445 // Preserve fast math flags
5446 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5447 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5448 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5449 SDLoc dl(Node);
5450 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5451 switch (Node->getOpcode()) {
5452 case ISD::CTTZ:
5453 case ISD::CTTZ_ZERO_UNDEF:
5454 case ISD::CTLZ:
5455 case ISD::CTPOP: {
5456 // Zero extend the argument unless its cttz, then use any_extend.
5457 if (Node->getOpcode() == ISD::CTTZ ||
5458 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
5459 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5460 else
5461 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5462
5463 unsigned NewOpc = Node->getOpcode();
5464 if (NewOpc == ISD::CTTZ) {
5465 // The count is the same in the promoted type except if the original
5466 // value was zero. This can be handled by setting the bit just off
5467 // the top of the original type.
5468 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5469 BitNo: OVT.getSizeInBits());
5470 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5471 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5472 NewOpc = ISD::CTTZ_ZERO_UNDEF;
5473 }
5474 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
5475 // already the correct result.
5476 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5477 if (NewOpc == ISD::CTLZ) {
5478 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5479 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5480 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5481 OVT.getSizeInBits(), DL: dl, VT: NVT));
5482 }
5483 Results.push_back(
5484 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5485 break;
5486 }
5487 case ISD::CTLZ_ZERO_UNDEF: {
5488 // We know that the argument is unlikely to be zero, hence we can take a
5489 // different approach as compared to ISD::CTLZ
5490
5491 // Any Extend the argument
5492 auto AnyExtendedNode =
5493 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5494
5495 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5496 auto ShiftConstant = DAG.getShiftAmountConstant(
5497 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5498 auto LeftShiftResult =
5499 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5500
5501 // Perform the larger operation
5502 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5503 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5504 break;
5505 }
5506 case ISD::BITREVERSE:
5507 case ISD::BSWAP: {
5508 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5509 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5510 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5511 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5512 N2: DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl));
5513
5514 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5515 break;
5516 }
5517 case ISD::FP_TO_UINT:
5518 case ISD::STRICT_FP_TO_UINT:
5519 case ISD::FP_TO_SINT:
5520 case ISD::STRICT_FP_TO_SINT:
5521 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5522 break;
5523 case ISD::FP_TO_UINT_SAT:
5524 case ISD::FP_TO_SINT_SAT:
5525 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5526 break;
5527 case ISD::UINT_TO_FP:
5528 case ISD::STRICT_UINT_TO_FP:
5529 case ISD::SINT_TO_FP:
5530 case ISD::STRICT_SINT_TO_FP:
5531 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5532 break;
5533 case ISD::VAARG: {
5534 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5535 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5536
5537 unsigned TruncOp;
5538 if (OVT.isVector()) {
5539 TruncOp = ISD::BITCAST;
5540 } else {
5541 assert(OVT.isInteger()
5542 && "VAARG promotion is supported only for vectors or integer types");
5543 TruncOp = ISD::TRUNCATE;
5544 }
5545
5546 // Perform the larger operation, then convert back
5547 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5548 Align: Node->getConstantOperandVal(Num: 3));
5549 Chain = Tmp1.getValue(R: 1);
5550
5551 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5552
5553 // Modified the chain result - switch anything that used the old chain to
5554 // use the new one.
5555 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5556 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5557 if (UpdatedNodes) {
5558 UpdatedNodes->insert(X: Tmp2.getNode());
5559 UpdatedNodes->insert(X: Chain.getNode());
5560 }
5561 ReplacedNode(N: Node);
5562 break;
5563 }
5564 case ISD::MUL:
5565 case ISD::SDIV:
5566 case ISD::SREM:
5567 case ISD::UDIV:
5568 case ISD::UREM:
5569 case ISD::SMIN:
5570 case ISD::SMAX:
5571 case ISD::UMIN:
5572 case ISD::UMAX:
5573 case ISD::AND:
5574 case ISD::OR:
5575 case ISD::XOR: {
5576 unsigned ExtOp, TruncOp;
5577 if (OVT.isVector()) {
5578 ExtOp = ISD::BITCAST;
5579 TruncOp = ISD::BITCAST;
5580 } else {
5581 assert(OVT.isInteger() && "Cannot promote logic operation");
5582
5583 switch (Node->getOpcode()) {
5584 default:
5585 ExtOp = ISD::ANY_EXTEND;
5586 break;
5587 case ISD::SDIV:
5588 case ISD::SREM:
5589 case ISD::SMIN:
5590 case ISD::SMAX:
5591 ExtOp = ISD::SIGN_EXTEND;
5592 break;
5593 case ISD::UDIV:
5594 case ISD::UREM:
5595 ExtOp = ISD::ZERO_EXTEND;
5596 break;
5597 case ISD::UMIN:
5598 case ISD::UMAX:
5599 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5600 ExtOp = ISD::SIGN_EXTEND;
5601 else
5602 ExtOp = ISD::ZERO_EXTEND;
5603 break;
5604 }
5605 TruncOp = ISD::TRUNCATE;
5606 }
5607 // Promote each of the values to the new type.
5608 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5609 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5610 // Perform the larger operation, then convert back
5611 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5612 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5613 break;
5614 }
5615 case ISD::UMUL_LOHI:
5616 case ISD::SMUL_LOHI: {
5617 // Promote to a multiply in a wider integer type.
5618 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5619 : ISD::SIGN_EXTEND;
5620 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5621 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5622 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5623
5624 unsigned OriginalSize = OVT.getScalarSizeInBits();
5625 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5626 N2: DAG.getShiftAmountConstant(Val: OriginalSize, VT: NVT, DL: dl));
5627 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5628 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5629 break;
5630 }
5631 case ISD::SELECT: {
5632 unsigned ExtOp, TruncOp;
5633 if (Node->getValueType(ResNo: 0).isVector() ||
5634 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5635 ExtOp = ISD::BITCAST;
5636 TruncOp = ISD::BITCAST;
5637 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5638 ExtOp = ISD::ANY_EXTEND;
5639 TruncOp = ISD::TRUNCATE;
5640 } else {
5641 ExtOp = ISD::FP_EXTEND;
5642 TruncOp = ISD::FP_ROUND;
5643 }
5644 Tmp1 = Node->getOperand(Num: 0);
5645 // Promote each of the values to the new type.
5646 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5647 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5648 // Perform the larger operation, then round down.
5649 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5650 if (TruncOp != ISD::FP_ROUND)
5651 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5652 else
5653 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5654 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5655 Results.push_back(Elt: Tmp1);
5656 break;
5657 }
5658 case ISD::VECTOR_SHUFFLE: {
5659 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5660
5661 // Cast the two input vectors.
5662 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5663 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5664
5665 // Convert the shuffle mask to the right # elements.
5666 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5667 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5668 Results.push_back(Elt: Tmp1);
5669 break;
5670 }
5671 case ISD::VECTOR_SPLICE_LEFT:
5672 case ISD::VECTOR_SPLICE_RIGHT: {
5673 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5674 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5675 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5676 N3: Node->getOperand(Num: 2));
5677 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5678 break;
5679 }
5680 case ISD::SELECT_CC: {
5681 SDValue Cond = Node->getOperand(Num: 4);
5682 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5683 // Type of the comparison operands.
5684 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5685 assert(CVT == OVT && "not handled");
5686
5687 unsigned ExtOp = ISD::FP_EXTEND;
5688 if (NVT.isInteger()) {
5689 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5690 }
5691
5692 // Promote the comparison operands, if needed.
5693 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5694 Tmp1 = Node->getOperand(Num: 0);
5695 Tmp2 = Node->getOperand(Num: 1);
5696 } else {
5697 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5698 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5699 }
5700 // Cast the true/false operands.
5701 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5702 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5703
5704 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5705 Flags: Node->getFlags());
5706
5707 // Cast the result back to the original type.
5708 if (ExtOp != ISD::FP_EXTEND)
5709 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5710 else
5711 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5712 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5713
5714 Results.push_back(Elt: Tmp1);
5715 break;
5716 }
5717 case ISD::SETCC:
5718 case ISD::STRICT_FSETCC:
5719 case ISD::STRICT_FSETCCS: {
5720 unsigned ExtOp = ISD::FP_EXTEND;
5721 if (NVT.isInteger()) {
5722 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5723 if (isSignedIntSetCC(Code: CCCode) ||
5724 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5725 ExtOp = ISD::SIGN_EXTEND;
5726 else
5727 ExtOp = ISD::ZERO_EXTEND;
5728 }
5729 if (Node->isStrictFPOpcode()) {
5730 SDValue InChain = Node->getOperand(Num: 0);
5731 std::tie(args&: Tmp1, args: std::ignore) =
5732 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5733 std::tie(args&: Tmp2, args: std::ignore) =
5734 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5735 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5736 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5737 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
5738 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5739 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5740 Flags: Node->getFlags()));
5741 Results.push_back(Elt: Results.back().getValue(R: 1));
5742 break;
5743 }
5744 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5745 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5746 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5747 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5748 break;
5749 }
5750 case ISD::BR_CC: {
5751 unsigned ExtOp = ISD::FP_EXTEND;
5752 if (NVT.isInteger()) {
5753 ISD::CondCode CCCode =
5754 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5755 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5756 }
5757 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5758 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5759 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5760 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5761 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5762 break;
5763 }
5764 case ISD::FADD:
5765 case ISD::FSUB:
5766 case ISD::FMUL:
5767 case ISD::FDIV:
5768 case ISD::FREM:
5769 case ISD::FMINNUM:
5770 case ISD::FMAXNUM:
5771 case ISD::FMINIMUM:
5772 case ISD::FMAXIMUM:
5773 case ISD::FMINIMUMNUM:
5774 case ISD::FMAXIMUMNUM:
5775 case ISD::FPOW:
5776 case ISD::FATAN2:
5777 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5778 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5779 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5780 Results.push_back(
5781 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5782 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5783 break;
5784
5785 case ISD::STRICT_FMINIMUM:
5786 case ISD::STRICT_FMAXIMUM: {
5787 SDValue InChain = Node->getOperand(Num: 0);
5788 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
5789 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5790 N2: Node->getOperand(Num: 1));
5791 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5792 N2: Node->getOperand(Num: 2));
5793 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
5794 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
5795 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
5796 N1: InChain, N2: Tmp3,
5797 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5798 Results.push_back(Elt: Tmp4);
5799 Results.push_back(Elt: Tmp4.getValue(R: 1));
5800 break;
5801 }
5802
5803 case ISD::STRICT_FADD:
5804 case ISD::STRICT_FSUB:
5805 case ISD::STRICT_FMUL:
5806 case ISD::STRICT_FDIV:
5807 case ISD::STRICT_FMINNUM:
5808 case ISD::STRICT_FMAXNUM:
5809 case ISD::STRICT_FREM:
5810 case ISD::STRICT_FPOW:
5811 case ISD::STRICT_FATAN2:
5812 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5813 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5814 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5815 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5816 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5817 N2: Tmp2.getValue(R: 1));
5818 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5819 Ops: {Tmp3, Tmp1, Tmp2});
5820 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5821 Ops: {Tmp1.getValue(R: 1), Tmp1,
5822 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5823 Results.push_back(Elt: Tmp1);
5824 Results.push_back(Elt: Tmp1.getValue(R: 1));
5825 break;
5826 case ISD::FMA:
5827 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5828 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5829 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5830 Results.push_back(
5831 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5832 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5833 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5834 break;
5835 case ISD::STRICT_FMA:
5836 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5837 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5838 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5839 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5840 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5841 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
5842 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5843 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
5844 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5845 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
5846 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5847 Ops: {Tmp4.getValue(R: 1), Tmp4,
5848 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5849 Results.push_back(Elt: Tmp4);
5850 Results.push_back(Elt: Tmp4.getValue(R: 1));
5851 break;
5852 case ISD::FCOPYSIGN:
5853 case ISD::FLDEXP:
5854 case ISD::FPOWI: {
5855 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5856 Tmp2 = Node->getOperand(Num: 1);
5857 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5858
5859 // fcopysign doesn't change anything but the sign bit, so
5860 // (fp_round (fcopysign (fpext a), b))
5861 // is as precise as
5862 // (fp_round (fpext a))
5863 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5864 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5865 Results.push_back(
5866 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5867 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5868 break;
5869 }
5870 case ISD::STRICT_FLDEXP: {
5871 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5872 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5873 Tmp2 = Node->getOperand(Num: 2);
5874 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
5875 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
5876 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5877 Ops: {Tmp3.getValue(R: 1), Tmp3,
5878 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5879 Results.push_back(Elt: Tmp4);
5880 Results.push_back(Elt: Tmp4.getValue(R: 1));
5881 break;
5882 }
5883 case ISD::STRICT_FPOWI:
5884 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5885 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5886 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5887 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
5888 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5889 Ops: {Tmp2.getValue(R: 1), Tmp2,
5890 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5891 Results.push_back(Elt: Tmp3);
5892 Results.push_back(Elt: Tmp3.getValue(R: 1));
5893 break;
5894 case ISD::FFREXP: {
5895 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5896 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5897
5898 Results.push_back(
5899 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5900 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5901
5902 Results.push_back(Elt: Tmp2.getValue(R: 1));
5903 break;
5904 }
5905 case ISD::FMODF:
5906 case ISD::FSINCOS:
5907 case ISD::FSINCOSPI: {
5908 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5909 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
5910 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
5911 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5912 Results.push_back(
5913 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
5914 break;
5915 }
5916 case ISD::FFLOOR:
5917 case ISD::FCEIL:
5918 case ISD::FRINT:
5919 case ISD::FNEARBYINT:
5920 case ISD::FROUND:
5921 case ISD::FROUNDEVEN:
5922 case ISD::FTRUNC:
5923 case ISD::FNEG:
5924 case ISD::FSQRT:
5925 case ISD::FSIN:
5926 case ISD::FCOS:
5927 case ISD::FTAN:
5928 case ISD::FASIN:
5929 case ISD::FACOS:
5930 case ISD::FATAN:
5931 case ISD::FSINH:
5932 case ISD::FCOSH:
5933 case ISD::FTANH:
5934 case ISD::FLOG:
5935 case ISD::FLOG2:
5936 case ISD::FLOG10:
5937 case ISD::FABS:
5938 case ISD::FEXP:
5939 case ISD::FEXP2:
5940 case ISD::FEXP10:
5941 case ISD::FCANONICALIZE:
5942 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5943 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5944 Results.push_back(
5945 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5946 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5947 break;
5948 case ISD::STRICT_FFLOOR:
5949 case ISD::STRICT_FCEIL:
5950 case ISD::STRICT_FRINT:
5951 case ISD::STRICT_FNEARBYINT:
5952 case ISD::STRICT_FROUND:
5953 case ISD::STRICT_FROUNDEVEN:
5954 case ISD::STRICT_FTRUNC:
5955 case ISD::STRICT_FSQRT:
5956 case ISD::STRICT_FSIN:
5957 case ISD::STRICT_FCOS:
5958 case ISD::STRICT_FTAN:
5959 case ISD::STRICT_FASIN:
5960 case ISD::STRICT_FACOS:
5961 case ISD::STRICT_FATAN:
5962 case ISD::STRICT_FSINH:
5963 case ISD::STRICT_FCOSH:
5964 case ISD::STRICT_FTANH:
5965 case ISD::STRICT_FLOG:
5966 case ISD::STRICT_FLOG2:
5967 case ISD::STRICT_FLOG10:
5968 case ISD::STRICT_FEXP:
5969 case ISD::STRICT_FEXP2:
5970 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5971 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5972 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5973 Ops: {Tmp1.getValue(R: 1), Tmp1});
5974 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5975 Ops: {Tmp2.getValue(R: 1), Tmp2,
5976 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5977 Results.push_back(Elt: Tmp3);
5978 Results.push_back(Elt: Tmp3.getValue(R: 1));
5979 break;
5980 case ISD::LLROUND:
5981 case ISD::LROUND:
5982 case ISD::LRINT:
5983 case ISD::LLRINT:
5984 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5985 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5986 Results.push_back(Elt: Tmp2);
5987 break;
5988 case ISD::STRICT_LLROUND:
5989 case ISD::STRICT_LROUND:
5990 case ISD::STRICT_LRINT:
5991 case ISD::STRICT_LLRINT:
5992 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5993 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5994 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5995 Ops: {Tmp1.getValue(R: 1), Tmp1});
5996 Results.push_back(Elt: Tmp2);
5997 Results.push_back(Elt: Tmp2.getValue(R: 1));
5998 break;
5999 case ISD::BUILD_VECTOR: {
6000 MVT EltVT = OVT.getVectorElementType();
6001 MVT NewEltVT = NVT.getVectorElementType();
6002
6003 // Handle bitcasts to a different vector type with the same total bit size
6004 //
6005 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
6006 // =>
6007 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
6008
6009 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6010 "Invalid promote type for build_vector");
6011 assert(NewEltVT.bitsLE(EltVT) && "not handled");
6012
6013 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6014
6015 SmallVector<SDValue, 8> NewOps;
6016 for (const SDValue &Op : Node->op_values())
6017 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
6018
6019 SDLoc SL(Node);
6020 SDValue Concat =
6021 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
6022 DL: SL, VT: NVT, Ops: NewOps);
6023 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6024 Results.push_back(Elt: CvtVec);
6025 break;
6026 }
6027 case ISD::EXTRACT_VECTOR_ELT: {
6028 MVT EltVT = OVT.getVectorElementType();
6029 MVT NewEltVT = NVT.getVectorElementType();
6030
6031 // Handle bitcasts to a different vector type with the same total bit size.
6032 //
6033 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
6034 // =>
6035 // v4i32:castx = bitcast x:v2i64
6036 //
6037 // i64 = bitcast
6038 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
6039 // (i32 (extract_vector_elt castx, (2 * y + 1)))
6040 //
6041
6042 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6043 "Invalid promote type for extract_vector_elt");
6044 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6045
6046 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6047 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6048
6049 SDValue Idx = Node->getOperand(Num: 1);
6050 EVT IdxVT = Idx.getValueType();
6051 SDLoc SL(Node);
6052 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
6053 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6054
6055 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6056
6057 SmallVector<SDValue, 8> NewOps;
6058 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6059 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6060 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6061
6062 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6063 N1: CastVec, N2: TmpIdx);
6064 NewOps.push_back(Elt);
6065 }
6066
6067 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
6068 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
6069 break;
6070 }
6071 case ISD::INSERT_VECTOR_ELT: {
6072 MVT EltVT = OVT.getVectorElementType();
6073 MVT NewEltVT = NVT.getVectorElementType();
6074
6075 // Handle bitcasts to a different vector type with the same total bit size
6076 //
6077 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
6078 // =>
6079 // v4i32:castx = bitcast x:v2i64
6080 // v2i32:casty = bitcast y:i64
6081 //
6082 // v2i64 = bitcast
6083 // (v4i32 insert_vector_elt
6084 // (v4i32 insert_vector_elt v4i32:castx,
6085 // (extract_vector_elt casty, 0), 2 * z),
6086 // (extract_vector_elt casty, 1), (2 * z + 1))
6087
6088 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6089 "Invalid promote type for insert_vector_elt");
6090 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6091
6092 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6093 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6094
6095 SDValue Val = Node->getOperand(Num: 1);
6096 SDValue Idx = Node->getOperand(Num: 2);
6097 EVT IdxVT = Idx.getValueType();
6098 SDLoc SL(Node);
6099
6100 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
6101 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6102
6103 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6104 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6105
6106 SDValue NewVec = CastVec;
6107 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6108 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6109 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6110
6111 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6112 N1: CastVal, N2: IdxOffset);
6113
6114 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
6115 N1: NewVec, N2: Elt, N3: InEltIdx);
6116 }
6117
6118 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
6119 break;
6120 }
6121 case ISD::SCALAR_TO_VECTOR: {
6122 MVT EltVT = OVT.getVectorElementType();
6123 MVT NewEltVT = NVT.getVectorElementType();
6124
6125 // Handle bitcasts to different vector type with the same total bit size.
6126 //
6127 // e.g. v2i64 = scalar_to_vector x:i64
6128 // =>
6129 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
6130 //
6131
6132 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6133 SDValue Val = Node->getOperand(Num: 0);
6134 SDLoc SL(Node);
6135
6136 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6137 SDValue Undef = DAG.getUNDEF(VT: MidVT);
6138
6139 SmallVector<SDValue, 8> NewElts;
6140 NewElts.push_back(Elt: CastVal);
6141 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
6142 NewElts.push_back(Elt: Undef);
6143
6144 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
6145 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6146 Results.push_back(Elt: CvtVec);
6147 break;
6148 }
6149 case ISD::ATOMIC_SWAP:
6150 case ISD::ATOMIC_STORE: {
6151 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6152 SDLoc SL(Node);
6153 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
6154 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6155 "unexpected promotion type");
6156 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6157 "unexpected atomic_swap with illegal type");
6158
6159 SDValue Op0 = AM->getBasePtr();
6160 SDValue Op1 = CastVal;
6161
6162 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
6163 // but really it should merge with ISD::STORE.
6164 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6165 std::swap(a&: Op0, b&: Op1);
6166
6167 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
6168 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
6169
6170 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6171 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6172 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6173 } else
6174 Results.push_back(Elt: NewAtomic);
6175 break;
6176 }
6177 case ISD::ATOMIC_LOAD: {
6178 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6179 SDLoc SL(Node);
6180 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6181 "unexpected promotion type");
6182 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6183 "unexpected atomic_load with illegal type");
6184
6185 SDValue NewAtomic =
6186 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
6187 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
6188 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6189 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6190 break;
6191 }
6192 case ISD::SPLAT_VECTOR: {
6193 SDValue Scalar = Node->getOperand(Num: 0);
6194 MVT ScalarType = Scalar.getSimpleValueType();
6195 MVT NewScalarType = NVT.getVectorElementType();
6196 if (ScalarType.isInteger()) {
6197 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6198 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6199 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
6200 break;
6201 }
6202 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6203 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6204 Results.push_back(
6205 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6206 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6207 break;
6208 }
6209 case ISD::VECREDUCE_FMAX:
6210 case ISD::VECREDUCE_FMIN:
6211 case ISD::VECREDUCE_FMAXIMUM:
6212 case ISD::VECREDUCE_FMINIMUM:
6213 case ISD::VP_REDUCE_FMAX:
6214 case ISD::VP_REDUCE_FMIN:
6215 case ISD::VP_REDUCE_FMAXIMUM:
6216 case ISD::VP_REDUCE_FMINIMUM:
6217 Results.push_back(Elt: PromoteReduction(Node));
6218 break;
6219 }
6220
6221 // Replace the original node with the legalized result.
6222 if (!Results.empty()) {
6223 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6224 ReplaceNode(Old: Node, New: Results.data());
6225 } else
6226 LLVM_DEBUG(dbgs() << "Could not promote node\n");
6227}
6228
6229/// This is the entry point for the file.
6230void SelectionDAG::Legalize() {
6231 AssignTopologicalOrder();
6232
6233 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6234 // Use a delete listener to remove nodes which were deleted during
6235 // legalization from LegalizeNodes. This is needed to handle the situation
6236 // where a new node is allocated by the object pool to the same address of a
6237 // previously deleted node.
6238 DAGNodeDeletedListener DeleteListener(
6239 *this,
6240 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
6241
6242 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6243
6244 // Visit all the nodes. We start in topological order, so that we see
6245 // nodes with their original operands intact. Legalization can produce
6246 // new nodes which may themselves need to be legalized. Iterate until all
6247 // nodes have been legalized.
6248 while (true) {
6249 bool AnyLegalized = false;
6250 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6251 --NI;
6252
6253 SDNode *N = &*NI;
6254 if (N->use_empty() && N != getRoot().getNode()) {
6255 ++NI;
6256 DeleteNode(N);
6257 continue;
6258 }
6259
6260 if (LegalizedNodes.insert(Ptr: N).second) {
6261 AnyLegalized = true;
6262 Legalizer.LegalizeOp(Node: N);
6263
6264 if (N->use_empty() && N != getRoot().getNode()) {
6265 ++NI;
6266 DeleteNode(N);
6267 }
6268 }
6269 }
6270 if (!AnyLegalized)
6271 break;
6272
6273 }
6274
6275 // Remove dead nodes now.
6276 RemoveDeadNodes();
6277}
6278
6279bool SelectionDAG::LegalizeOp(SDNode *N,
6280 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6281 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6282 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6283
6284 // Directly insert the node in question, and legalize it. This will recurse
6285 // as needed through operands.
6286 LegalizedNodes.insert(Ptr: N);
6287 Legalizer.LegalizeOp(Node: N);
6288
6289 return LegalizedNodes.count(Ptr: N);
6290}
6291