1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SelectionDAGNodes.h"
32#include "llvm/CodeGen/TargetFrameLowering.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/CodeGenTypes/MachineValueType.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Compiler.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/MathExtras.h"
49#include "llvm/Support/raw_ostream.h"
50#include "llvm/Target/TargetMachine.h"
51#include "llvm/Target/TargetOptions.h"
52#include <cassert>
53#include <cstdint>
54#include <tuple>
55#include <utility>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "legalizedag"
60
61namespace {
62
63/// Keeps track of state when getting the sign of a floating-point value as an
64/// integer.
65struct FloatSignAsInt {
66 EVT FloatVT;
67 SDValue Chain;
68 SDValue FloatPtr;
69 SDValue IntPtr;
70 MachinePointerInfo IntPointerInfo;
71 MachinePointerInfo FloatPointerInfo;
72 SDValue IntValue;
73 APInt SignMask;
74 uint8_t SignBit;
75};
76
77//===----------------------------------------------------------------------===//
78/// This takes an arbitrary SelectionDAG as input and
79/// hacks on it until the target machine can handle it. This involves
80/// eliminating value sizes the machine cannot handle (promoting small sizes to
81/// large sizes or splitting up large values into small values) as well as
82/// eliminating operations the machine cannot handle.
83///
84/// This code also does a small amount of optimization and recognition of idioms
85/// as part of its processing. For example, if a target does not support a
86/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
87/// will attempt merge setcc and brc instructions into brcc's.
88class SelectionDAGLegalize {
89 const TargetMachine &TM;
90 const TargetLowering &TLI;
91 SelectionDAG &DAG;
92
93 /// The set of nodes which have already been legalized. We hold a
94 /// reference to it in order to update as necessary on node deletion.
95 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
96
97 /// A set of all the nodes updated during legalization.
98 SmallSetVector<SDNode *, 16> *UpdatedNodes;
99
100 EVT getSetCCResultType(EVT VT) const {
101 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
102 }
103
104 // Libcall insertion helpers.
105
106public:
107 SelectionDAGLegalize(SelectionDAG &DAG,
108 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
109 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
110 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
112
113 /// Legalizes the given operation.
114 void LegalizeOp(SDNode *Node);
115
116private:
117 SDValue OptimizeFloatStore(StoreSDNode *ST);
118
119 void LegalizeLoadOps(SDNode *Node);
120 void LegalizeStoreOps(SDNode *Node);
121
122 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
123
124 /// Return a vector shuffle operation which
125 /// performs the same shuffe in terms of order or result bytes, but on a type
126 /// whose vector element type is narrower than the original shuffle type.
127 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
128 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
129 SDValue N1, SDValue N2,
130 ArrayRef<int> Mask) const;
131
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 TargetLowering::ArgListTy &&Args,
134 bool IsSigned, EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
136
137 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 SmallVectorImpl<SDValue> &Results);
139 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
140 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
141 RTLIB::Libcall Call_F128,
142 RTLIB::Libcall Call_PPCF128,
143 SmallVectorImpl<SDValue> &Results);
144
145 void
146 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
147 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
148 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
149 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
150 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
151 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
152 SmallVectorImpl<SDValue> &Results);
153
154 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
155 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
156 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
157 void ExpandArgFPLibCall(SDNode *Node,
158 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
159 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
160 RTLIB::Libcall Call_PPCF128,
161 SmallVectorImpl<SDValue> &Results);
162 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
163 RTLIB::Libcall CallI64,
164 RTLIB::Libcall CallI128);
165 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
166
167 SDValue ExpandSincosStretLibCall(SDNode *Node) const;
168
169 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
170 const SDLoc &dl);
171 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
172 const SDLoc &dl, SDValue ChainIn);
173 SDValue ExpandBUILD_VECTOR(SDNode *Node);
174 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
175 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
176 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
177 SmallVectorImpl<SDValue> &Results);
178 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
179 SDValue Value) const;
180 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
181 SDValue NewIntValue) const;
182 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
183 SDValue ExpandFABS(SDNode *Node) const;
184 SDValue ExpandFNEG(SDNode *Node) const;
185 SDValue expandLdexp(SDNode *Node) const;
186 SDValue expandFrexp(SDNode *Node) const;
187 SDValue expandModf(SDNode *Node) const;
188
189 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
190 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
191 SmallVectorImpl<SDValue> &Results);
192 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
193 SmallVectorImpl<SDValue> &Results);
194 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
195
196 /// Implements vector reduce operation promotion.
197 ///
198 /// All vector operands are promoted to a vector type with larger element
199 /// type, and the start value is promoted to a larger scalar type. Then the
200 /// result is truncated back to the original scalar type.
201 SDValue PromoteReduction(SDNode *Node);
202
203 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
204
205 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
206 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
207 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
208 SDValue ExpandConcatVectors(SDNode *Node);
209
210 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
211 SDValue ExpandConstant(ConstantSDNode *CP);
212
213 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
214 bool ExpandNode(SDNode *Node);
215 void ConvertNodeToLibcall(SDNode *Node);
216 void PromoteNode(SDNode *Node);
217
218public:
219 // Node replacement helpers
220
221 void ReplacedNode(SDNode *N) {
222 LegalizedNodes.erase(Ptr: N);
223 if (UpdatedNodes)
224 UpdatedNodes->insert(X: N);
225 }
226
227 void ReplaceNode(SDNode *Old, SDNode *New) {
228 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
229 dbgs() << " with: "; New->dump(&DAG));
230
231 assert(Old->getNumValues() == New->getNumValues() &&
232 "Replacing one node with another that produces a different number "
233 "of values!");
234 DAG.ReplaceAllUsesWith(From: Old, To: New);
235 if (UpdatedNodes)
236 UpdatedNodes->insert(X: New);
237 ReplacedNode(N: Old);
238 }
239
240 void ReplaceNode(SDValue Old, SDValue New) {
241 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
242 dbgs() << " with: "; New->dump(&DAG));
243
244 DAG.ReplaceAllUsesWith(From: Old, To: New);
245 if (UpdatedNodes)
246 UpdatedNodes->insert(X: New.getNode());
247 ReplacedNode(N: Old.getNode());
248 }
249
250 void ReplaceNode(SDNode *Old, const SDValue *New) {
251 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
252
253 DAG.ReplaceAllUsesWith(From: Old, To: New);
254 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
255 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
256 New[i]->dump(&DAG));
257 if (UpdatedNodes)
258 UpdatedNodes->insert(X: New[i].getNode());
259 }
260 ReplacedNode(N: Old);
261 }
262
263 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
264 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
265 dbgs() << " with: "; New->dump(&DAG));
266
267 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
268 if (UpdatedNodes)
269 UpdatedNodes->insert(X: New.getNode());
270 ReplacedNode(N: Old.getNode());
271 }
272};
273
274} // end anonymous namespace
275
276// Helper function that generates an MMO that considers the alignment of the
277// stack, and the size of the stack object
278static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
279 MachineFunction &MF,
280 bool isObjectScalable) {
281 auto &MFI = MF.getFrameInfo();
282 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
283 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
284 LocationSize ObjectSize = isObjectScalable
285 ? LocationSize::beforeOrAfterPointer()
286 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
287 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
288 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
289}
290
291/// Return a vector shuffle operation which
292/// performs the same shuffle in terms of order or result bytes, but on a type
293/// whose vector element type is narrower than the original shuffle type.
294/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
295SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
296 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
297 ArrayRef<int> Mask) const {
298 unsigned NumMaskElts = VT.getVectorNumElements();
299 unsigned NumDestElts = NVT.getVectorNumElements();
300 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
301
302 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
303
304 if (NumEltsGrowth == 1)
305 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
306
307 SmallVector<int, 8> NewMask;
308 for (unsigned i = 0; i != NumMaskElts; ++i) {
309 int Idx = Mask[i];
310 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
311 if (Idx < 0)
312 NewMask.push_back(Elt: -1);
313 else
314 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
315 }
316 }
317 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
318 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
319 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
320}
321
322/// Expands the ConstantFP node to an integer constant or
323/// a load from the constant pool.
324SDValue
325SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
326 bool Extend = false;
327 SDLoc dl(CFP);
328
329 // If a FP immediate is precise when represented as a float and if the
330 // target can do an extending load from float to double, we put it into
331 // the constant pool as a float, even if it's is statically typed as a
332 // double. This shrinks FP constants and canonicalizes them for targets where
333 // an FP extending load is the same cost as a normal load (such as on the x87
334 // fp stack or PPC FP unit).
335 EVT VT = CFP->getValueType(ResNo: 0);
336 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
337 if (!UseCP) {
338 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
339 return DAG.getConstant(Val: LLVMC->getValueAPF().bitcastToAPInt(), DL: dl,
340 VT: (VT == MVT::f64) ? MVT::i64 : MVT::i32);
341 }
342
343 APFloat APF = CFP->getValueAPF();
344 EVT OrigVT = VT;
345 EVT SVT = VT;
346
347 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
348 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
349 if (!APF.isSignaling()) {
350 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
351 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
352 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
353 // Only do this if the target has a native EXTLOAD instruction from
354 // smaller type.
355 TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: OrigVT, MemVT: SVT) &&
356 TLI.ShouldShrinkFPConstant(OrigVT)) {
357 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
358 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
359 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
360 VT = SVT;
361 Extend = true;
362 }
363 }
364 }
365
366 SDValue CPIdx =
367 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
368 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
369 if (Extend) {
370 SDValue Result = DAG.getExtLoad(
371 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
372 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
373 Alignment);
374 return Result;
375 }
376 SDValue Result = DAG.getLoad(
377 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
378 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
379 return Result;
380}
381
382/// Expands the Constant node to a load from the constant pool.
383SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
384 SDLoc dl(CP);
385 EVT VT = CP->getValueType(ResNo: 0);
386 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
387 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
388 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
389 SDValue Result = DAG.getLoad(
390 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
391 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
392 return Result;
393}
394
395SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
396 SDValue Vec = Op.getOperand(i: 0);
397 SDValue Val = Op.getOperand(i: 1);
398 SDValue Idx = Op.getOperand(i: 2);
399 SDLoc dl(Op);
400
401 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
402 // SCALAR_TO_VECTOR requires that the type of the value being inserted
403 // match the element type of the vector being created, except for
404 // integers in which case the inserted value can be over width.
405 EVT EltVT = Vec.getValueType().getVectorElementType();
406 if (Val.getValueType() == EltVT ||
407 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
408 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
409 VT: Vec.getValueType(), Operand: Val);
410
411 unsigned NumElts = Vec.getValueType().getVectorNumElements();
412 // We generate a shuffle of InVec and ScVec, so the shuffle mask
413 // should be 0,1,2,3,4,5... with the appropriate element replaced with
414 // elt 0 of the RHS.
415 SmallVector<int, 8> ShufOps;
416 for (unsigned i = 0; i != NumElts; ++i)
417 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
418
419 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
420 }
421 }
422 return ExpandInsertToVectorThroughStack(Op);
423}
424
425SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
426 if (!ISD::isNormalStore(N: ST))
427 return SDValue();
428
429 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
430 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
431 // FIXME: move this to the DAG Combiner! Note that we can't regress due
432 // to phase ordering between legalized code and the dag combiner. This
433 // probably means that we need to integrate dag combiner and legalizer
434 // together.
435 // We generally can't do this one for long doubles.
436 SDValue Chain = ST->getChain();
437 SDValue Ptr = ST->getBasePtr();
438 SDValue Value = ST->getValue();
439 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
440 AAMDNodes AAInfo = ST->getAAInfo();
441 SDLoc dl(ST);
442
443 // Don't optimise TargetConstantFP
444 if (Value.getOpcode() == ISD::TargetConstantFP)
445 return SDValue();
446
447 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
448 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
449 TLI.isTypeLegal(VT: MVT::i32)) {
450 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
451 bitcastToAPInt().zextOrTrunc(width: 32),
452 DL: SDLoc(CFP), VT: MVT::i32);
453 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
454 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
455 }
456
457 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
458 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
459 // If this target supports 64-bit registers, do a single 64-bit store.
460 if (TLI.isTypeLegal(VT: MVT::i64)) {
461 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
462 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
463 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
464 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
465 }
466
467 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
468 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
469 // stores. If the target supports neither 32- nor 64-bits, this
470 // xform is certainly not worth it.
471 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
472 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
473 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
474 if (DAG.getDataLayout().isBigEndian())
475 std::swap(a&: Lo, b&: Hi);
476
477 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
478 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
479 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
480 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
481 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
482 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
483
484 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
485 }
486 }
487 }
488 return SDValue();
489}
490
491void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
492 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
493 SDValue Chain = ST->getChain();
494 SDValue Ptr = ST->getBasePtr();
495 SDLoc dl(Node);
496
497 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
498 AAMDNodes AAInfo = ST->getAAInfo();
499
500 if (!ST->isTruncatingStore()) {
501 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
502 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
503 ReplaceNode(Old: ST, New: OptStore);
504 return;
505 }
506
507 SDValue Value = ST->getValue();
508 MVT VT = Value.getSimpleValueType();
509 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
510 default: llvm_unreachable("This action is not supported yet!");
511 case TargetLowering::Legal: {
512 // If this is an unaligned store and the target doesn't support it,
513 // expand it.
514 EVT MemVT = ST->getMemoryVT();
515 const DataLayout &DL = DAG.getDataLayout();
516 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
517 MMO: *ST->getMemOperand())) {
518 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
519 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
520 ReplaceNode(Old: SDValue(ST, 0), New: Result);
521 } else
522 LLVM_DEBUG(dbgs() << "Legal store\n");
523 break;
524 }
525 case TargetLowering::Custom: {
526 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
527 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
528 if (Res && Res != SDValue(Node, 0))
529 ReplaceNode(Old: SDValue(Node, 0), New: Res);
530 return;
531 }
532 case TargetLowering::Promote: {
533 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
534 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
535 "Can only promote stores to same size type");
536 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
537 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
538 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
539 ReplaceNode(Old: SDValue(Node, 0), New: Result);
540 break;
541 }
542 }
543 return;
544 }
545
546 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
547 SDValue Value = ST->getValue();
548 EVT StVT = ST->getMemoryVT();
549 TypeSize StWidth = StVT.getSizeInBits();
550 TypeSize StSize = StVT.getStoreSizeInBits();
551 auto &DL = DAG.getDataLayout();
552
553 if (StWidth != StSize) {
554 // Promote to a byte-sized store with upper bits zero if not
555 // storing an integral number of bytes. For example, promote
556 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
557 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
558 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
559 SDValue Result =
560 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
561 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
562 ReplaceNode(Old: SDValue(Node, 0), New: Result);
563 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
564 // If not storing a power-of-2 number of bits, expand as two stores.
565 assert(!StVT.isVector() && "Unsupported truncstore!");
566 unsigned StWidthBits = StWidth.getFixedValue();
567 unsigned LogStWidth = Log2_32(Value: StWidthBits);
568 assert(LogStWidth < 32);
569 unsigned RoundWidth = 1 << LogStWidth;
570 assert(RoundWidth < StWidthBits);
571 unsigned ExtraWidth = StWidthBits - RoundWidth;
572 assert(ExtraWidth < RoundWidth);
573 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
574 "Store size not an integral number of bytes!");
575 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
576 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
577 SDValue Lo, Hi;
578 unsigned IncrementSize;
579
580 if (DL.isLittleEndian()) {
581 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
582 // Store the bottom RoundWidth bits.
583 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
584 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
585
586 // Store the remaining ExtraWidth bits.
587 IncrementSize = RoundWidth / 8;
588 Ptr =
589 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
590 Hi = DAG.getNode(
591 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
592 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Value.getValueType(), DL: dl));
593 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
594 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
595 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
596 } else {
597 // Big endian - avoid unaligned stores.
598 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
599 // Store the top RoundWidth bits.
600 Hi = DAG.getNode(
601 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
602 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Value.getValueType(), DL: dl));
603 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
604 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
605
606 // Store the remaining ExtraWidth bits.
607 IncrementSize = RoundWidth / 8;
608 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
609 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
610 VT: Ptr.getValueType()));
611 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
612 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
613 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
614 }
615
616 // The order of the stores doesn't matter.
617 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
618 ReplaceNode(Old: SDValue(Node, 0), New: Result);
619 } else {
620 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT)) {
621 default: llvm_unreachable("This action is not supported yet!");
622 case TargetLowering::Legal: {
623 EVT MemVT = ST->getMemoryVT();
624 // If this is an unaligned store and the target doesn't support it,
625 // expand it.
626 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
627 MMO: *ST->getMemOperand())) {
628 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
629 ReplaceNode(Old: SDValue(ST, 0), New: Result);
630 }
631 break;
632 }
633 case TargetLowering::Custom: {
634 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
635 if (Res && Res != SDValue(Node, 0))
636 ReplaceNode(Old: SDValue(Node, 0), New: Res);
637 return;
638 }
639 case TargetLowering::Expand:
640 assert(!StVT.isVector() &&
641 "Vector Stores are handled in LegalizeVectorOps");
642
643 SDValue Result;
644
645 // TRUNCSTORE:i16 i32 -> STORE i16
646 if (TLI.isTypeLegal(VT: StVT)) {
647 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
648 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
649 Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
650 } else {
651 // The in-memory type isn't legal. Truncate to the type it would promote
652 // to, and then do a truncstore.
653 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
654 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
655 Operand: Value);
656 Result = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
657 SVT: StVT, Alignment: ST->getBaseAlign(), MMOFlags, AAInfo);
658 }
659
660 ReplaceNode(Old: SDValue(Node, 0), New: Result);
661 break;
662 }
663 }
664}
665
666void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
667 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
668 SDValue Chain = LD->getChain(); // The chain.
669 SDValue Ptr = LD->getBasePtr(); // The base pointer.
670 SDValue Value; // The value returned by the load op.
671 SDLoc dl(Node);
672
673 ISD::LoadExtType ExtType = LD->getExtensionType();
674 if (ExtType == ISD::NON_EXTLOAD) {
675 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
676 MVT VT = Node->getSimpleValueType(ResNo: 0);
677 SDValue RVal = SDValue(Node, 0);
678 SDValue RChain = SDValue(Node, 1);
679
680 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
681 default: llvm_unreachable("This action is not supported yet!");
682 case TargetLowering::Legal: {
683 EVT MemVT = LD->getMemoryVT();
684 const DataLayout &DL = DAG.getDataLayout();
685 // If this is an unaligned load and the target doesn't support it,
686 // expand it.
687 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
688 MMO: *LD->getMemOperand())) {
689 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
690 }
691 break;
692 }
693 case TargetLowering::Custom:
694 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
695 RVal = Res;
696 RChain = Res.getValue(R: 1);
697 }
698 break;
699
700 case TargetLowering::Promote: {
701 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
702 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
703 "Can only promote loads to same size type");
704
705 // If the range metadata type does not match the legalized memory
706 // operation type, remove the range metadata.
707 if (const MDNode *MD = LD->getRanges()) {
708 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
709 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
710 !NVT.isInteger())
711 LD->getMemOperand()->clearRanges();
712 }
713 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
714 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
715 RChain = Res.getValue(R: 1);
716 break;
717 }
718 }
719 if (RChain.getNode() != Node) {
720 assert(RVal.getNode() != Node && "Load must be completely replaced");
721 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
722 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
723 if (UpdatedNodes) {
724 UpdatedNodes->insert(X: RVal.getNode());
725 UpdatedNodes->insert(X: RChain.getNode());
726 }
727 ReplacedNode(N: Node);
728 }
729 return;
730 }
731
732 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
733 EVT SrcVT = LD->getMemoryVT();
734 TypeSize SrcWidth = SrcVT.getSizeInBits();
735 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
736 AAMDNodes AAInfo = LD->getAAInfo();
737
738 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
739 // Some targets pretend to have an i1 loading operation, and actually
740 // load an i8. This trick is correct for ZEXTLOAD because the top 7
741 // bits are guaranteed to be zero; it helps the optimizers understand
742 // that these bits are zero. It is also useful for EXTLOAD, since it
743 // tells the optimizers that those bits are undefined. It would be
744 // nice to have an effective generic way of getting these benefits...
745 // Until such a way is found, don't insist on promoting i1 here.
746 (SrcVT != MVT::i1 ||
747 TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1) ==
748 TargetLowering::Promote)) {
749 // Promote to a byte-sized load if not loading an integral number of
750 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
751 unsigned NewWidth = SrcVT.getStoreSizeInBits();
752 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
753 SDValue Ch;
754
755 // The extra bits are guaranteed to be zero, since we stored them that
756 // way. A zext load from NVT thus automatically gives zext from SrcVT.
757
758 ISD::LoadExtType NewExtType =
759 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
760
761 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
762 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
763 Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
764
765 Ch = Result.getValue(R: 1); // The chain.
766
767 if (ExtType == ISD::SEXTLOAD)
768 // Having the top bits zero doesn't help when sign extending.
769 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
770 VT: Result.getValueType(),
771 N1: Result, N2: DAG.getValueType(SrcVT));
772 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
773 // All the top bits are guaranteed to be zero - inform the optimizers.
774 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
775 VT: Result.getValueType(), N1: Result,
776 N2: DAG.getValueType(SrcVT));
777
778 Value = Result;
779 Chain = Ch;
780 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
781 // If not loading a power-of-2 number of bits, expand as two loads.
782 assert(!SrcVT.isVector() && "Unsupported extload!");
783 unsigned SrcWidthBits = SrcWidth.getFixedValue();
784 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
785 assert(LogSrcWidth < 32);
786 unsigned RoundWidth = 1 << LogSrcWidth;
787 assert(RoundWidth < SrcWidthBits);
788 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
789 assert(ExtraWidth < RoundWidth);
790 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
791 "Load size not an integral number of bytes!");
792 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
793 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
794 SDValue Lo, Hi, Ch;
795 unsigned IncrementSize;
796 auto &DL = DAG.getDataLayout();
797
798 if (DL.isLittleEndian()) {
799 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
800 // Load the bottom RoundWidth bits.
801 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
802 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
803 MMOFlags, AAInfo);
804
805 // Load the remaining ExtraWidth bits.
806 IncrementSize = RoundWidth / 8;
807 Ptr =
808 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
809 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
810 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
811 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
812
813 // Build a factor node to remember that this load is independent of
814 // the other one.
815 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
816 N2: Hi.getValue(R: 1));
817
818 // Move the top bits to the right place.
819 Hi = DAG.getNode(
820 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
821 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Hi.getValueType(), DL: dl));
822
823 // Join the hi and lo parts.
824 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
825 } else {
826 // Big endian - avoid unaligned loads.
827 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
828 // Load the top RoundWidth bits.
829 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
830 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
831 MMOFlags, AAInfo);
832
833 // Load the remaining ExtraWidth bits.
834 IncrementSize = RoundWidth / 8;
835 Ptr =
836 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
837 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
838 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
839 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, AAInfo);
840
841 // Build a factor node to remember that this load is independent of
842 // the other one.
843 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
844 N2: Hi.getValue(R: 1));
845
846 // Move the top bits to the right place.
847 Hi = DAG.getNode(
848 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
849 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Hi.getValueType(), DL: dl));
850
851 // Join the hi and lo parts.
852 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
853 }
854
855 Chain = Ch;
856 } else {
857 bool isCustom = false;
858 switch (TLI.getLoadExtAction(ExtType, ValVT: Node->getValueType(ResNo: 0),
859 MemVT: SrcVT.getSimpleVT())) {
860 default: llvm_unreachable("This action is not supported yet!");
861 case TargetLowering::Custom:
862 isCustom = true;
863 [[fallthrough]];
864 case TargetLowering::Legal:
865 Value = SDValue(Node, 0);
866 Chain = SDValue(Node, 1);
867
868 if (isCustom) {
869 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
870 Value = Res;
871 Chain = Res.getValue(R: 1);
872 }
873 } else {
874 // If this is an unaligned load and the target doesn't support it,
875 // expand it.
876 EVT MemVT = LD->getMemoryVT();
877 const DataLayout &DL = DAG.getDataLayout();
878 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
879 MMO: *LD->getMemOperand())) {
880 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
881 }
882 }
883 break;
884
885 case TargetLowering::Expand: {
886 EVT DestVT = Node->getValueType(ResNo: 0);
887 if (!TLI.isLoadExtLegal(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SrcVT)) {
888 // If the source type is not legal, see if there is a legal extload to
889 // an intermediate type that we can then extend further.
890 EVT LoadVT = TLI.getRegisterType(VT: SrcVT.getSimpleVT());
891 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
892 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
893 TLI.isLoadExtLegal(ExtType, ValVT: LoadVT, MemVT: SrcVT))) {
894 // If we are loading a legal type, this is a non-extload followed by a
895 // full extend.
896 ISD::LoadExtType MidExtType =
897 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
898
899 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
900 MemVT: SrcVT, MMO: LD->getMemOperand());
901 unsigned ExtendOp =
902 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
903 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
904 Chain = Load.getValue(R: 1);
905 break;
906 }
907
908 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
909 // normal undefined upper bits behavior to allow using an in-reg extend
910 // with the illegal FP type, so load as an integer and do the
911 // from-integer conversion.
912 EVT SVT = SrcVT.getScalarType();
913 if (SVT == MVT::f16 || SVT == MVT::bf16) {
914 EVT ISrcVT = SrcVT.changeTypeToInteger();
915 EVT IDestVT = DestVT.changeTypeToInteger();
916 EVT ILoadVT = TLI.getRegisterType(VT: IDestVT.getSimpleVT());
917
918 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
919 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
920 Value =
921 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
922 DL: dl, VT: DestVT, Operand: Result);
923 Chain = Result.getValue(R: 1);
924 break;
925 }
926 }
927
928 assert(!SrcVT.isVector() &&
929 "Vector Loads are handled in LegalizeVectorOps");
930
931 // FIXME: This does not work for vectors on most targets. Sign-
932 // and zero-extend operations are currently folded into extending
933 // loads, whether they are legal or not, and then we end up here
934 // without any support for legalizing them.
935 assert(ExtType != ISD::EXTLOAD &&
936 "EXTLOAD should always be supported!");
937 // Turn the unsupported load into an EXTLOAD followed by an
938 // explicit zero/sign extend inreg.
939 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
940 VT: Node->getValueType(ResNo: 0),
941 Chain, Ptr, MemVT: SrcVT,
942 MMO: LD->getMemOperand());
943 SDValue ValRes;
944 if (ExtType == ISD::SEXTLOAD)
945 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
946 VT: Result.getValueType(),
947 N1: Result, N2: DAG.getValueType(SrcVT));
948 else
949 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
950 Value = ValRes;
951 Chain = Result.getValue(R: 1);
952 break;
953 }
954 }
955 }
956
957 // Since loads produce two values, make sure to remember that we legalized
958 // both of them.
959 if (Chain.getNode() != Node) {
960 assert(Value.getNode() != Node && "Load must be completely replaced");
961 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
962 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
963 if (UpdatedNodes) {
964 UpdatedNodes->insert(X: Value.getNode());
965 UpdatedNodes->insert(X: Chain.getNode());
966 }
967 ReplacedNode(N: Node);
968 }
969}
970
971/// Return a legal replacement for the given operation, with all legal operands.
972void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
973 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
974
975 // Allow illegal target nodes and illegal registers.
976 if (Node->getOpcode() == ISD::TargetConstant ||
977 Node->getOpcode() == ISD::Register)
978 return;
979
980#ifndef NDEBUG
981 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
982 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
983 TargetLowering::TypeLegal &&
984 "Unexpected illegal type!");
985
986 for (const SDValue &Op : Node->op_values())
987 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
988 TargetLowering::TypeLegal ||
989 Op.getOpcode() == ISD::TargetConstant ||
990 Op.getOpcode() == ISD::Register) &&
991 "Unexpected illegal type!");
992#endif
993
994 // Figure out the correct action; the way to query this varies by opcode
995 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
996 bool SimpleFinishLegalizing = true;
997 switch (Node->getOpcode()) {
998 case ISD::POISON: {
999 // TODO: Currently, POISON is being lowered to UNDEF here. However, there is
1000 // an open concern that this transformation may not be ideal, as targets
1001 // should ideally handle POISON directly. Changing this behavior would
1002 // require adding support for POISON in TableGen, which is a large change.
1003 // Additionally, many existing test cases rely on the current behavior
1004 // (e.g., llvm/test/CodeGen/PowerPC/vec_shuffle.ll). A broader discussion
1005 // and incremental changes might be needed to properly support POISON
1006 // without breaking existing targets and tests.
1007 SDValue UndefNode = DAG.getUNDEF(VT: Node->getValueType(ResNo: 0));
1008 ReplaceNode(Old: Node, New: UndefNode.getNode());
1009 return;
1010 }
1011 case ISD::INTRINSIC_W_CHAIN:
1012 case ISD::INTRINSIC_WO_CHAIN:
1013 case ISD::INTRINSIC_VOID:
1014 case ISD::STACKSAVE:
1015 case ISD::STACKADDRESS:
1016 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1017 break;
1018 case ISD::GET_DYNAMIC_AREA_OFFSET:
1019 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1020 VT: Node->getValueType(ResNo: 0));
1021 break;
1022 case ISD::VAARG:
1023 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1024 VT: Node->getValueType(ResNo: 0));
1025 if (Action != TargetLowering::Promote)
1026 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1027 break;
1028 case ISD::SET_FPENV:
1029 case ISD::SET_FPMODE:
1030 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1031 VT: Node->getOperand(Num: 1).getValueType());
1032 break;
1033 case ISD::FP_TO_FP16:
1034 case ISD::FP_TO_BF16:
1035 case ISD::SINT_TO_FP:
1036 case ISD::UINT_TO_FP:
1037 case ISD::EXTRACT_VECTOR_ELT:
1038 case ISD::LROUND:
1039 case ISD::LLROUND:
1040 case ISD::LRINT:
1041 case ISD::LLRINT:
1042 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1043 VT: Node->getOperand(Num: 0).getValueType());
1044 break;
1045 case ISD::STRICT_FP_TO_FP16:
1046 case ISD::STRICT_FP_TO_BF16:
1047 case ISD::STRICT_SINT_TO_FP:
1048 case ISD::STRICT_UINT_TO_FP:
1049 case ISD::STRICT_LRINT:
1050 case ISD::STRICT_LLRINT:
1051 case ISD::STRICT_LROUND:
1052 case ISD::STRICT_LLROUND:
1053 // These pseudo-ops are the same as the other STRICT_ ops except
1054 // they are registered with setOperationAction() using the input type
1055 // instead of the output type.
1056 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1057 VT: Node->getOperand(Num: 1).getValueType());
1058 break;
1059 case ISD::SIGN_EXTEND_INREG: {
1060 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1061 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1062 break;
1063 }
1064 case ISD::ATOMIC_STORE:
1065 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1066 VT: Node->getOperand(Num: 1).getValueType());
1067 break;
1068 case ISD::SELECT_CC:
1069 case ISD::STRICT_FSETCC:
1070 case ISD::STRICT_FSETCCS:
1071 case ISD::SETCC:
1072 case ISD::SETCCCARRY:
1073 case ISD::VP_SETCC:
1074 case ISD::BR_CC: {
1075 unsigned Opc = Node->getOpcode();
1076 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1077 : Opc == ISD::STRICT_FSETCC ? 3
1078 : Opc == ISD::STRICT_FSETCCS ? 3
1079 : Opc == ISD::SETCCCARRY ? 3
1080 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1081 : 1;
1082 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1083 : Opc == ISD::STRICT_FSETCC ? 1
1084 : Opc == ISD::STRICT_FSETCCS ? 1
1085 : 0;
1086 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1087 ISD::CondCode CCCode =
1088 cast<CondCodeSDNode>(Val: Node->getOperand(Num: CCOperand))->get();
1089 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1090 if (Action == TargetLowering::Legal) {
1091 if (Node->getOpcode() == ISD::SELECT_CC)
1092 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1093 VT: Node->getValueType(ResNo: 0));
1094 else
1095 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1096 }
1097 break;
1098 }
1099 case ISD::LOAD:
1100 case ISD::STORE:
1101 // FIXME: Model these properly. LOAD and STORE are complicated, and
1102 // STORE expects the unlegalized operand in some cases.
1103 SimpleFinishLegalizing = false;
1104 break;
1105 case ISD::CALLSEQ_START:
1106 case ISD::CALLSEQ_END:
1107 // FIXME: This shouldn't be necessary. These nodes have special properties
1108 // dealing with the recursive nature of legalization. Removing this
1109 // special case should be done as part of making LegalizeDAG non-recursive.
1110 SimpleFinishLegalizing = false;
1111 break;
1112 case ISD::EXTRACT_ELEMENT:
1113 case ISD::GET_ROUNDING:
1114 case ISD::MERGE_VALUES:
1115 case ISD::EH_RETURN:
1116 case ISD::FRAME_TO_ARGS_OFFSET:
1117 case ISD::EH_DWARF_CFA:
1118 case ISD::EH_SJLJ_SETJMP:
1119 case ISD::EH_SJLJ_LONGJMP:
1120 case ISD::EH_SJLJ_SETUP_DISPATCH:
1121 // These operations lie about being legal: when they claim to be legal,
1122 // they should actually be expanded.
1123 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1124 if (Action == TargetLowering::Legal)
1125 Action = TargetLowering::Expand;
1126 break;
1127 case ISD::INIT_TRAMPOLINE:
1128 case ISD::ADJUST_TRAMPOLINE:
1129 case ISD::FRAMEADDR:
1130 case ISD::RETURNADDR:
1131 case ISD::ADDROFRETURNADDR:
1132 case ISD::SPONENTRY:
1133 // These operations lie about being legal: when they claim to be legal,
1134 // they should actually be custom-lowered.
1135 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1136 if (Action == TargetLowering::Legal)
1137 Action = TargetLowering::Custom;
1138 break;
1139 case ISD::CLEAR_CACHE:
1140 // This operation is typically going to be LibCall unless the target wants
1141 // something differrent.
1142 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1143 break;
1144 case ISD::READCYCLECOUNTER:
1145 case ISD::READSTEADYCOUNTER:
1146 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1147 // legalization might have expanded that to several smaller types.
1148 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1149 break;
1150 case ISD::READ_REGISTER:
1151 case ISD::WRITE_REGISTER:
1152 // Named register is legal in the DAG, but blocked by register name
1153 // selection if not implemented by target (to chose the correct register)
1154 // They'll be converted to Copy(To/From)Reg.
1155 Action = TargetLowering::Legal;
1156 break;
1157 case ISD::UBSANTRAP:
1158 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1159 if (Action == TargetLowering::Expand) {
1160 // replace ISD::UBSANTRAP with ISD::TRAP
1161 SDValue NewVal;
1162 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1163 N: Node->getOperand(Num: 0));
1164 ReplaceNode(Old: Node, New: NewVal.getNode());
1165 LegalizeOp(Node: NewVal.getNode());
1166 return;
1167 }
1168 break;
1169 case ISD::DEBUGTRAP:
1170 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1171 if (Action == TargetLowering::Expand) {
1172 // replace ISD::DEBUGTRAP with ISD::TRAP
1173 SDValue NewVal;
1174 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1175 N: Node->getOperand(Num: 0));
1176 ReplaceNode(Old: Node, New: NewVal.getNode());
1177 LegalizeOp(Node: NewVal.getNode());
1178 return;
1179 }
1180 break;
1181 case ISD::SADDSAT:
1182 case ISD::UADDSAT:
1183 case ISD::SSUBSAT:
1184 case ISD::USUBSAT:
1185 case ISD::SSHLSAT:
1186 case ISD::USHLSAT:
1187 case ISD::SCMP:
1188 case ISD::UCMP:
1189 case ISD::FP_TO_SINT_SAT:
1190 case ISD::FP_TO_UINT_SAT:
1191 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1192 break;
1193 case ISD::SMULFIX:
1194 case ISD::SMULFIXSAT:
1195 case ISD::UMULFIX:
1196 case ISD::UMULFIXSAT:
1197 case ISD::SDIVFIX:
1198 case ISD::SDIVFIXSAT:
1199 case ISD::UDIVFIX:
1200 case ISD::UDIVFIXSAT: {
1201 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1202 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1203 VT: Node->getValueType(ResNo: 0), Scale);
1204 break;
1205 }
1206 case ISD::MSCATTER:
1207 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1208 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1209 break;
1210 case ISD::MSTORE:
1211 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1212 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1213 break;
1214 case ISD::VP_SCATTER:
1215 Action = TLI.getOperationAction(
1216 Op: Node->getOpcode(),
1217 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1218 break;
1219 case ISD::VP_STORE:
1220 Action = TLI.getOperationAction(
1221 Op: Node->getOpcode(),
1222 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1223 break;
1224 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1225 Action = TLI.getOperationAction(
1226 Op: Node->getOpcode(),
1227 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1228 break;
1229 case ISD::VECREDUCE_FADD:
1230 case ISD::VECREDUCE_FMUL:
1231 case ISD::VECREDUCE_ADD:
1232 case ISD::VECREDUCE_MUL:
1233 case ISD::VECREDUCE_AND:
1234 case ISD::VECREDUCE_OR:
1235 case ISD::VECREDUCE_XOR:
1236 case ISD::VECREDUCE_SMAX:
1237 case ISD::VECREDUCE_SMIN:
1238 case ISD::VECREDUCE_UMAX:
1239 case ISD::VECREDUCE_UMIN:
1240 case ISD::VECREDUCE_FMAX:
1241 case ISD::VECREDUCE_FMIN:
1242 case ISD::VECREDUCE_FMAXIMUM:
1243 case ISD::VECREDUCE_FMINIMUM:
1244 case ISD::IS_FPCLASS:
1245 Action = TLI.getOperationAction(
1246 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1247 break;
1248 case ISD::VECREDUCE_SEQ_FADD:
1249 case ISD::VECREDUCE_SEQ_FMUL:
1250 case ISD::VP_REDUCE_FADD:
1251 case ISD::VP_REDUCE_FMUL:
1252 case ISD::VP_REDUCE_ADD:
1253 case ISD::VP_REDUCE_MUL:
1254 case ISD::VP_REDUCE_AND:
1255 case ISD::VP_REDUCE_OR:
1256 case ISD::VP_REDUCE_XOR:
1257 case ISD::VP_REDUCE_SMAX:
1258 case ISD::VP_REDUCE_SMIN:
1259 case ISD::VP_REDUCE_UMAX:
1260 case ISD::VP_REDUCE_UMIN:
1261 case ISD::VP_REDUCE_FMAX:
1262 case ISD::VP_REDUCE_FMIN:
1263 case ISD::VP_REDUCE_FMAXIMUM:
1264 case ISD::VP_REDUCE_FMINIMUM:
1265 case ISD::VP_REDUCE_SEQ_FADD:
1266 case ISD::VP_REDUCE_SEQ_FMUL:
1267 Action = TLI.getOperationAction(
1268 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1269 break;
1270 case ISD::VP_CTTZ_ELTS:
1271 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1272 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1273 VT: Node->getOperand(Num: 0).getValueType());
1274 break;
1275 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1276 Action = TLI.getOperationAction(
1277 Op: Node->getOpcode(),
1278 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1279 break;
1280 default:
1281 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1282 Action = TLI.getCustomOperationAction(Op&: *Node);
1283 } else {
1284 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1285 }
1286 break;
1287 }
1288
1289 if (SimpleFinishLegalizing) {
1290 SDNode *NewNode = Node;
1291 switch (Node->getOpcode()) {
1292 default: break;
1293 case ISD::SHL:
1294 case ISD::SRL:
1295 case ISD::SRA:
1296 case ISD::ROTL:
1297 case ISD::ROTR:
1298 case ISD::SSHLSAT:
1299 case ISD::USHLSAT: {
1300 // Legalizing shifts/rotates requires adjusting the shift amount
1301 // to the appropriate width.
1302 SDValue Op0 = Node->getOperand(Num: 0);
1303 SDValue Op1 = Node->getOperand(Num: 1);
1304 if (!Op1.getValueType().isVector()) {
1305 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1306 // The getShiftAmountOperand() may create a new operand node or
1307 // return the existing one. If new operand is created we need
1308 // to update the parent node.
1309 // Do not try to legalize SAO here! It will be automatically legalized
1310 // in the next round.
1311 if (SAO != Op1)
1312 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1313 }
1314 break;
1315 }
1316 case ISD::FSHL:
1317 case ISD::FSHR:
1318 case ISD::SRL_PARTS:
1319 case ISD::SRA_PARTS:
1320 case ISD::SHL_PARTS: {
1321 // Legalizing shifts/rotates requires adjusting the shift amount
1322 // to the appropriate width.
1323 SDValue Op0 = Node->getOperand(Num: 0);
1324 SDValue Op1 = Node->getOperand(Num: 1);
1325 SDValue Op2 = Node->getOperand(Num: 2);
1326 if (!Op2.getValueType().isVector()) {
1327 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1328 // The getShiftAmountOperand() may create a new operand node or
1329 // return the existing one. If new operand is created we need
1330 // to update the parent node.
1331 if (SAO != Op2)
1332 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1333 }
1334 break;
1335 }
1336 }
1337
1338 if (NewNode != Node) {
1339 ReplaceNode(Old: Node, New: NewNode);
1340 Node = NewNode;
1341 }
1342 switch (Action) {
1343 case TargetLowering::Legal:
1344 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1345 return;
1346 case TargetLowering::Custom:
1347 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1348 // FIXME: The handling for custom lowering with multiple results is
1349 // a complete mess.
1350 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1351 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1352 return;
1353
1354 if (Node->getNumValues() == 1) {
1355 // Verify the new types match the original. Glue is waived because
1356 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1357 assert((Res.getValueType() == Node->getValueType(0) ||
1358 Node->getValueType(0) == MVT::Glue) &&
1359 "Type mismatch for custom legalized operation");
1360 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1361 // We can just directly replace this node with the lowered value.
1362 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1363 return;
1364 }
1365
1366 SmallVector<SDValue, 8> ResultVals;
1367 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1368 // Verify the new types match the original. Glue is waived because
1369 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1370 assert((Res->getValueType(i) == Node->getValueType(i) ||
1371 Node->getValueType(i) == MVT::Glue) &&
1372 "Type mismatch for custom legalized operation");
1373 ResultVals.push_back(Elt: Res.getValue(R: i));
1374 }
1375 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1376 ReplaceNode(Old: Node, New: ResultVals.data());
1377 return;
1378 }
1379 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1380 [[fallthrough]];
1381 case TargetLowering::Expand:
1382 if (ExpandNode(Node))
1383 return;
1384 [[fallthrough]];
1385 case TargetLowering::LibCall:
1386 ConvertNodeToLibcall(Node);
1387 return;
1388 case TargetLowering::Promote:
1389 PromoteNode(Node);
1390 return;
1391 }
1392 }
1393
1394 switch (Node->getOpcode()) {
1395 default:
1396#ifndef NDEBUG
1397 dbgs() << "NODE: ";
1398 Node->dump( &DAG);
1399 dbgs() << "\n";
1400#endif
1401 llvm_unreachable("Do not know how to legalize this operator!");
1402
1403 case ISD::CALLSEQ_START:
1404 case ISD::CALLSEQ_END:
1405 break;
1406 case ISD::LOAD:
1407 return LegalizeLoadOps(Node);
1408 case ISD::STORE:
1409 return LegalizeStoreOps(Node);
1410 }
1411}
1412
1413SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1414 SDValue Vec = Op.getOperand(i: 0);
1415 SDValue Idx = Op.getOperand(i: 1);
1416 SDLoc dl(Op);
1417
1418 // Before we generate a new store to a temporary stack slot, see if there is
1419 // already one that we can use. There often is because when we scalarize
1420 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1421 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1422 // the vector. If all are expanded here, we don't want one store per vector
1423 // element.
1424
1425 // Caches for hasPredecessorHelper
1426 SmallPtrSet<const SDNode *, 32> Visited;
1427 SmallVector<const SDNode *, 16> Worklist;
1428 Visited.insert(Ptr: Op.getNode());
1429 Worklist.push_back(Elt: Idx.getNode());
1430 SDValue StackPtr, Ch;
1431 for (SDNode *User : Vec.getNode()->users()) {
1432 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1433 if (ST->isIndexed() || ST->isTruncatingStore() ||
1434 ST->getValue() != Vec)
1435 continue;
1436
1437 // Make sure that nothing else could have stored into the destination of
1438 // this store.
1439 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1440 continue;
1441
1442 // If the index is dependent on the store we will introduce a cycle when
1443 // creating the load (the load uses the index, and by replacing the chain
1444 // we will make the index dependent on the load). Also, the store might be
1445 // dependent on the extractelement and introduce a cycle when creating
1446 // the load.
1447 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1448 ST->hasPredecessor(N: Op.getNode()))
1449 continue;
1450
1451 StackPtr = ST->getBasePtr();
1452 Ch = SDValue(ST, 0);
1453 break;
1454 }
1455 }
1456
1457 EVT VecVT = Vec.getValueType();
1458
1459 if (!Ch.getNode()) {
1460 // Store the value to a temporary stack slot, then LOAD the returned part.
1461 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1462 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1463 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1464 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1465 }
1466
1467 SDValue NewLoad;
1468 Align ElementAlignment =
1469 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1470 b: DAG.getDataLayout().getPrefTypeAlign(
1471 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1472
1473 if (Op.getValueType().isVector()) {
1474 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1475 SubVecVT: Op.getValueType(), Index: Idx);
1476 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1477 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1478 } else {
1479 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1480 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1481 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1482 Alignment: ElementAlignment);
1483 }
1484
1485 // Replace the chain going out of the store, by the one out of the load.
1486 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1487
1488 // We introduced a cycle though, so update the loads operands, making sure
1489 // to use the original store's chain as an incoming chain.
1490 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1491 NewLoadOperands[0] = Ch;
1492 NewLoad =
1493 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1494 return NewLoad;
1495}
1496
1497SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1498 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1499
1500 SDValue Vec = Op.getOperand(i: 0);
1501 SDValue Part = Op.getOperand(i: 1);
1502 SDValue Idx = Op.getOperand(i: 2);
1503 SDLoc dl(Op);
1504
1505 // Store the value to a temporary stack slot, then LOAD the returned part.
1506 EVT VecVT = Vec.getValueType();
1507 EVT PartVT = Part.getValueType();
1508 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1509 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1510 MachinePointerInfo PtrInfo =
1511 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1512
1513 // First store the whole vector.
1514 Align BaseVecAlignment =
1515 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1516 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1517 Alignment: BaseVecAlignment);
1518
1519 // Freeze the index so we don't poison the clamping code we're about to emit.
1520 Idx = DAG.getFreeze(V: Idx);
1521
1522 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1523 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1524
1525 // Then store the inserted part.
1526 if (PartVT.isVector()) {
1527 SDValue SubStackPtr =
1528 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1529
1530 // Store the subvector.
1531 Ch = DAG.getStore(
1532 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1533 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1534 Alignment: PartAlignment);
1535 } else {
1536 SDValue SubStackPtr =
1537 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1538
1539 // Store the scalar value.
1540 Ch = DAG.getTruncStore(
1541 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1542 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1543 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1544 }
1545
1546 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1547 "ElementAlignment does not match!");
1548
1549 // Finally, load the updated vector.
1550 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1551 Alignment: BaseVecAlignment);
1552}
1553
1554SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1555 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1556 SDLoc DL(Node);
1557 SmallVector<SDValue, 16> Ops;
1558 unsigned NumOperands = Node->getNumOperands();
1559 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1560 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1561 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1562 EVT ElementValueType = TLI.getTypeToTransformTo(
1563 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1564 for (unsigned I = 0; I < NumOperands; ++I) {
1565 SDValue SubOp = Node->getOperand(Num: I);
1566 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1567 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1568 N1: SubOp,
1569 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1570 }
1571 }
1572 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1573}
1574
1575SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1576 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1577 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1578 "Unexpected opcode!");
1579
1580 // We can't handle this case efficiently. Allocate a sufficiently
1581 // aligned object on the stack, store each operand into it, then load
1582 // the result as a vector.
1583 // Create the stack frame object.
1584 EVT VT = Node->getValueType(ResNo: 0);
1585 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1586 : Node->getOperand(Num: 0).getValueType();
1587 SDLoc dl(Node);
1588 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1589 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1590 MachinePointerInfo PtrInfo =
1591 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1592
1593 // Emit a store of each element to the stack slot.
1594 SmallVector<SDValue, 8> Stores;
1595 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1596 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1597
1598 // If the destination vector element type of a BUILD_VECTOR is narrower than
1599 // the source element type, only store the bits necessary.
1600 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1601 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1602
1603 // Store (in the right endianness) the elements to memory.
1604 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1605 // Ignore undef elements.
1606 if (Node->getOperand(Num: i).isUndef()) continue;
1607
1608 unsigned Offset = TypeByteSize*i;
1609
1610 SDValue Idx =
1611 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1612
1613 if (Truncate)
1614 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1615 Val: Node->getOperand(Num: i), Ptr: Idx,
1616 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1617 else
1618 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1619 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1620 }
1621
1622 SDValue StoreChain;
1623 if (!Stores.empty()) // Not all undef elements?
1624 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1625 else
1626 StoreChain = DAG.getEntryNode();
1627
1628 // Result is a load from the stack slot.
1629 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1630}
1631
1632/// Bitcast a floating-point value to an integer value. Only bitcast the part
1633/// containing the sign bit if the target has no integer value capable of
1634/// holding all bits of the floating-point value.
1635void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1636 const SDLoc &DL,
1637 SDValue Value) const {
1638 EVT FloatVT = Value.getValueType();
1639 unsigned NumBits = FloatVT.getScalarSizeInBits();
1640 State.FloatVT = FloatVT;
1641 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1642 // Convert to an integer of the same size.
1643 if (TLI.isTypeLegal(VT: IVT)) {
1644 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1645 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1646 State.SignBit = NumBits - 1;
1647 return;
1648 }
1649
1650 auto &DataLayout = DAG.getDataLayout();
1651 // Store the float to memory, then load the sign part out as an integer.
1652 MVT LoadTy = TLI.getRegisterType(VT: MVT::i8);
1653 // First create a temporary that is aligned for both the load and store.
1654 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1655 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1656 // Then store the float to it.
1657 State.FloatPtr = StackPtr;
1658 MachineFunction &MF = DAG.getMachineFunction();
1659 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1660 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1661 PtrInfo: State.FloatPointerInfo);
1662
1663 SDValue IntPtr;
1664 if (DataLayout.isBigEndian()) {
1665 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1666 // Load out a legal integer with the same sign bit as the float.
1667 IntPtr = StackPtr;
1668 State.IntPointerInfo = State.FloatPointerInfo;
1669 } else {
1670 // Advance the pointer so that the loaded byte will contain the sign bit.
1671 unsigned ByteOffset = (NumBits / 8) - 1;
1672 IntPtr =
1673 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1674 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1675 Offset: ByteOffset);
1676 }
1677
1678 State.IntPtr = IntPtr;
1679 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1680 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1681 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1682 State.SignBit = 7;
1683}
1684
1685/// Replace the integer value produced by getSignAsIntValue() with a new value
1686/// and cast the result back to a floating-point type.
1687SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1688 const SDLoc &DL,
1689 SDValue NewIntValue) const {
1690 if (!State.Chain)
1691 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1692
1693 // Override the part containing the sign bit in the value stored on the stack.
1694 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1695 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1696 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1697 PtrInfo: State.FloatPointerInfo);
1698}
1699
1700SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1701 SDLoc DL(Node);
1702 SDValue Mag = Node->getOperand(Num: 0);
1703 SDValue Sign = Node->getOperand(Num: 1);
1704
1705 // Get sign bit into an integer value.
1706 FloatSignAsInt SignAsInt;
1707 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1708
1709 EVT IntVT = SignAsInt.IntValue.getValueType();
1710 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1711 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1712 N2: SignMask);
1713
1714 // If FABS is legal transform
1715 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1716 EVT FloatVT = Mag.getValueType();
1717 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1718 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1719 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1720 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1721 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1722 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1723 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1724 }
1725
1726 // Transform Mag value to integer, and clear the sign bit.
1727 FloatSignAsInt MagAsInt;
1728 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1729 EVT MagVT = MagAsInt.IntValue.getValueType();
1730 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1731 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1732 N2: ClearSignMask);
1733
1734 // Get the signbit at the right position for MagAsInt.
1735 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1736 EVT ShiftVT = IntVT;
1737 if (SignBit.getScalarValueSizeInBits() <
1738 ClearedSign.getScalarValueSizeInBits()) {
1739 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1740 ShiftVT = MagVT;
1741 }
1742 if (ShiftAmount > 0) {
1743 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1744 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1745 } else if (ShiftAmount < 0) {
1746 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1747 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1748 }
1749 if (SignBit.getScalarValueSizeInBits() >
1750 ClearedSign.getScalarValueSizeInBits()) {
1751 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1752 }
1753
1754 // Store the part with the modified sign and convert back to float.
1755 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1756 Flags: SDNodeFlags::Disjoint);
1757
1758 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1759}
1760
1761SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1762 // Get the sign bit as an integer.
1763 SDLoc DL(Node);
1764 FloatSignAsInt SignAsInt;
1765 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1766 EVT IntVT = SignAsInt.IntValue.getValueType();
1767
1768 // Flip the sign.
1769 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1770 SDValue SignFlip =
1771 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1772
1773 // Convert back to float.
1774 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1775}
1776
1777SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1778 SDLoc DL(Node);
1779 SDValue Value = Node->getOperand(Num: 0);
1780
1781 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1782 EVT FloatVT = Value.getValueType();
1783 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1784 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1785 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1786 }
1787
1788 // Transform value to integer, clear the sign bit and transform back.
1789 FloatSignAsInt ValueAsInt;
1790 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1791 EVT IntVT = ValueAsInt.IntValue.getValueType();
1792 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1793 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1794 N2: ClearSignMask);
1795 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1796}
1797
1798void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1799 SmallVectorImpl<SDValue> &Results) {
1800 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1801 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1802 " not tell us which reg is the stack pointer!");
1803 SDLoc dl(Node);
1804 EVT VT = Node->getValueType(ResNo: 0);
1805 SDValue Tmp1 = SDValue(Node, 0);
1806 SDValue Tmp2 = SDValue(Node, 1);
1807 SDValue Tmp3 = Node->getOperand(Num: 2);
1808 SDValue Chain = Tmp1.getOperand(i: 0);
1809
1810 // Chain the dynamic stack allocation so that it doesn't modify the stack
1811 // pointer when other instructions are using the stack.
1812 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1813
1814 SDValue Size = Tmp2.getOperand(i: 1);
1815 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1816 Chain = SP.getValue(R: 1);
1817 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1818 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1819 unsigned Opc =
1820 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1821 ISD::ADD : ISD::SUB;
1822
1823 Align StackAlign = TFL->getStackAlign();
1824 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1825 if (Alignment > StackAlign)
1826 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1827 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1828 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1829
1830 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1831
1832 Results.push_back(Elt: Tmp1);
1833 Results.push_back(Elt: Tmp2);
1834}
1835
1836/// Emit a store/load combination to the stack. This stores
1837/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1838/// a load from the stack slot to DestVT, extending it if needed.
1839/// The resultant code need not be legal.
1840SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1841 EVT DestVT, const SDLoc &dl) {
1842 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1843}
1844
1845SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1846 EVT DestVT, const SDLoc &dl,
1847 SDValue Chain) {
1848 EVT SrcVT = SrcOp.getValueType();
1849 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1850 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1851
1852 // Don't convert with stack if the load/store is expensive.
1853 if ((SrcVT.bitsGT(VT: SlotVT) &&
1854 !TLI.isTruncStoreLegalOrCustom(ValVT: SrcOp.getValueType(), MemVT: SlotVT)) ||
1855 (SlotVT.bitsLT(VT: DestVT) &&
1856 !TLI.isLoadExtLegalOrCustom(ExtType: ISD::EXTLOAD, ValVT: DestVT, MemVT: SlotVT)))
1857 return SDValue();
1858
1859 // Create the stack frame object.
1860 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1861 Ty: SrcOp.getValueType().getTypeForEVT(Context&: *DAG.getContext()));
1862 SDValue FIPtr = DAG.CreateStackTemporary(Bytes: SlotVT.getStoreSize(), Alignment: SrcAlign);
1863
1864 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: FIPtr);
1865 int SPFI = StackPtrFI->getIndex();
1866 MachinePointerInfo PtrInfo =
1867 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI);
1868
1869 // Emit a store to the stack slot. Use a truncstore if the input value is
1870 // later than DestVT.
1871 SDValue Store;
1872
1873 if (SrcVT.bitsGT(VT: SlotVT))
1874 Store = DAG.getTruncStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo,
1875 SVT: SlotVT, Alignment: SrcAlign);
1876 else {
1877 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1878 Store = DAG.getStore(Chain, dl, Val: SrcOp, Ptr: FIPtr, PtrInfo, Alignment: SrcAlign);
1879 }
1880
1881 // Result is a load from the stack slot.
1882 if (SlotVT.bitsEq(VT: DestVT))
1883 return DAG.getLoad(VT: DestVT, dl, Chain: Store, Ptr: FIPtr, PtrInfo, Alignment: DestAlign);
1884
1885 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1886 return DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: Store, Ptr: FIPtr, PtrInfo, MemVT: SlotVT,
1887 Alignment: DestAlign);
1888}
1889
1890SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1891 SDLoc dl(Node);
1892 // Create a vector sized/aligned stack slot, store the value to element #0,
1893 // then load the whole vector back out.
1894 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1895
1896 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1897 int SPFI = StackPtrFI->getIndex();
1898
1899 SDValue Ch = DAG.getTruncStore(
1900 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1901 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1902 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1903 return DAG.getLoad(
1904 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1905 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1906}
1907
1908static bool
1909ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1910 const TargetLowering &TLI, SDValue &Res) {
1911 unsigned NumElems = Node->getNumOperands();
1912 SDLoc dl(Node);
1913 EVT VT = Node->getValueType(ResNo: 0);
1914
1915 // Try to group the scalars into pairs, shuffle the pairs together, then
1916 // shuffle the pairs of pairs together, etc. until the vector has
1917 // been built. This will work only if all of the necessary shuffle masks
1918 // are legal.
1919
1920 // We do this in two phases; first to check the legality of the shuffles,
1921 // and next, assuming that all shuffles are legal, to create the new nodes.
1922 for (int Phase = 0; Phase < 2; ++Phase) {
1923 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1924 NewIntermedVals;
1925 for (unsigned i = 0; i < NumElems; ++i) {
1926 SDValue V = Node->getOperand(Num: i);
1927 if (V.isUndef())
1928 continue;
1929
1930 SDValue Vec;
1931 if (Phase)
1932 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1933 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1934 }
1935
1936 while (IntermedVals.size() > 2) {
1937 NewIntermedVals.clear();
1938 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1939 // This vector and the next vector are shuffled together (simply to
1940 // append the one to the other).
1941 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1942
1943 SmallVector<int, 16> FinalIndices;
1944 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1945 IntermedVals[i+1].second.size());
1946
1947 int k = 0;
1948 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1949 ++j, ++k) {
1950 ShuffleVec[k] = j;
1951 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1952 }
1953 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1954 ++j, ++k) {
1955 ShuffleVec[k] = NumElems + j;
1956 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1957 }
1958
1959 SDValue Shuffle;
1960 if (Phase)
1961 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1962 N2: IntermedVals[i+1].first,
1963 Mask: ShuffleVec);
1964 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1965 return false;
1966 NewIntermedVals.push_back(
1967 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1968 }
1969
1970 // If we had an odd number of defined values, then append the last
1971 // element to the array of new vectors.
1972 if ((IntermedVals.size() & 1) != 0)
1973 NewIntermedVals.push_back(Elt: IntermedVals.back());
1974
1975 IntermedVals.swap(RHS&: NewIntermedVals);
1976 }
1977
1978 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1979 "Invalid number of intermediate vectors");
1980 SDValue Vec1 = IntermedVals[0].first;
1981 SDValue Vec2;
1982 if (IntermedVals.size() > 1)
1983 Vec2 = IntermedVals[1].first;
1984 else if (Phase)
1985 Vec2 = DAG.getPOISON(VT);
1986
1987 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1988 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1989 ShuffleVec[IntermedVals[0].second[i]] = i;
1990 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1991 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1992
1993 if (Phase)
1994 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1995 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1996 return false;
1997 }
1998
1999 return true;
2000}
2001
2002/// Expand a BUILD_VECTOR node on targets that don't
2003/// support the operation, but do support the resultant vector type.
2004SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2005 unsigned NumElems = Node->getNumOperands();
2006 SDValue Value1, Value2;
2007 SDLoc dl(Node);
2008 EVT VT = Node->getValueType(ResNo: 0);
2009 EVT OpVT = Node->getOperand(Num: 0).getValueType();
2010 EVT EltVT = VT.getVectorElementType();
2011
2012 // If the only non-undef value is the low element, turn this into a
2013 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
2014 bool isOnlyLowElement = true;
2015 bool MoreThanTwoValues = false;
2016 bool isConstant = true;
2017 for (unsigned i = 0; i < NumElems; ++i) {
2018 SDValue V = Node->getOperand(Num: i);
2019 if (V.isUndef())
2020 continue;
2021 if (i > 0)
2022 isOnlyLowElement = false;
2023 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2024 isConstant = false;
2025
2026 if (!Value1.getNode()) {
2027 Value1 = V;
2028 } else if (!Value2.getNode()) {
2029 if (V != Value1)
2030 Value2 = V;
2031 } else if (V != Value1 && V != Value2) {
2032 MoreThanTwoValues = true;
2033 }
2034 }
2035
2036 if (!Value1.getNode())
2037 return DAG.getUNDEF(VT);
2038
2039 if (isOnlyLowElement)
2040 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2041
2042 // If all elements are constants, create a load from the constant pool.
2043 if (isConstant) {
2044 SmallVector<Constant*, 16> CV;
2045 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2046 if (ConstantFPSDNode *V =
2047 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2048 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2049 } else if (ConstantSDNode *V =
2050 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2051 if (OpVT==EltVT)
2052 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2053 else {
2054 // If OpVT and EltVT don't match, EltVT is not legal and the
2055 // element values have been promoted/truncated earlier. Undo this;
2056 // we don't want a v16i8 to become a v16i32 for example.
2057 const ConstantInt *CI = V->getConstantIntValue();
2058 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2059 V: CI->getZExtValue(), /*IsSigned=*/false,
2060 /*ImplicitTrunc=*/true));
2061 }
2062 } else {
2063 assert(Node->getOperand(i).isUndef());
2064 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2065 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2066 }
2067 }
2068 Constant *CP = ConstantVector::get(V: CV);
2069 SDValue CPIdx =
2070 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2071 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2072 return DAG.getLoad(
2073 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2074 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2075 Alignment);
2076 }
2077
2078 SmallSet<SDValue, 16> DefinedValues;
2079 for (unsigned i = 0; i < NumElems; ++i) {
2080 if (Node->getOperand(Num: i).isUndef())
2081 continue;
2082 DefinedValues.insert(V: Node->getOperand(Num: i));
2083 }
2084
2085 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2086 if (!MoreThanTwoValues) {
2087 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2088 for (unsigned i = 0; i < NumElems; ++i) {
2089 SDValue V = Node->getOperand(Num: i);
2090 if (V.isUndef())
2091 continue;
2092 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2093 }
2094 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2095 // Get the splatted value into the low element of a vector register.
2096 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2097 SDValue Vec2;
2098 if (Value2.getNode())
2099 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2100 else
2101 Vec2 = DAG.getPOISON(VT);
2102
2103 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2104 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2105 }
2106 } else {
2107 SDValue Res;
2108 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2109 return Res;
2110 }
2111 }
2112
2113 // Otherwise, we can't handle this case efficiently.
2114 return ExpandVectorBuildThroughStack(Node);
2115}
2116
2117SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2118 SDLoc DL(Node);
2119 EVT VT = Node->getValueType(ResNo: 0);
2120 SDValue SplatVal = Node->getOperand(Num: 0);
2121
2122 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2123}
2124
2125// Expand a node into a call to a libcall, returning the value as the first
2126// result and the chain as the second. If the result value does not fit into a
2127// register, return the lo part and set the hi part to the by-reg argument in
2128// the first. If it does fit into a single register, return the result and
2129// leave the Hi part unset.
2130std::pair<SDValue, SDValue>
2131SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2132 TargetLowering::ArgListTy &&Args,
2133 bool IsSigned, EVT RetVT) {
2134 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2135 SDValue Callee;
2136 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2137 if (LCImpl != RTLIB::Unsupported)
2138 Callee = DAG.getExternalSymbol(LCImpl, VT: CodePtrTy);
2139 else {
2140 Callee = DAG.getPOISON(VT: CodePtrTy);
2141 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2142 Node->getOperationName(G: &DAG));
2143 }
2144
2145 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2146
2147 // By default, the input chain to this libcall is the entry node of the
2148 // function. If the libcall is going to be emitted as a tail call then
2149 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2150 // node which is being folded has a non-entry input chain.
2151 SDValue InChain = DAG.getEntryNode();
2152
2153 // isTailCall may be true since the callee does not reference caller stack
2154 // frame. Check if it's in the right position and that the return types match.
2155 SDValue TCChain = InChain;
2156 const Function &F = DAG.getMachineFunction().getFunction();
2157 bool isTailCall =
2158 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2159 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2160 if (isTailCall)
2161 InChain = TCChain;
2162
2163 TargetLowering::CallLoweringInfo CLI(DAG);
2164 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2165 CLI.setDebugLoc(SDLoc(Node))
2166 .setChain(InChain)
2167 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
2168 Target: Callee, ArgsList: std::move(Args))
2169 .setTailCall(isTailCall)
2170 .setSExtResult(signExtend)
2171 .setZExtResult(!signExtend)
2172 .setIsPostTypeLegalization(true);
2173
2174 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2175
2176 if (!CallInfo.second.getNode()) {
2177 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2178 // It's a tailcall, return the chain (which is the DAG root).
2179 return {DAG.getRoot(), DAG.getRoot()};
2180 }
2181
2182 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2183 return CallInfo;
2184}
2185
2186std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2187 bool isSigned) {
2188 TargetLowering::ArgListTy Args;
2189 for (const SDValue &Op : Node->op_values()) {
2190 EVT ArgVT = Op.getValueType();
2191 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2192 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2193 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2194 Entry.IsZExt = !Entry.IsSExt;
2195 Args.push_back(x: Entry);
2196 }
2197
2198 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2199 RetVT: Node->getValueType(ResNo: 0));
2200}
2201
2202void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2203 RTLIB::Libcall LC,
2204 SmallVectorImpl<SDValue> &Results) {
2205 if (LC == RTLIB::UNKNOWN_LIBCALL)
2206 llvm_unreachable("Can't create an unknown libcall!");
2207
2208 if (Node->isStrictFPOpcode()) {
2209 EVT RetVT = Node->getValueType(ResNo: 0);
2210 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2211 TargetLowering::MakeLibCallOptions CallOptions;
2212 CallOptions.IsPostTypeLegalization = true;
2213 // FIXME: This doesn't support tail calls.
2214 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2215 Ops, CallOptions,
2216 dl: SDLoc(Node),
2217 Chain: Node->getOperand(Num: 0));
2218 Results.push_back(Elt: Tmp.first);
2219 Results.push_back(Elt: Tmp.second);
2220 } else {
2221 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2222 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2223 Results.push_back(Elt: Tmp);
2224 }
2225}
2226
2227/// Expand the node to a libcall based on the result type.
2228void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2229 RTLIB::Libcall Call_F32,
2230 RTLIB::Libcall Call_F64,
2231 RTLIB::Libcall Call_F80,
2232 RTLIB::Libcall Call_F128,
2233 RTLIB::Libcall Call_PPCF128,
2234 SmallVectorImpl<SDValue> &Results) {
2235 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: Node->getSimpleValueType(ResNo: 0),
2236 Call_F32, Call_F64, Call_F80,
2237 Call_F128, Call_PPCF128);
2238 ExpandFPLibCall(Node, LC, Results);
2239}
2240
2241void SelectionDAGLegalize::ExpandFastFPLibCall(
2242 SDNode *Node, bool IsFast,
2243 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2244 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2245 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2246 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2247 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2248 SmallVectorImpl<SDValue> &Results) {
2249
2250 EVT VT = Node->getSimpleValueType(ResNo: 0);
2251
2252 RTLIB::Libcall LC;
2253
2254 // FIXME: Probably should define fast to respect nan/inf and only be
2255 // approximate functions.
2256
2257 if (IsFast) {
2258 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.first, Call_F64: Call_F64.first, Call_F80: Call_F80.first,
2259 Call_F128: Call_F128.first, Call_PPCF128: Call_PPCF128.first);
2260 }
2261
2262 if (!IsFast || DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
2263 // Fall back if we don't have a fast implementation.
2264 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.second, Call_F64: Call_F64.second,
2265 Call_F80: Call_F80.second, Call_F128: Call_F128.second,
2266 Call_PPCF128: Call_PPCF128.second);
2267 }
2268
2269 ExpandFPLibCall(Node, LC, Results);
2270}
2271
2272SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2273 RTLIB::Libcall Call_I8,
2274 RTLIB::Libcall Call_I16,
2275 RTLIB::Libcall Call_I32,
2276 RTLIB::Libcall Call_I64,
2277 RTLIB::Libcall Call_I128) {
2278 RTLIB::Libcall LC;
2279 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2280 default: llvm_unreachable("Unexpected request for libcall!");
2281 case MVT::i8: LC = Call_I8; break;
2282 case MVT::i16: LC = Call_I16; break;
2283 case MVT::i32: LC = Call_I32; break;
2284 case MVT::i64: LC = Call_I64; break;
2285 case MVT::i128: LC = Call_I128; break;
2286 }
2287 return ExpandLibCall(LC, Node, isSigned).first;
2288}
2289
2290/// Expand the node to a libcall based on first argument type (for instance
2291/// lround and its variant).
2292void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2293 RTLIB::Libcall Call_F32,
2294 RTLIB::Libcall Call_F64,
2295 RTLIB::Libcall Call_F80,
2296 RTLIB::Libcall Call_F128,
2297 RTLIB::Libcall Call_PPCF128,
2298 SmallVectorImpl<SDValue> &Results) {
2299 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2300 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2301 Call_F32, Call_F64, Call_F80,
2302 Call_F128, Call_PPCF128);
2303 ExpandFPLibCall(Node, LC, Results);
2304}
2305
2306SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2307 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2308 RTLIB::Libcall CallI128) {
2309 RTLIB::Libcall LC;
2310 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2311 default:
2312 llvm_unreachable("Unexpected request for libcall!");
2313 case MVT::i32:
2314 LC = CallI32;
2315 break;
2316 case MVT::i64:
2317 LC = CallI64;
2318 break;
2319 case MVT::i128:
2320 LC = CallI128;
2321 break;
2322 }
2323
2324 // Bit-counting libcalls have one unsigned argument and return `int`.
2325 // Note that `int` may be illegal on this target; ExpandLibCall will
2326 // take care of promoting it to a legal type.
2327 SDValue Op = Node->getOperand(Num: 0);
2328 EVT IntVT =
2329 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2330
2331 EVT ArgVT = Op.getValueType();
2332 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2333 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2334 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2335 Arg.IsZExt = !Arg.IsSExt;
2336
2337 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2338 /*IsSigned=*/true, RetVT: IntVT)
2339 .first;
2340
2341 // If ExpandLibCall created a tail call, the result was already
2342 // of the correct type. Otherwise, we need to sign extend it.
2343 if (Res.getValueType() != MVT::Other)
2344 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2345 return Res;
2346}
2347
2348/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2349void
2350SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2351 SmallVectorImpl<SDValue> &Results) {
2352 unsigned Opcode = Node->getOpcode();
2353 bool isSigned = Opcode == ISD::SDIVREM;
2354
2355 RTLIB::Libcall LC;
2356 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2357 default: llvm_unreachable("Unexpected request for libcall!");
2358 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2359 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2360 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2361 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2362 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2363 }
2364
2365 // The input chain to this libcall is the entry node of the function.
2366 // Legalizing the call will automatically add the previous call to the
2367 // dependence.
2368 SDValue InChain = DAG.getEntryNode();
2369
2370 EVT RetVT = Node->getValueType(ResNo: 0);
2371 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2372
2373 TargetLowering::ArgListTy Args;
2374 for (const SDValue &Op : Node->op_values()) {
2375 EVT ArgVT = Op.getValueType();
2376 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2377 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2378 Entry.IsSExt = isSigned;
2379 Entry.IsZExt = !isSigned;
2380 Args.push_back(x: Entry);
2381 }
2382
2383 // Also pass the return address of the remainder.
2384 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2385 TargetLowering::ArgListEntry Entry(
2386 FIPtr, PointerType::getUnqual(C&: RetTy->getContext()));
2387 Entry.IsSExt = isSigned;
2388 Entry.IsZExt = !isSigned;
2389 Args.push_back(x: Entry);
2390
2391 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2392 if (LibcallImpl == RTLIB::Unsupported) {
2393 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2394 Node->getOperationName(G: &DAG));
2395 SDValue Poison = DAG.getPOISON(VT: RetVT);
2396 Results.push_back(Elt: Poison);
2397 Results.push_back(Elt: Poison);
2398 return;
2399 }
2400
2401 SDValue Callee =
2402 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2403
2404 SDLoc dl(Node);
2405 TargetLowering::CallLoweringInfo CLI(DAG);
2406 CLI.setDebugLoc(dl)
2407 .setChain(InChain)
2408 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
2409 ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2410 .setSExtResult(isSigned)
2411 .setZExtResult(!isSigned);
2412
2413 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2414
2415 // Remainder is loaded back from the stack frame.
2416 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
2417 MachinePointerInfo PtrInfo =
2418 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2419
2420 SDValue Rem = DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo);
2421 Results.push_back(Elt: CallInfo.first);
2422 Results.push_back(Elt: Rem);
2423}
2424
2425/// Return true if sincos or __sincos_stret libcall is available.
2426static bool isSinCosLibcallAvailable(SDNode *Node,
2427 const LibcallLoweringInfo &Libcalls) {
2428 MVT::SimpleValueType VT = Node->getSimpleValueType(ResNo: 0).SimpleTy;
2429 return Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS(RetVT: VT)) != RTLIB::Unsupported ||
2430 Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS_STRET(RetVT: VT)) !=
2431 RTLIB::Unsupported;
2432}
2433
2434/// Only issue sincos libcall if both sin and cos are needed.
2435static bool useSinCos(SDNode *Node) {
2436 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2437 ? ISD::FCOS : ISD::FSIN;
2438
2439 SDValue Op0 = Node->getOperand(Num: 0);
2440 for (const SDNode *User : Op0.getNode()->users()) {
2441 if (User == Node)
2442 continue;
2443 // The other user might have been turned into sincos already.
2444 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2445 return true;
2446 }
2447 return false;
2448}
2449
2450SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2451 // For iOS, we want to call an alternative entry point: __sincos_stret,
2452 // which returns the values in two S / D registers.
2453 SDLoc dl(Node);
2454 SDValue Arg = Node->getOperand(Num: 0);
2455 EVT ArgVT = Arg.getValueType();
2456 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(RetVT: ArgVT);
2457 RTLIB::LibcallImpl SincosStret = DAG.getLibcalls().getLibcallImpl(Call: LC);
2458 if (SincosStret == RTLIB::Unsupported)
2459 return SDValue();
2460
2461 /// There are 3 different ABI cases to handle:
2462 /// - Direct return of separate fields in registers
2463 /// - Single return as vector elements
2464 /// - sret struct
2465
2466 const RTLIB::RuntimeLibcallsInfo &CallsInfo = TLI.getRuntimeLibcallsInfo();
2467
2468 const DataLayout &DL = DAG.getDataLayout();
2469
2470 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2471 Ctx&: *DAG.getContext(), TT: TM.getTargetTriple(), DL, LibcallImpl: SincosStret);
2472
2473 Type *SincosStretRetTy = FuncTy->getReturnType();
2474 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(Call: SincosStret);
2475
2476 SDValue Callee =
2477 DAG.getExternalSymbol(LCImpl: SincosStret, VT: TLI.getProgramPointerTy(DL));
2478
2479 TargetLowering::ArgListTy Args;
2480 SDValue SRet;
2481
2482 int FrameIdx;
2483 if (FuncTy->getParamType(i: 0)->isPointerTy()) {
2484 // Uses sret
2485 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2486
2487 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(ArgNo: 0);
2488 Type *StructTy = PtrAttrs.getStructRetType();
2489 const uint64_t ByteSize = DL.getTypeAllocSize(Ty: StructTy);
2490 const Align StackAlign = DL.getPrefTypeAlign(Ty: StructTy);
2491
2492 FrameIdx = MFI.CreateStackObject(Size: ByteSize, Alignment: StackAlign, isSpillSlot: false);
2493 SRet = DAG.getFrameIndex(FI: FrameIdx, VT: TLI.getFrameIndexTy(DL));
2494
2495 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(i: 0));
2496 Entry.IsSRet = true;
2497 Entry.IndirectType = StructTy;
2498 Entry.Alignment = StackAlign;
2499
2500 Args.push_back(x: Entry);
2501 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 1));
2502 } else {
2503 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 0));
2504 }
2505
2506 TargetLowering::CallLoweringInfo CLI(DAG);
2507 CLI.setDebugLoc(dl)
2508 .setChain(DAG.getEntryNode())
2509 .setLibCallee(CC: CallConv, ResultType: SincosStretRetTy, Target: Callee, ArgsList: std::move(Args))
2510 .setIsPostTypeLegalization();
2511
2512 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2513
2514 if (SRet) {
2515 MachinePointerInfo PtrInfo =
2516 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2517 SDValue LoadSin = DAG.getLoad(VT: ArgVT, dl, Chain: CallResult.second, Ptr: SRet, PtrInfo);
2518
2519 TypeSize StoreSize = ArgVT.getStoreSize();
2520
2521 // Address of cos field.
2522 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr: SRet, Offset: StoreSize);
2523 SDValue LoadCos = DAG.getLoad(VT: ArgVT, dl, Chain: LoadSin.getValue(R: 1), Ptr: Add,
2524 PtrInfo: PtrInfo.getWithOffset(O: StoreSize));
2525
2526 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2527 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: LoadSin.getValue(R: 0),
2528 N2: LoadCos.getValue(R: 0));
2529 }
2530
2531 if (!CallResult.first.getValueType().isVector())
2532 return CallResult.first;
2533
2534 SDValue SinVal =
2535 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2536 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2537 SDValue CosVal =
2538 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2539 N2: DAG.getVectorIdxConstant(Val: 1, DL: dl));
2540 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2541 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: SinVal, N2: CosVal);
2542}
2543
2544SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2545 SDLoc dl(Node);
2546 EVT VT = Node->getValueType(ResNo: 0);
2547 SDValue X = Node->getOperand(Num: 0);
2548 SDValue N = Node->getOperand(Num: 1);
2549 EVT ExpVT = N.getValueType();
2550 EVT AsIntVT = VT.changeTypeToInteger();
2551 if (AsIntVT == EVT()) // TODO: How to handle f80?
2552 return SDValue();
2553
2554 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2555 return SDValue();
2556
2557 SDNodeFlags NSW;
2558 NSW.setNoSignedWrap(true);
2559 SDNodeFlags NUW_NSW;
2560 NUW_NSW.setNoUnsignedWrap(true);
2561 NUW_NSW.setNoSignedWrap(true);
2562
2563 EVT SetCCVT =
2564 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2565 const fltSemantics &FltSem = VT.getFltSemantics();
2566
2567 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2568 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2569 const int Precision = APFloat::semanticsPrecision(FltSem);
2570
2571 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2572 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2573
2574 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2575
2576 const APFloat One(FltSem, "1.0");
2577 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2578
2579 // Offset by precision to avoid denormal range.
2580 APFloat ScaleDownK =
2581 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2582
2583 // TODO: Should really introduce control flow and use a block for the >
2584 // MaxExp, < MinExp cases
2585
2586 // First, handle exponents Exp > MaxExp and scale down.
2587 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2588
2589 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2590 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2591 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2592 SDValue DecN1 =
2593 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2594
2595 SDValue ScaleUpTwice =
2596 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2597
2598 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2599 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2600 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2601
2602 SDValue SelectN_Big =
2603 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2604 SDValue SelectX_Big =
2605 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2606
2607 // Now handle exponents Exp < MinExp
2608 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2609
2610 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2611 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2612
2613 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2614
2615 SDValue ClampMinVal =
2616 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2617 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2618 SDValue IncN1 =
2619 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2620
2621 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2622 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2623 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2624
2625 SDValue ScaleDownTwice = DAG.getSetCC(
2626 DL: dl, VT: SetCCVT, LHS: N,
2627 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2628
2629 SDValue SelectN_Small =
2630 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2631 SDValue SelectX_Small =
2632 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2633
2634 // Now combine the two out of range exponent handling cases with the base
2635 // case.
2636 SDValue NewX = DAG.getNode(
2637 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2638 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2639
2640 SDValue NewN = DAG.getNode(
2641 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2642 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2643
2644 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2645
2646 SDValue ExponentShiftAmt =
2647 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2648 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2649
2650 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2651 N2: ExponentShiftAmt, Flags: NUW_NSW);
2652 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2653 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2654}
2655
2656SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2657 SDLoc dl(Node);
2658 SDValue Val = Node->getOperand(Num: 0);
2659 EVT VT = Val.getValueType();
2660 EVT ExpVT = Node->getValueType(ResNo: 1);
2661 EVT AsIntVT = VT.changeTypeToInteger();
2662 if (AsIntVT == EVT()) // TODO: How to handle f80?
2663 return SDValue();
2664
2665 const fltSemantics &FltSem = VT.getFltSemantics();
2666 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2667 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2668 const unsigned BitSize = VT.getScalarSizeInBits();
2669
2670 // TODO: Could introduce control flow and skip over the denormal handling.
2671
2672 // scale_up = fmul value, scalbn(1.0, precision + 1)
2673 // extracted_exp = (bitcast value to uint) >> precision - 1
2674 // biased_exp = extracted_exp + min_exp
2675 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2676 //
2677 // is_denormal = val < smallest_normalized
2678 // computed_fract = is_denormal ? scale_up : extracted_fract
2679 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2680 //
2681 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2682 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2683
2684 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2685 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2686 VT: AsIntVT);
2687
2688 SDValue SmallestNormalizedInt = DAG.getConstant(
2689 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2690 VT: AsIntVT);
2691
2692 // Masks out the exponent bits.
2693 SDValue ExpMask =
2694 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2695
2696 // Mask out the exponent part of the value.
2697 //
2698 // e.g, for f32 FractSignMaskVal = 0x807fffff
2699 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2700 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2701
2702 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2703 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2704
2705 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2706
2707 const APFloat One(FltSem, "1.0");
2708 // Scale a possible denormal input.
2709 // e.g., for f64, 0x1p+54
2710 APFloat ScaleUpKVal =
2711 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2712
2713 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2714 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2715
2716 EVT SetCCVT =
2717 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2718
2719 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2720
2721 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2722
2723 SDValue AddNegSmallestNormal =
2724 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2725 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2726 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2727
2728 SDValue IsDenormal =
2729 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2730
2731 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2732 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2733
2734 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2735 SDValue ScaledSelect =
2736 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2737
2738 SDValue ExpMaskScaled =
2739 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2740
2741 SDValue ScaledValue =
2742 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2743
2744 // Extract the exponent bits.
2745 SDValue ExponentShiftAmt =
2746 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2747 SDValue ShiftedExp =
2748 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2749 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2750
2751 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2752 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2753 SDValue DenormalExpBias =
2754 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2755
2756 SDValue MaskedFractAsInt =
2757 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2758 const APFloat Half(FltSem, "0.5");
2759 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2760 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2761 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2762
2763 SDValue ComputedExp =
2764 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2765
2766 SDValue Result0 =
2767 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2768
2769 SDValue Result1 =
2770 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2771
2772 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2773}
2774
2775SDValue SelectionDAGLegalize::expandModf(SDNode *Node) const {
2776 SDLoc dl(Node);
2777 SDValue Val = Node->getOperand(Num: 0);
2778 EVT VT = Val.getValueType();
2779 SDNodeFlags Flags = Node->getFlags();
2780
2781 SDValue IntPart = DAG.getNode(Opcode: ISD::FTRUNC, DL: dl, VT, Operand: Val, Flags);
2782 SDValue FracPart = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT, N1: Val, N2: IntPart, Flags);
2783
2784 SDValue FracToUse;
2785 if (Flags.hasNoInfs()) {
2786 FracToUse = FracPart;
2787 } else {
2788 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val, Flags);
2789 SDValue Inf =
2790 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
2791 EVT SetCCVT =
2792 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2793 SDValue IsInf = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: Inf, Cond: ISD::SETOEQ);
2794 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: dl, VT);
2795 FracToUse = DAG.getSelect(DL: dl, VT, Cond: IsInf, LHS: Zero, RHS: FracPart);
2796 }
2797
2798 SDValue ResultFrac =
2799 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT, N1: FracToUse, N2: Val, Flags);
2800 return DAG.getMergeValues(Ops: {ResultFrac, IntPart}, dl);
2801}
2802
2803/// This function is responsible for legalizing a
2804/// INT_TO_FP operation of the specified operand when the target requests that
2805/// we expand it. At this point, we know that the result and operand types are
2806/// legal for the target.
2807SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2808 SDValue &Chain) {
2809 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2810 Node->getOpcode() == ISD::SINT_TO_FP);
2811 EVT DestVT = Node->getValueType(ResNo: 0);
2812 SDLoc dl(Node);
2813 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2814 SDValue Op0 = Node->getOperand(Num: OpNo);
2815 EVT SrcVT = Op0.getValueType();
2816
2817 // TODO: Should any fast-math-flags be set for the created nodes?
2818 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2819 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2820 (DestVT.bitsLE(VT: MVT::f64) ||
2821 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2822 : ISD::FP_EXTEND,
2823 VT: DestVT))) {
2824 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2825 "expansion\n");
2826
2827 // Get the stack frame index of a 8 byte buffer.
2828 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2829
2830 SDValue Lo = Op0;
2831 // if signed map to unsigned space
2832 if (isSigned) {
2833 // Invert sign bit (signed to unsigned mapping).
2834 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2835 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2836 }
2837 // Initial hi portion of constructed double.
2838 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2839
2840 // If this a big endian target, swap the lo and high data.
2841 if (DAG.getDataLayout().isBigEndian())
2842 std::swap(a&: Lo, b&: Hi);
2843
2844 SDValue MemChain = DAG.getEntryNode();
2845
2846 // Store the lo of the constructed double.
2847 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2848 PtrInfo: MachinePointerInfo());
2849 // Store the hi of the constructed double.
2850 SDValue HiPtr =
2851 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2852 SDValue Store2 =
2853 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2854 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2855
2856 // load the constructed double
2857 SDValue Load =
2858 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2859 // FP constant to bias correct the final result
2860 SDValue Bias = DAG.getConstantFP(
2861 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2862 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2863 DL: dl, VT: MVT::f64);
2864 // Subtract the bias and get the final result.
2865 SDValue Sub;
2866 SDValue Result;
2867 if (Node->isStrictFPOpcode()) {
2868 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2869 Ops: {Node->getOperand(Num: 0), Load, Bias});
2870 Chain = Sub.getValue(R: 1);
2871 if (DestVT != Sub.getValueType()) {
2872 std::pair<SDValue, SDValue> ResultPair;
2873 ResultPair =
2874 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2875 Result = ResultPair.first;
2876 Chain = ResultPair.second;
2877 }
2878 else
2879 Result = Sub;
2880 } else {
2881 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2882 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2883 }
2884 return Result;
2885 }
2886
2887 if (isSigned)
2888 return SDValue();
2889
2890 // TODO: Generalize this for use with other types.
2891 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2892 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2893 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2894 // For unsigned conversions, convert them to signed conversions using the
2895 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2896 // should be valid for i32->f32 as well.
2897
2898 // More generally this transform should be valid if there are 3 more bits
2899 // in the integer type than the significand. Rounding uses the first bit
2900 // after the width of the significand and the OR of all bits after that. So
2901 // we need to be able to OR the shifted out bit into one of the bits that
2902 // participate in the OR.
2903
2904 // TODO: This really should be implemented using a branch rather than a
2905 // select. We happen to get lucky and machinesink does the right
2906 // thing most of the time. This would be a good candidate for a
2907 // pseudo-op, or, even better, for whole-function isel.
2908 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2909
2910 SDValue SignBitTest = DAG.getSetCC(
2911 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2912
2913 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL: dl);
2914 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2915 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2916 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2917 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2918
2919 SDValue Slow, Fast;
2920 if (Node->isStrictFPOpcode()) {
2921 // In strict mode, we must avoid spurious exceptions, and therefore
2922 // must make sure to only emit a single STRICT_SINT_TO_FP.
2923 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2924 // The STRICT_SINT_TO_FP inherits the exception mode from the
2925 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2926 // never raise any exception.
2927 SDNodeFlags Flags;
2928 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2929 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: {DestVT, MVT::Other},
2930 Ops: {Node->getOperand(Num: 0), InCvt}, Flags);
2931 Flags.setNoFPExcept(true);
2932 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: {DestVT, MVT::Other},
2933 Ops: {Fast.getValue(R: 1), Fast, Fast}, Flags);
2934 Chain = Slow.getValue(R: 1);
2935 } else {
2936 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2937 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2938 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2939 }
2940
2941 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2942 }
2943
2944 // Don't expand it if there isn't cheap fadd.
2945 if (!TLI.isOperationLegalOrCustom(
2946 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2947 return SDValue();
2948
2949 // The following optimization is valid only if every value in SrcVT (when
2950 // treated as signed) is representable in DestVT. Check that the mantissa
2951 // size of DestVT is >= than the number of bits in SrcVT -1.
2952 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2953 SrcVT.getSizeInBits() - 1 &&
2954 "Cannot perform lossless SINT_TO_FP!");
2955
2956 SDValue Tmp1;
2957 if (Node->isStrictFPOpcode()) {
2958 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2959 Ops: { Node->getOperand(Num: 0), Op0 });
2960 } else
2961 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2962
2963 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2964 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2965 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2966 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2967 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2968 Cond: SignSet, LHS: Four, RHS: Zero);
2969
2970 // If the sign bit of the integer is set, the large number will be treated
2971 // as a negative number. To counteract this, the dynamic code adds an
2972 // offset depending on the data type.
2973 uint64_t FF;
2974 switch (SrcVT.getSimpleVT().SimpleTy) {
2975 default:
2976 return SDValue();
2977 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2978 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2979 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2980 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2981 }
2982 if (DAG.getDataLayout().isLittleEndian())
2983 FF <<= 32;
2984 Constant *FudgeFactor = ConstantInt::get(
2985 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2986
2987 SDValue CPIdx =
2988 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2989 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2990 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2991 Alignment = commonAlignment(A: Alignment, Offset: 4);
2992 SDValue FudgeInReg;
2993 if (DestVT == MVT::f32)
2994 FudgeInReg = DAG.getLoad(
2995 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2996 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2997 Alignment);
2998 else {
2999 SDValue Load = DAG.getExtLoad(
3000 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
3001 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
3002 Alignment);
3003 HandleSDNode Handle(Load);
3004 LegalizeOp(Node: Load.getNode());
3005 FudgeInReg = Handle.getValue();
3006 }
3007
3008 if (Node->isStrictFPOpcode()) {
3009 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
3010 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
3011 Chain = Result.getValue(R: 1);
3012 return Result;
3013 }
3014
3015 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
3016}
3017
3018/// This function is responsible for legalizing a
3019/// *INT_TO_FP operation of the specified operand when the target requests that
3020/// we promote it. At this point, we know that the result and operand types are
3021/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
3022/// operation that takes a larger input.
3023void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
3024 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
3025 bool IsStrict = N->isStrictFPOpcode();
3026 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
3027 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
3028 EVT DestVT = N->getValueType(ResNo: 0);
3029 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3030 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
3031 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
3032
3033 // First step, figure out the appropriate *INT_TO_FP operation to use.
3034 EVT NewInTy = LegalOp.getValueType();
3035
3036 unsigned OpToUse = 0;
3037
3038 // Scan for the appropriate larger type to use.
3039 while (true) {
3040 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
3041 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3042
3043 // If the target supports SINT_TO_FP of this type, use it.
3044 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
3045 OpToUse = SIntOp;
3046 break;
3047 }
3048 if (IsSigned)
3049 continue;
3050
3051 // If the target supports UINT_TO_FP of this type, use it.
3052 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
3053 OpToUse = UIntOp;
3054 break;
3055 }
3056
3057 // Otherwise, try a larger type.
3058 }
3059
3060 // Okay, we found the operation and type to use. Zero extend our input to the
3061 // desired type then run the operation on it.
3062 if (IsStrict) {
3063 SDValue Res =
3064 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
3065 Ops: {N->getOperand(Num: 0),
3066 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3067 DL: dl, VT: NewInTy, Operand: LegalOp)});
3068 Results.push_back(Elt: Res);
3069 Results.push_back(Elt: Res.getValue(R: 1));
3070 return;
3071 }
3072
3073 Results.push_back(
3074 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
3075 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3076 DL: dl, VT: NewInTy, Operand: LegalOp)));
3077}
3078
3079/// This function is responsible for legalizing a
3080/// FP_TO_*INT operation of the specified operand when the target requests that
3081/// we promote it. At this point, we know that the result and operand types are
3082/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
3083/// operation that returns a larger result.
3084void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3085 SmallVectorImpl<SDValue> &Results) {
3086 bool IsStrict = N->isStrictFPOpcode();
3087 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3088 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3089 EVT DestVT = N->getValueType(ResNo: 0);
3090 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3091 // First step, figure out the appropriate FP_TO*INT operation to use.
3092 EVT NewOutTy = DestVT;
3093
3094 unsigned OpToUse = 0;
3095
3096 // Scan for the appropriate larger type to use.
3097 while (true) {
3098 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
3099 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3100
3101 // A larger signed type can hold all unsigned values of the requested type,
3102 // so using FP_TO_SINT is valid
3103 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
3104 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3105 break;
3106
3107 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
3108 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
3109 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3110 break;
3111
3112 // Otherwise, try a larger type.
3113 }
3114
3115 // Okay, we found the operation and type to use.
3116 SDValue Operation;
3117 if (IsStrict) {
3118 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
3119 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
3120 } else
3121 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
3122
3123 // Truncate the result of the extended FP_TO_*INT operation to the desired
3124 // size.
3125 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
3126 Results.push_back(Elt: Trunc);
3127 if (IsStrict)
3128 Results.push_back(Elt: Operation.getValue(R: 1));
3129}
3130
3131/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
3132/// the result and operand types are legal and there must be a legal
3133/// FP_TO_*INT_SAT operation for a larger result type.
3134SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3135 const SDLoc &dl) {
3136 unsigned Opcode = Node->getOpcode();
3137
3138 // Scan for the appropriate larger type to use.
3139 EVT NewOutTy = Node->getValueType(ResNo: 0);
3140 while (true) {
3141 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
3142 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3143
3144 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
3145 break;
3146 }
3147
3148 // Saturation width is determined by second operand, so we don't have to
3149 // perform any fixup and can directly truncate the result.
3150 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
3151 N2: Node->getOperand(Num: 1));
3152 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
3153}
3154
3155/// Open code the operations for PARITY of the specified operation.
3156SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3157 EVT VT = Op.getValueType();
3158 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3159 unsigned Sz = VT.getScalarSizeInBits();
3160
3161 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3162 SDValue Result;
3163 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3164 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3165 } else {
3166 Result = Op;
3167 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3168 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3169 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3170 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3171 }
3172 }
3173
3174 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3175}
3176
3177SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3178 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3179 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3180 : Node->getOperand(Num: 0).getSimpleValueType();
3181 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3182 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3183 MVT NewScalarVT = NewVecVT.getVectorElementType();
3184
3185 SDLoc DL(Node);
3186 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3187
3188 // FIXME: Support integer.
3189 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3190 "Only FP promotion is supported");
3191
3192 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3193 if (Node->getOperand(Num: j).getValueType().isVector() &&
3194 !(IsVPOpcode &&
3195 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3196 // promote the vector operand.
3197 // FIXME: Support integer.
3198 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3199 "Only FP promotion is supported");
3200 Operands[j] =
3201 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3202 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3203 // promote the initial value.
3204 Operands[j] =
3205 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3206 } else {
3207 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3208 }
3209
3210 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3211 Flags: Node->getFlags());
3212
3213 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3214 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3215 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3216}
3217
3218bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3219 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3220 SmallVector<SDValue, 8> Results;
3221 SDLoc dl(Node);
3222 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3223 bool NeedInvert;
3224 switch (Node->getOpcode()) {
3225 case ISD::ABS:
3226 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3227 Results.push_back(Elt: Tmp1);
3228 break;
3229 case ISD::ABDS:
3230 case ISD::ABDU:
3231 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3232 Results.push_back(Elt: Tmp1);
3233 break;
3234 case ISD::AVGCEILS:
3235 case ISD::AVGCEILU:
3236 case ISD::AVGFLOORS:
3237 case ISD::AVGFLOORU:
3238 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3239 Results.push_back(Elt: Tmp1);
3240 break;
3241 case ISD::CTPOP:
3242 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3243 Results.push_back(Elt: Tmp1);
3244 break;
3245 case ISD::CTLZ:
3246 case ISD::CTLZ_ZERO_UNDEF:
3247 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3248 Results.push_back(Elt: Tmp1);
3249 break;
3250 case ISD::CTLS:
3251 if ((Tmp1 = TLI.expandCTLS(N: Node, DAG)))
3252 Results.push_back(Elt: Tmp1);
3253 break;
3254 case ISD::CTTZ:
3255 case ISD::CTTZ_ZERO_UNDEF:
3256 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3257 Results.push_back(Elt: Tmp1);
3258 break;
3259 case ISD::BITREVERSE:
3260 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3261 Results.push_back(Elt: Tmp1);
3262 break;
3263 case ISD::BSWAP:
3264 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3265 Results.push_back(Elt: Tmp1);
3266 break;
3267 case ISD::PARITY:
3268 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3269 break;
3270 case ISD::FRAMEADDR:
3271 case ISD::RETURNADDR:
3272 case ISD::FRAME_TO_ARGS_OFFSET:
3273 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3274 break;
3275 case ISD::EH_DWARF_CFA: {
3276 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3277 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3278 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3279 VT: CfaArg.getValueType(),
3280 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3281 VT: CfaArg.getValueType()),
3282 N2: CfaArg);
3283 SDValue FA = DAG.getNode(
3284 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3285 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3286 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3287 N1: FA, N2: Offset));
3288 break;
3289 }
3290 case ISD::GET_ROUNDING:
3291 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3292 Results.push_back(Elt: Node->getOperand(Num: 0));
3293 break;
3294 case ISD::EH_RETURN:
3295 case ISD::EH_LABEL:
3296 case ISD::PREFETCH:
3297 case ISD::VAEND:
3298 case ISD::EH_SJLJ_LONGJMP:
3299 // If the target didn't expand these, there's nothing to do, so just
3300 // preserve the chain and be done.
3301 Results.push_back(Elt: Node->getOperand(Num: 0));
3302 break;
3303 case ISD::READCYCLECOUNTER:
3304 case ISD::READSTEADYCOUNTER:
3305 // If the target didn't expand this, just return 'zero' and preserve the
3306 // chain.
3307 Results.append(NumInputs: Node->getNumValues() - 1,
3308 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3309 Results.push_back(Elt: Node->getOperand(Num: 0));
3310 break;
3311 case ISD::EH_SJLJ_SETJMP:
3312 // If the target didn't expand this, just return 'zero' and preserve the
3313 // chain.
3314 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3315 Results.push_back(Elt: Node->getOperand(Num: 0));
3316 break;
3317 case ISD::ATOMIC_LOAD: {
3318 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3319 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3320 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3321 SDValue Swap = DAG.getAtomicCmpSwap(
3322 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3323 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3324 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3325 Results.push_back(Elt: Swap.getValue(R: 0));
3326 Results.push_back(Elt: Swap.getValue(R: 1));
3327 break;
3328 }
3329 case ISD::ATOMIC_STORE: {
3330 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3331 SDValue Swap = DAG.getAtomic(
3332 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3333 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3334 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3335 Results.push_back(Elt: Swap.getValue(R: 1));
3336 break;
3337 }
3338 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3339 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3340 // splits out the success value as a comparison. Expanding the resulting
3341 // ATOMIC_CMP_SWAP will produce a libcall.
3342 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3343 SDValue Res = DAG.getAtomicCmpSwap(
3344 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3345 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3346 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3347
3348 SDValue ExtRes = Res;
3349 SDValue LHS = Res;
3350 SDValue RHS = Node->getOperand(Num: 1);
3351
3352 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3353 EVT OuterType = Node->getValueType(ResNo: 0);
3354 switch (TLI.getExtendForAtomicOps()) {
3355 case ISD::SIGN_EXTEND:
3356 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3357 N2: DAG.getValueType(AtomicType));
3358 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3359 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3360 ExtRes = LHS;
3361 break;
3362 case ISD::ZERO_EXTEND:
3363 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3364 N2: DAG.getValueType(AtomicType));
3365 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3366 ExtRes = LHS;
3367 break;
3368 case ISD::ANY_EXTEND:
3369 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3370 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3371 break;
3372 default:
3373 llvm_unreachable("Invalid atomic op extension");
3374 }
3375
3376 SDValue Success =
3377 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3378
3379 Results.push_back(Elt: ExtRes.getValue(R: 0));
3380 Results.push_back(Elt: Success);
3381 Results.push_back(Elt: Res.getValue(R: 1));
3382 break;
3383 }
3384 case ISD::ATOMIC_LOAD_SUB: {
3385 SDLoc DL(Node);
3386 EVT VT = Node->getValueType(ResNo: 0);
3387 SDValue RHS = Node->getOperand(Num: 2);
3388 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3389 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3390 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3391 RHS = RHS->getOperand(Num: 0);
3392 SDValue NewRHS =
3393 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3394 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3395 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3396 Val: NewRHS, MMO: AN->getMemOperand());
3397 Results.push_back(Elt: Res);
3398 Results.push_back(Elt: Res.getValue(R: 1));
3399 break;
3400 }
3401 case ISD::DYNAMIC_STACKALLOC:
3402 ExpandDYNAMIC_STACKALLOC(Node, Results);
3403 break;
3404 case ISD::MERGE_VALUES:
3405 for (unsigned i = 0; i < Node->getNumValues(); i++)
3406 Results.push_back(Elt: Node->getOperand(Num: i));
3407 break;
3408 case ISD::POISON:
3409 case ISD::UNDEF: {
3410 EVT VT = Node->getValueType(ResNo: 0);
3411 if (VT.isInteger())
3412 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3413 else {
3414 assert(VT.isFloatingPoint() && "Unknown value type!");
3415 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3416 }
3417 break;
3418 }
3419 case ISD::STRICT_FP_ROUND:
3420 // When strict mode is enforced we can't do expansion because it
3421 // does not honor the "strict" properties. Only libcall is allowed.
3422 if (TLI.isStrictFPEnabled())
3423 break;
3424 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3425 // since this operation is more efficient than stack operation.
3426 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3427 VT: Node->getValueType(ResNo: 0))
3428 == TargetLowering::Legal)
3429 break;
3430 // We fall back to use stack operation when the FP_ROUND operation
3431 // isn't available.
3432 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3433 DestVT: Node->getValueType(ResNo: 0), dl,
3434 Chain: Node->getOperand(Num: 0)))) {
3435 ReplaceNode(Old: Node, New: Tmp1.getNode());
3436 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3437 return true;
3438 }
3439 break;
3440 case ISD::FP_ROUND: {
3441 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3442 Results.push_back(Elt: Tmp1);
3443 break;
3444 }
3445
3446 [[fallthrough]];
3447 }
3448 case ISD::BITCAST:
3449 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3450 DestVT: Node->getValueType(ResNo: 0), dl)))
3451 Results.push_back(Elt: Tmp1);
3452 break;
3453 case ISD::STRICT_FP_EXTEND:
3454 // When strict mode is enforced we can't do expansion because it
3455 // does not honor the "strict" properties. Only libcall is allowed.
3456 if (TLI.isStrictFPEnabled())
3457 break;
3458 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3459 // since this operation is more efficient than stack operation.
3460 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3461 VT: Node->getValueType(ResNo: 0))
3462 == TargetLowering::Legal)
3463 break;
3464 // We fall back to use stack operation when the FP_EXTEND operation
3465 // isn't available.
3466 if ((Tmp1 = EmitStackConvert(
3467 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3468 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3469 ReplaceNode(Old: Node, New: Tmp1.getNode());
3470 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3471 return true;
3472 }
3473 break;
3474 case ISD::FP_EXTEND: {
3475 SDValue Op = Node->getOperand(Num: 0);
3476 EVT SrcVT = Op.getValueType();
3477 EVT DstVT = Node->getValueType(ResNo: 0);
3478 if (SrcVT.getScalarType() == MVT::bf16) {
3479 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3480 break;
3481 }
3482
3483 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3484 Results.push_back(Elt: Tmp1);
3485 break;
3486 }
3487 case ISD::BF16_TO_FP: {
3488 // Always expand bf16 to f32 casts, they lower to ext + shift.
3489 //
3490 // Note that the operand of this code can be bf16 or an integer type in case
3491 // bf16 is not supported on the target and was softened.
3492 SDValue Op = Node->getOperand(Num: 0);
3493 if (Op.getValueType() == MVT::bf16) {
3494 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3495 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3496 } else {
3497 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3498 }
3499 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3500 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3501 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3502 // Add fp_extend in case the output is bigger than f32.
3503 if (Node->getValueType(ResNo: 0) != MVT::f32)
3504 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3505 Results.push_back(Elt: Op);
3506 break;
3507 }
3508 case ISD::FP_TO_BF16: {
3509 SDValue Op = Node->getOperand(Num: 0);
3510 if (Op.getValueType() != MVT::f32)
3511 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3512 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3513 // Certain SNaNs will turn into infinities if we do a simple shift right.
3514 if (!DAG.isKnownNeverSNaN(Op)) {
3515 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3516 }
3517 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
3518 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3519 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3520 // The result of this node can be bf16 or an integer type in case bf16 is
3521 // not supported on the target and was softened to i16 for storage.
3522 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3523 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3524 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3525 } else {
3526 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3527 }
3528 Results.push_back(Elt: Op);
3529 break;
3530 }
3531 case ISD::FCANONICALIZE: {
3532 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
3533 // suggested in
3534 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
3535 // It uses strict_fp operations even outside a strict_fp context in order
3536 // to guarantee that the canonicalization is not optimized away by later
3537 // passes. The result chain introduced by that is intentionally ignored
3538 // since no ordering requirement is intended here.
3539
3540 // Create strict multiplication by 1.0.
3541 SDValue Operand = Node->getOperand(Num: 0);
3542 EVT VT = Operand.getValueType();
3543 SDValue One = DAG.getConstantFP(Val: 1.0, DL: dl, VT);
3544 SDValue Chain = DAG.getEntryNode();
3545 // Propagate existing flags on canonicalize, and additionally set
3546 // NoFPExcept.
3547 SDNodeFlags CanonicalizeFlags = Node->getFlags();
3548 CanonicalizeFlags.setNoFPExcept(true);
3549 SDValue Mul = DAG.getNode(Opcode: ISD::STRICT_FMUL, DL: dl, ResultTys: {VT, MVT::Other},
3550 Ops: {Chain, Operand, One}, Flags: CanonicalizeFlags);
3551
3552 Results.push_back(Elt: Mul);
3553 break;
3554 }
3555 case ISD::SIGN_EXTEND_INREG: {
3556 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3557 EVT VT = Node->getValueType(ResNo: 0);
3558
3559 // An in-register sign-extend of a boolean is a negation:
3560 // 'true' (1) sign-extended is -1.
3561 // 'false' (0) sign-extended is 0.
3562 // However, we must mask the high bits of the source operand because the
3563 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3564
3565 // TODO: Do this for vectors too?
3566 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3567 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3568 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3569 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3570 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3571 Results.push_back(Elt: Neg);
3572 break;
3573 }
3574
3575 // NOTE: we could fall back on load/store here too for targets without
3576 // SRA. However, it is doubtful that any exist.
3577 unsigned BitsDiff = VT.getScalarSizeInBits() -
3578 ExtraVT.getScalarSizeInBits();
3579 SDValue ShiftCst = DAG.getShiftAmountConstant(Val: BitsDiff, VT, DL: dl);
3580 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: ShiftCst);
3581 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Tmp1, N2: ShiftCst);
3582 Results.push_back(Elt: Tmp1);
3583 break;
3584 }
3585 case ISD::UINT_TO_FP:
3586 case ISD::STRICT_UINT_TO_FP:
3587 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3588 Results.push_back(Elt: Tmp1);
3589 if (Node->isStrictFPOpcode())
3590 Results.push_back(Elt: Tmp2);
3591 break;
3592 }
3593 [[fallthrough]];
3594 case ISD::SINT_TO_FP:
3595 case ISD::STRICT_SINT_TO_FP:
3596 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3597 Results.push_back(Elt: Tmp1);
3598 if (Node->isStrictFPOpcode())
3599 Results.push_back(Elt: Tmp2);
3600 }
3601 break;
3602 case ISD::FP_TO_SINT:
3603 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3604 Results.push_back(Elt: Tmp1);
3605 break;
3606 case ISD::STRICT_FP_TO_SINT:
3607 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3608 ReplaceNode(Old: Node, New: Tmp1.getNode());
3609 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3610 return true;
3611 }
3612 break;
3613 case ISD::FP_TO_UINT:
3614 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3615 Results.push_back(Elt: Tmp1);
3616 break;
3617 case ISD::STRICT_FP_TO_UINT:
3618 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3619 // Relink the chain.
3620 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3621 // Replace the new UINT result.
3622 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3623 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3624 return true;
3625 }
3626 break;
3627 case ISD::FP_TO_SINT_SAT:
3628 case ISD::FP_TO_UINT_SAT:
3629 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3630 break;
3631 case ISD::LROUND:
3632 case ISD::LLROUND: {
3633 SDValue Arg = Node->getOperand(Num: 0);
3634 EVT ArgVT = Arg.getValueType();
3635 EVT ResVT = Node->getValueType(ResNo: 0);
3636 SDLoc dl(Node);
3637 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3638 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3639 break;
3640 }
3641 case ISD::VAARG:
3642 Results.push_back(Elt: DAG.expandVAArg(Node));
3643 Results.push_back(Elt: Results[0].getValue(R: 1));
3644 break;
3645 case ISD::VACOPY:
3646 Results.push_back(Elt: DAG.expandVACopy(Node));
3647 break;
3648 case ISD::EXTRACT_VECTOR_ELT:
3649 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3650 // This must be an access of the only element. Return it.
3651 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3652 Operand: Node->getOperand(Num: 0));
3653 else
3654 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3655 Results.push_back(Elt: Tmp1);
3656 break;
3657 case ISD::EXTRACT_SUBVECTOR:
3658 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3659 break;
3660 case ISD::INSERT_SUBVECTOR:
3661 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3662 break;
3663 case ISD::CONCAT_VECTORS:
3664 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3665 VectorValueType.isScalableVector() ||
3666 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3667 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3668 else
3669 Results.push_back(Elt: ExpandConcatVectors(Node));
3670 break;
3671 case ISD::SCALAR_TO_VECTOR:
3672 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3673 break;
3674 case ISD::INSERT_VECTOR_ELT:
3675 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3676 break;
3677 case ISD::VECTOR_SHUFFLE: {
3678 SmallVector<int, 32> NewMask;
3679 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3680
3681 EVT VT = Node->getValueType(ResNo: 0);
3682 EVT EltVT = VT.getVectorElementType();
3683 SDValue Op0 = Node->getOperand(Num: 0);
3684 SDValue Op1 = Node->getOperand(Num: 1);
3685 if (!TLI.isTypeLegal(VT: EltVT)) {
3686 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3687
3688 // BUILD_VECTOR operands are allowed to be wider than the element type.
3689 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3690 // it.
3691 if (NewEltVT.bitsLT(VT: EltVT)) {
3692 // Convert shuffle node.
3693 // If original node was v4i64 and the new EltVT is i32,
3694 // cast operands to v8i32 and re-build the mask.
3695
3696 // Calculate new VT, the size of the new VT should be equal to original.
3697 EVT NewVT =
3698 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3699 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3700 assert(NewVT.bitsEq(VT));
3701
3702 // cast operands to new VT
3703 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3704 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3705
3706 // Convert the shuffle mask
3707 unsigned int factor =
3708 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3709
3710 // EltVT gets smaller
3711 assert(factor > 0);
3712
3713 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3714 if (Mask[i] < 0) {
3715 for (unsigned fi = 0; fi < factor; ++fi)
3716 NewMask.push_back(Elt: Mask[i]);
3717 }
3718 else {
3719 for (unsigned fi = 0; fi < factor; ++fi)
3720 NewMask.push_back(Elt: Mask[i]*factor+fi);
3721 }
3722 }
3723 Mask = NewMask;
3724 VT = NewVT;
3725 }
3726 EltVT = NewEltVT;
3727 }
3728 unsigned NumElems = VT.getVectorNumElements();
3729 SmallVector<SDValue, 16> Ops;
3730 for (unsigned i = 0; i != NumElems; ++i) {
3731 if (Mask[i] < 0) {
3732 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3733 continue;
3734 }
3735 unsigned Idx = Mask[i];
3736 if (Idx < NumElems)
3737 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3738 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3739 else
3740 Ops.push_back(
3741 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3742 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3743 }
3744
3745 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3746 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3747 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3748 Results.push_back(Elt: Tmp1);
3749 break;
3750 }
3751 case ISD::VECTOR_SPLICE_LEFT:
3752 case ISD::VECTOR_SPLICE_RIGHT: {
3753 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3754 break;
3755 }
3756 case ISD::VECTOR_DEINTERLEAVE: {
3757 unsigned Factor = Node->getNumOperands();
3758 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
3759 break;
3760 SmallVector<SDValue, 8> Ops(Node->ops());
3761 EVT VecVT = Node->getValueType(ResNo: 0);
3762 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3763 // Deinterleave at Factor/2 so each result contains two factors interleaved:
3764 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
3765 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3766 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
3767 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: HalfVTs,
3768 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
3769 Results.resize(N: Factor);
3770 // Deinterleave the 2 factors out:
3771 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
3772 for (unsigned I = 0; I < Factor / 2; I++) {
3773 SDValue Deinterleave =
3774 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3775 Ops: {L.getValue(R: I), R.getValue(R: I)});
3776 Results[I] = Deinterleave.getValue(R: 0);
3777 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
3778 }
3779 break;
3780 }
3781 case ISD::VECTOR_INTERLEAVE: {
3782 unsigned Factor = Node->getNumOperands();
3783 if (Factor <= 2 || !isPowerOf2_32(Value: Factor))
3784 break;
3785 EVT VecVT = Node->getValueType(ResNo: 0);
3786 SmallVector<EVT> HalfVTs(Factor / 2, VecVT);
3787 SmallVector<SDValue, 8> LOps, ROps;
3788 // Interleave so we have 2 factors per result:
3789 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
3790 for (unsigned I = 0; I < Factor / 2; I++) {
3791 SDValue Interleave =
3792 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: {VecVT, VecVT},
3793 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
3794 LOps.push_back(Elt: Interleave.getValue(R: 0));
3795 ROps.push_back(Elt: Interleave.getValue(R: 1));
3796 }
3797 // Interleave at Factor/2:
3798 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
3799 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: LOps);
3800 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL: dl, ResultTys: HalfVTs, Ops: ROps);
3801 for (unsigned I = 0; I < Factor / 2; I++)
3802 Results.push_back(Elt: L.getValue(R: I));
3803 for (unsigned I = 0; I < Factor / 2; I++)
3804 Results.push_back(Elt: R.getValue(R: I));
3805 break;
3806 }
3807 case ISD::EXTRACT_ELEMENT: {
3808 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3809 if (Node->getConstantOperandVal(Num: 1)) {
3810 // 1 -> Hi
3811 Tmp1 = DAG.getNode(
3812 Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3813 N2: DAG.getShiftAmountConstant(Val: OpTy.getSizeInBits() / 2, VT: OpTy, DL: dl));
3814 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3815 } else {
3816 // 0 -> Lo
3817 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3818 Operand: Node->getOperand(Num: 0));
3819 }
3820 Results.push_back(Elt: Tmp1);
3821 break;
3822 }
3823 case ISD::STACKADDRESS:
3824 case ISD::STACKSAVE:
3825 // Expand to CopyFromReg if the target set
3826 // StackPointerRegisterToSaveRestore.
3827 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3828 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3829 VT: Node->getValueType(ResNo: 0)));
3830 Results.push_back(Elt: Results[0].getValue(R: 1));
3831 } else {
3832 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3833 Results.push_back(Elt: Node->getOperand(Num: 0));
3834
3835 StringRef IntrinsicName = Node->getOpcode() == ISD::STACKADDRESS
3836 ? "llvm.stackaddress"
3837 : "llvm.stacksave";
3838 DAG.getContext()->diagnose(DI: DiagnosticInfoLegalizationFailure(
3839 Twine(IntrinsicName) + " is not supported on this target.",
3840 DAG.getMachineFunction().getFunction(), dl.getDebugLoc()));
3841 }
3842 break;
3843 case ISD::STACKRESTORE:
3844 // Expand to CopyToReg if the target set
3845 // StackPointerRegisterToSaveRestore.
3846 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3847 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3848 N: Node->getOperand(Num: 1)));
3849 } else {
3850 Results.push_back(Elt: Node->getOperand(Num: 0));
3851 }
3852 break;
3853 case ISD::GET_DYNAMIC_AREA_OFFSET:
3854 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3855 Results.push_back(Elt: Results[0].getValue(R: 0));
3856 break;
3857 case ISD::FCOPYSIGN:
3858 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3859 break;
3860 case ISD::FNEG:
3861 Results.push_back(Elt: ExpandFNEG(Node));
3862 break;
3863 case ISD::FABS:
3864 Results.push_back(Elt: ExpandFABS(Node));
3865 break;
3866 case ISD::IS_FPCLASS: {
3867 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3868 if (SDValue Expanded =
3869 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3870 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3871 Results.push_back(Elt: Expanded);
3872 break;
3873 }
3874 case ISD::SMIN:
3875 case ISD::SMAX:
3876 case ISD::UMIN:
3877 case ISD::UMAX: {
3878 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3879 ISD::CondCode Pred;
3880 switch (Node->getOpcode()) {
3881 default: llvm_unreachable("How did we get here?");
3882 case ISD::SMAX: Pred = ISD::SETGT; break;
3883 case ISD::SMIN: Pred = ISD::SETLT; break;
3884 case ISD::UMAX: Pred = ISD::SETUGT; break;
3885 case ISD::UMIN: Pred = ISD::SETULT; break;
3886 }
3887 Tmp1 = Node->getOperand(Num: 0);
3888 Tmp2 = Node->getOperand(Num: 1);
3889 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3890 Results.push_back(Elt: Tmp1);
3891 break;
3892 }
3893 case ISD::FMINNUM:
3894 case ISD::FMAXNUM: {
3895 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3896 Results.push_back(Elt: Expanded);
3897 break;
3898 }
3899 case ISD::FMINIMUM:
3900 case ISD::FMAXIMUM: {
3901 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
3902 Results.push_back(Elt: Expanded);
3903 break;
3904 }
3905 case ISD::FMINIMUMNUM:
3906 case ISD::FMAXIMUMNUM: {
3907 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
3908 break;
3909 }
3910 case ISD::FSIN:
3911 case ISD::FCOS: {
3912 EVT VT = Node->getValueType(ResNo: 0);
3913 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3914 // fcos which share the same operand and both are used.
3915 if ((TLI.isOperationLegal(Op: ISD::FSINCOS, VT) ||
3916 isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls())) &&
3917 useSinCos(Node)) {
3918 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3919 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3920 if (Node->getOpcode() == ISD::FCOS)
3921 Tmp1 = Tmp1.getValue(R: 1);
3922 Results.push_back(Elt: Tmp1);
3923 }
3924 break;
3925 }
3926 case ISD::FLDEXP:
3927 case ISD::STRICT_FLDEXP: {
3928 EVT VT = Node->getValueType(ResNo: 0);
3929 RTLIB::Libcall LC = RTLIB::getLDEXP(RetVT: VT);
3930 // Use the LibCall instead, it is very likely faster
3931 // FIXME: Use separate LibCall action.
3932 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3933 break;
3934
3935 if (SDValue Expanded = expandLdexp(Node)) {
3936 Results.push_back(Elt: Expanded);
3937 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3938 Results.push_back(Elt: Expanded.getValue(R: 1));
3939 }
3940
3941 break;
3942 }
3943 case ISD::FFREXP: {
3944 RTLIB::Libcall LC = RTLIB::getFREXP(RetVT: Node->getValueType(ResNo: 0));
3945 // Use the LibCall instead, it is very likely faster
3946 // FIXME: Use separate LibCall action.
3947 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3948 break;
3949
3950 if (SDValue Expanded = expandFrexp(Node)) {
3951 Results.push_back(Elt: Expanded);
3952 Results.push_back(Elt: Expanded.getValue(R: 1));
3953 }
3954 break;
3955 }
3956 case ISD::FMODF: {
3957 RTLIB::Libcall LC = RTLIB::getMODF(VT: Node->getValueType(ResNo: 0));
3958 // Use the LibCall instead, it is very likely faster
3959 // FIXME: Use separate LibCall action.
3960 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3961 break;
3962
3963 if (SDValue Expanded = expandModf(Node)) {
3964 Results.push_back(Elt: Expanded);
3965 Results.push_back(Elt: Expanded.getValue(R: 1));
3966 }
3967 break;
3968 }
3969 case ISD::FSINCOS: {
3970 if (isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls()))
3971 break;
3972 EVT VT = Node->getValueType(ResNo: 0);
3973 SDValue Op = Node->getOperand(Num: 0);
3974 SDNodeFlags Flags = Node->getFlags();
3975 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
3976 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
3977 Results.append(IL: {Tmp1, Tmp2});
3978 break;
3979 }
3980 case ISD::FMAD:
3981 llvm_unreachable("Illegal fmad should never be formed");
3982
3983 case ISD::FP16_TO_FP:
3984 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3985 // We can extend to types bigger than f32 in two steps without changing
3986 // the result. Since "f16 -> f32" is much more commonly available, give
3987 // CodeGen the option of emitting that before resorting to a libcall.
3988 SDValue Res =
3989 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
3990 Results.push_back(
3991 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
3992 }
3993 break;
3994 case ISD::STRICT_BF16_TO_FP:
3995 case ISD::STRICT_FP16_TO_FP:
3996 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3997 // We can extend to types bigger than f32 in two steps without changing
3998 // the result. Since "f16 -> f32" is much more commonly available, give
3999 // CodeGen the option of emitting that before resorting to a libcall.
4000 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
4001 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
4002 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
4003 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
4004 Ops: {Res.getValue(R: 1), Res});
4005 Results.push_back(Elt: Res);
4006 Results.push_back(Elt: Res.getValue(R: 1));
4007 }
4008 break;
4009 case ISD::FP_TO_FP16:
4010 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
4011 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
4012 SDValue Op = Node->getOperand(Num: 0);
4013 MVT SVT = Op.getSimpleValueType();
4014 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
4015 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
4016 // Under fastmath, we can expand this node into a fround followed by
4017 // a float-half conversion.
4018 SDValue FloatVal =
4019 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
4020 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
4021 Results.push_back(
4022 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
4023 }
4024 }
4025 break;
4026 case ISD::ConstantFP: {
4027 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
4028 // Check to see if this FP immediate is already legal.
4029 // If this is a legal constant, turn it into a TargetConstantFP node.
4030 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(ResNo: 0),
4031 ForCodeSize: DAG.shouldOptForSize()))
4032 Results.push_back(Elt: ExpandConstantFP(CFP, UseCP: true));
4033 break;
4034 }
4035 case ISD::Constant: {
4036 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
4037 Results.push_back(Elt: ExpandConstant(CP));
4038 break;
4039 }
4040 case ISD::FSUB: {
4041 EVT VT = Node->getValueType(ResNo: 0);
4042 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
4043 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
4044 const SDNodeFlags Flags = Node->getFlags();
4045 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
4046 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
4047 Results.push_back(Elt: Tmp1);
4048 }
4049 break;
4050 }
4051 case ISD::SUB: {
4052 EVT VT = Node->getValueType(ResNo: 0);
4053 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
4054 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
4055 "Don't know how to expand this subtraction!");
4056 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
4057 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
4058 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
4059 break;
4060 }
4061 case ISD::UREM:
4062 case ISD::SREM:
4063 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
4064 Results.push_back(Elt: Tmp1);
4065 break;
4066 case ISD::UDIV:
4067 case ISD::SDIV: {
4068 bool isSigned = Node->getOpcode() == ISD::SDIV;
4069 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4070 EVT VT = Node->getValueType(ResNo: 0);
4071 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
4072 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4073 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4074 N2: Node->getOperand(Num: 1));
4075 Results.push_back(Elt: Tmp1);
4076 }
4077 break;
4078 }
4079 case ISD::MULHU:
4080 case ISD::MULHS: {
4081 unsigned ExpandOpcode =
4082 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
4083 EVT VT = Node->getValueType(ResNo: 0);
4084 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4085
4086 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4087 N2: Node->getOperand(Num: 1));
4088 Results.push_back(Elt: Tmp1.getValue(R: 1));
4089 break;
4090 }
4091 case ISD::UMUL_LOHI:
4092 case ISD::SMUL_LOHI: {
4093 SDValue LHS = Node->getOperand(Num: 0);
4094 SDValue RHS = Node->getOperand(Num: 1);
4095 EVT VT = LHS.getValueType();
4096 unsigned MULHOpcode =
4097 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
4098
4099 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
4100 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
4101 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
4102 break;
4103 }
4104
4105 SmallVector<SDValue, 4> Halves;
4106 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4107 assert(TLI.isTypeLegal(HalfType));
4108 if (TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
4109 HiLoVT: HalfType, DAG,
4110 Kind: TargetLowering::MulExpansionKind::Always)) {
4111 for (unsigned i = 0; i < 2; ++i) {
4112 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
4113 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
4114 SDValue Shift =
4115 DAG.getShiftAmountConstant(Val: HalfType.getScalarSizeInBits(), VT, DL: dl);
4116 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4117 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4118 }
4119 break;
4120 }
4121 break;
4122 }
4123 case ISD::MUL: {
4124 EVT VT = Node->getValueType(ResNo: 0);
4125 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4126 // See if multiply or divide can be lowered using two-result operations.
4127 // We just need the low half of the multiply; try both the signed
4128 // and unsigned forms. If the target supports both SMUL_LOHI and
4129 // UMUL_LOHI, form a preference by checking which forms of plain
4130 // MULH it supports.
4131 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
4132 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
4133 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
4134 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
4135 unsigned OpToUse = 0;
4136 if (HasSMUL_LOHI && !HasMULHS) {
4137 OpToUse = ISD::SMUL_LOHI;
4138 } else if (HasUMUL_LOHI && !HasMULHU) {
4139 OpToUse = ISD::UMUL_LOHI;
4140 } else if (HasSMUL_LOHI) {
4141 OpToUse = ISD::SMUL_LOHI;
4142 } else if (HasUMUL_LOHI) {
4143 OpToUse = ISD::UMUL_LOHI;
4144 }
4145 if (OpToUse) {
4146 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4147 N2: Node->getOperand(Num: 1)));
4148 break;
4149 }
4150
4151 SDValue Lo, Hi;
4152 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4153 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
4154 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
4155 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
4156 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
4157 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
4158 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4159 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
4160 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
4161 SDValue Shift =
4162 DAG.getShiftAmountConstant(Val: HalfType.getSizeInBits(), VT, DL: dl);
4163 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4164 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4165 }
4166 break;
4167 }
4168 case ISD::FSHL:
4169 case ISD::FSHR:
4170 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
4171 Results.push_back(Elt: Expanded);
4172 break;
4173 case ISD::ROTL:
4174 case ISD::ROTR:
4175 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
4176 Results.push_back(Elt: Expanded);
4177 break;
4178 case ISD::CLMUL:
4179 case ISD::CLMULR:
4180 case ISD::CLMULH:
4181 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG))
4182 Results.push_back(Elt: Expanded);
4183 break;
4184 case ISD::SADDSAT:
4185 case ISD::UADDSAT:
4186 case ISD::SSUBSAT:
4187 case ISD::USUBSAT:
4188 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
4189 break;
4190 case ISD::SCMP:
4191 case ISD::UCMP:
4192 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
4193 break;
4194 case ISD::SSHLSAT:
4195 case ISD::USHLSAT:
4196 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
4197 break;
4198 case ISD::SMULFIX:
4199 case ISD::SMULFIXSAT:
4200 case ISD::UMULFIX:
4201 case ISD::UMULFIXSAT:
4202 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
4203 break;
4204 case ISD::SDIVFIX:
4205 case ISD::SDIVFIXSAT:
4206 case ISD::UDIVFIX:
4207 case ISD::UDIVFIXSAT:
4208 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
4209 LHS: Node->getOperand(Num: 0),
4210 RHS: Node->getOperand(Num: 1),
4211 Scale: Node->getConstantOperandVal(Num: 2),
4212 DAG)) {
4213 Results.push_back(Elt: V);
4214 break;
4215 }
4216 // FIXME: We might want to retry here with a wider type if we fail, if that
4217 // type is legal.
4218 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4219 // <= 128 (which is the case for all of the default Embedded-C types),
4220 // we will only get here with types and scales that we could always expand
4221 // if we were allowed to generate libcalls to division functions of illegal
4222 // type. But we cannot do that.
4223 llvm_unreachable("Cannot expand DIVFIX!");
4224 case ISD::UADDO_CARRY:
4225 case ISD::USUBO_CARRY: {
4226 SDValue LHS = Node->getOperand(Num: 0);
4227 SDValue RHS = Node->getOperand(Num: 1);
4228 SDValue Carry = Node->getOperand(Num: 2);
4229
4230 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4231
4232 // Initial add of the 2 operands.
4233 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4234 EVT VT = LHS.getValueType();
4235 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4236
4237 // Initial check for overflow.
4238 EVT CarryType = Node->getValueType(ResNo: 1);
4239 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4240 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4241 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4242
4243 // Add of the sum and the carry.
4244 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4245 SDValue CarryExt =
4246 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4247 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4248
4249 // Second check for overflow. If we are adding, we can only overflow if the
4250 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4251 // If we are subtracting, we can only overflow if the initial sum is 0 and
4252 // the carry is set, resulting in a new sum of all 1s.
4253 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4254 SDValue Overflow2 =
4255 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4256 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4257 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4258 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4259
4260 SDValue ResultCarry =
4261 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4262
4263 Results.push_back(Elt: Sum2);
4264 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4265 break;
4266 }
4267 case ISD::SADDO:
4268 case ISD::SSUBO: {
4269 SDValue Result, Overflow;
4270 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4271 Results.push_back(Elt: Result);
4272 Results.push_back(Elt: Overflow);
4273 break;
4274 }
4275 case ISD::UADDO:
4276 case ISD::USUBO: {
4277 SDValue Result, Overflow;
4278 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4279 Results.push_back(Elt: Result);
4280 Results.push_back(Elt: Overflow);
4281 break;
4282 }
4283 case ISD::UMULO:
4284 case ISD::SMULO: {
4285 SDValue Result, Overflow;
4286 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4287 Results.push_back(Elt: Result);
4288 Results.push_back(Elt: Overflow);
4289 }
4290 break;
4291 }
4292 case ISD::BUILD_PAIR: {
4293 EVT PairTy = Node->getValueType(ResNo: 0);
4294 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4295 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4296 Tmp2 = DAG.getNode(
4297 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4298 N2: DAG.getShiftAmountConstant(Val: PairTy.getSizeInBits() / 2, VT: PairTy, DL: dl));
4299 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4300 break;
4301 }
4302 case ISD::SELECT:
4303 Tmp1 = Node->getOperand(Num: 0);
4304 Tmp2 = Node->getOperand(Num: 1);
4305 Tmp3 = Node->getOperand(Num: 2);
4306 if (Tmp1.getOpcode() == ISD::SETCC) {
4307 Tmp1 = DAG.getSelectCC(
4308 DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), True: Tmp2, False: Tmp3,
4309 Cond: cast<CondCodeSDNode>(Val: Tmp1.getOperand(i: 2))->get(), Flags: Node->getFlags());
4310 } else {
4311 Tmp1 =
4312 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4313 True: Tmp2, False: Tmp3, Cond: ISD::SETNE, Flags: Node->getFlags());
4314 }
4315 Results.push_back(Elt: Tmp1);
4316 break;
4317 case ISD::BR_JT: {
4318 SDValue Chain = Node->getOperand(Num: 0);
4319 SDValue Table = Node->getOperand(Num: 1);
4320 SDValue Index = Node->getOperand(Num: 2);
4321 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4322
4323 const DataLayout &TD = DAG.getDataLayout();
4324 EVT PTy = TLI.getPointerTy(DL: TD);
4325
4326 unsigned EntrySize =
4327 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4328
4329 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4330 // This transformation needs to be done here since otherwise the MIPS
4331 // backend will end up emitting a three instruction multiply sequence
4332 // instead of a single shift and MSP430 will call a runtime function.
4333 if (llvm::isPowerOf2_32(Value: EntrySize))
4334 Index = DAG.getNode(
4335 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4336 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4337 else
4338 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4339 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4340 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4341
4342 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4343 SDValue LD = DAG.getExtLoad(
4344 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4345 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4346 Addr = LD;
4347 if (TLI.isJumpTableRelative()) {
4348 // For PIC, the sequence is:
4349 // BRIND(RelocBase + load(Jumptable + index))
4350 // RelocBase can be JumpTable, GOT or some sort of global base.
4351 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4352 Offset: Addr, DL: dl);
4353 }
4354
4355 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4356 Results.push_back(Elt: Tmp1);
4357 break;
4358 }
4359 case ISD::BRCOND:
4360 // Expand brcond's setcc into its constituent parts and create a BR_CC
4361 // Node.
4362 Tmp1 = Node->getOperand(Num: 0);
4363 Tmp2 = Node->getOperand(Num: 1);
4364 if (Tmp2.getOpcode() == ISD::SETCC &&
4365 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4366 VT: Tmp2.getOperand(i: 0).getValueType())) {
4367 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4368 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4369 N5: Node->getOperand(Num: 2));
4370 } else {
4371 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4372 if (Tmp2.isUndef() ||
4373 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4374 Tmp3 = Tmp2;
4375 else
4376 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4377 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4378 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4379 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4380 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4381 N5: Node->getOperand(Num: 2));
4382 }
4383 Results.push_back(Elt: Tmp1);
4384 break;
4385 case ISD::SETCC:
4386 case ISD::VP_SETCC:
4387 case ISD::STRICT_FSETCC:
4388 case ISD::STRICT_FSETCCS: {
4389 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4390 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4391 Node->getOpcode() == ISD::STRICT_FSETCCS;
4392 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4393 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4394 unsigned Offset = IsStrict ? 1 : 0;
4395 Tmp1 = Node->getOperand(Num: 0 + Offset);
4396 Tmp2 = Node->getOperand(Num: 1 + Offset);
4397 Tmp3 = Node->getOperand(Num: 2 + Offset);
4398 SDValue Mask, EVL;
4399 if (IsVP) {
4400 Mask = Node->getOperand(Num: 3 + Offset);
4401 EVL = Node->getOperand(Num: 4 + Offset);
4402 }
4403 bool Legalized = TLI.LegalizeSetCCCondCode(
4404 DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3, Mask, EVL, NeedInvert, dl,
4405 Chain, IsSignaling);
4406
4407 if (Legalized) {
4408 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4409 // condition code, create a new SETCC node.
4410 if (Tmp3.getNode()) {
4411 if (IsStrict) {
4412 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4413 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4414 Chain = Tmp1.getValue(R: 1);
4415 } else if (IsVP) {
4416 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0),
4417 Ops: {Tmp1, Tmp2, Tmp3, Mask, EVL}, Flags: Node->getFlags());
4418 } else {
4419 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4420 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4421 }
4422 }
4423
4424 // If we expanded the SETCC by inverting the condition code, then wrap
4425 // the existing SETCC in a NOT to restore the intended condition.
4426 if (NeedInvert) {
4427 if (!IsVP)
4428 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4429 else
4430 Tmp1 =
4431 DAG.getVPLogicalNOT(DL: dl, Val: Tmp1, Mask, EVL, VT: Tmp1->getValueType(ResNo: 0));
4432 }
4433
4434 Results.push_back(Elt: Tmp1);
4435 if (IsStrict)
4436 Results.push_back(Elt: Chain);
4437
4438 break;
4439 }
4440
4441 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4442 // understand if this code is useful for strict nodes.
4443 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4444
4445 // Otherwise, SETCC for the given comparison type must be completely
4446 // illegal; expand it into a SELECT_CC.
4447 // FIXME: This drops the mask/evl for VP_SETCC.
4448 EVT VT = Node->getValueType(ResNo: 0);
4449 EVT Tmp1VT = Tmp1.getValueType();
4450 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4451 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4452 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3,
4453 Flags: Node->getFlags());
4454 Results.push_back(Elt: Tmp1);
4455 break;
4456 }
4457 case ISD::SELECT_CC: {
4458 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4459 Tmp1 = Node->getOperand(Num: 0); // LHS
4460 Tmp2 = Node->getOperand(Num: 1); // RHS
4461 Tmp3 = Node->getOperand(Num: 2); // True
4462 Tmp4 = Node->getOperand(Num: 3); // False
4463 EVT VT = Node->getValueType(ResNo: 0);
4464 SDValue Chain;
4465 SDValue CC = Node->getOperand(Num: 4);
4466 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4467
4468 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4469 // If the condition code is legal, then we need to expand this
4470 // node using SETCC and SELECT.
4471 EVT CmpVT = Tmp1.getValueType();
4472 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4473 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4474 "expanded.");
4475 EVT CCVT = getSetCCResultType(VT: CmpVT);
4476 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4477 Results.push_back(
4478 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4479 break;
4480 }
4481
4482 // SELECT_CC is legal, so the condition code must not be.
4483 bool Legalized = false;
4484 // Try to legalize by inverting the condition. This is for targets that
4485 // might support an ordered version of a condition, but not the unordered
4486 // version (or vice versa).
4487 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4488 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4489 // Use the new condition code and swap true and false
4490 Legalized = true;
4491 Tmp1 =
4492 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC, Flags: Node->getFlags());
4493 } else {
4494 // If The inverse is not legal, then try to swap the arguments using
4495 // the inverse condition code.
4496 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4497 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4498 // The swapped inverse condition is legal, so swap true and false,
4499 // lhs and rhs.
4500 Legalized = true;
4501 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC,
4502 Flags: Node->getFlags());
4503 }
4504 }
4505
4506 if (!Legalized) {
4507 Legalized = TLI.LegalizeSetCCCondCode(
4508 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4509 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4510
4511 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4512
4513 // If we expanded the SETCC by inverting the condition code, then swap
4514 // the True/False operands to match.
4515 if (NeedInvert)
4516 std::swap(a&: Tmp3, b&: Tmp4);
4517
4518 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4519 // condition code, create a new SELECT_CC node.
4520 if (CC.getNode()) {
4521 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4522 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4523 } else {
4524 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4525 CC = DAG.getCondCode(Cond: ISD::SETNE);
4526 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4527 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4528 }
4529 }
4530 Results.push_back(Elt: Tmp1);
4531 break;
4532 }
4533 case ISD::BR_CC: {
4534 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4535 SDValue Chain;
4536 Tmp1 = Node->getOperand(Num: 0); // Chain
4537 Tmp2 = Node->getOperand(Num: 2); // LHS
4538 Tmp3 = Node->getOperand(Num: 3); // RHS
4539 Tmp4 = Node->getOperand(Num: 1); // CC
4540
4541 bool Legalized = TLI.LegalizeSetCCCondCode(
4542 DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()), LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4,
4543 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4544 (void)Legalized;
4545 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4546
4547 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4548 // node.
4549 if (Tmp4.getNode()) {
4550 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4551
4552 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4553 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4554 } else {
4555 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4556 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4557 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4558 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4559 }
4560 Results.push_back(Elt: Tmp1);
4561 break;
4562 }
4563 case ISD::BUILD_VECTOR:
4564 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4565 break;
4566 case ISD::SPLAT_VECTOR:
4567 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4568 break;
4569 case ISD::SRA:
4570 case ISD::SRL:
4571 case ISD::SHL: {
4572 // Scalarize vector SRA/SRL/SHL.
4573 EVT VT = Node->getValueType(ResNo: 0);
4574 assert(VT.isVector() && "Unable to legalize non-vector shift");
4575 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4576 unsigned NumElem = VT.getVectorNumElements();
4577
4578 SmallVector<SDValue, 8> Scalars;
4579 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4580 SDValue Ex =
4581 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4582 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4583 SDValue Sh =
4584 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4585 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4586 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4587 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4588 }
4589
4590 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4591 Results.push_back(Elt: Result);
4592 break;
4593 }
4594 case ISD::VECREDUCE_FADD:
4595 case ISD::VECREDUCE_FMUL:
4596 case ISD::VECREDUCE_ADD:
4597 case ISD::VECREDUCE_MUL:
4598 case ISD::VECREDUCE_AND:
4599 case ISD::VECREDUCE_OR:
4600 case ISD::VECREDUCE_XOR:
4601 case ISD::VECREDUCE_SMAX:
4602 case ISD::VECREDUCE_SMIN:
4603 case ISD::VECREDUCE_UMAX:
4604 case ISD::VECREDUCE_UMIN:
4605 case ISD::VECREDUCE_FMAX:
4606 case ISD::VECREDUCE_FMIN:
4607 case ISD::VECREDUCE_FMAXIMUM:
4608 case ISD::VECREDUCE_FMINIMUM:
4609 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4610 break;
4611 case ISD::VP_CTTZ_ELTS:
4612 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4613 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4614 break;
4615 case ISD::CLEAR_CACHE:
4616 // The default expansion of llvm.clear_cache is simply a no-op for those
4617 // targets where it is not needed.
4618 Results.push_back(Elt: Node->getOperand(Num: 0));
4619 break;
4620 case ISD::LRINT:
4621 case ISD::LLRINT: {
4622 SDValue Arg = Node->getOperand(Num: 0);
4623 EVT ArgVT = Arg.getValueType();
4624 EVT ResVT = Node->getValueType(ResNo: 0);
4625 SDLoc dl(Node);
4626 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL: dl, VT: ArgVT, Operand: Arg);
4627 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
4628 break;
4629 }
4630 case ISD::ADDRSPACECAST:
4631 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4632 break;
4633 case ISD::GLOBAL_OFFSET_TABLE:
4634 case ISD::GlobalAddress:
4635 case ISD::GlobalTLSAddress:
4636 case ISD::ExternalSymbol:
4637 case ISD::ConstantPool:
4638 case ISD::JumpTable:
4639 case ISD::INTRINSIC_W_CHAIN:
4640 case ISD::INTRINSIC_WO_CHAIN:
4641 case ISD::INTRINSIC_VOID:
4642 // FIXME: Custom lowering for these operations shouldn't return null!
4643 // Return true so that we don't call ConvertNodeToLibcall which also won't
4644 // do anything.
4645 return true;
4646 }
4647
4648 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4649 // FIXME: We were asked to expand a strict floating-point operation,
4650 // but there is currently no expansion implemented that would preserve
4651 // the "strict" properties. For now, we just fall back to the non-strict
4652 // version if that is legal on the target. The actual mutation of the
4653 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4654 switch (Node->getOpcode()) {
4655 default:
4656 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4657 VT: Node->getValueType(ResNo: 0))
4658 == TargetLowering::Legal)
4659 return true;
4660 break;
4661 case ISD::STRICT_FSUB: {
4662 if (TLI.getStrictFPOperationAction(
4663 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4664 return true;
4665 if (TLI.getStrictFPOperationAction(
4666 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4667 break;
4668
4669 EVT VT = Node->getValueType(ResNo: 0);
4670 const SDNodeFlags Flags = Node->getFlags();
4671 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4672 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4673 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4674 Flags);
4675
4676 Results.push_back(Elt: Fadd);
4677 Results.push_back(Elt: Fadd.getValue(R: 1));
4678 break;
4679 }
4680 case ISD::STRICT_SINT_TO_FP:
4681 case ISD::STRICT_UINT_TO_FP:
4682 case ISD::STRICT_LRINT:
4683 case ISD::STRICT_LLRINT:
4684 case ISD::STRICT_LROUND:
4685 case ISD::STRICT_LLROUND:
4686 // These are registered by the operand type instead of the value
4687 // type. Reflect that here.
4688 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4689 VT: Node->getOperand(Num: 1).getValueType())
4690 == TargetLowering::Legal)
4691 return true;
4692 break;
4693 }
4694 }
4695
4696 // Replace the original node with the legalized result.
4697 if (Results.empty()) {
4698 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4699 return false;
4700 }
4701
4702 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4703 ReplaceNode(Old: Node, New: Results.data());
4704 return true;
4705}
4706
4707/// Return if we can use the FAST_* variant of a math libcall for the node.
4708/// FIXME: This is just guessing, we probably should have unique specific sets
4709/// flags required per libcall.
4710static bool canUseFastMathLibcall(const SDNode *Node) {
4711 // FIXME: Probably should define fast to respect nan/inf and only be
4712 // approximate functions.
4713
4714 SDNodeFlags Flags = Node->getFlags();
4715 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4716 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4717}
4718
4719void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4720 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4721 SmallVector<SDValue, 8> Results;
4722 SDLoc dl(Node);
4723 TargetLowering::MakeLibCallOptions CallOptions;
4724 CallOptions.IsPostTypeLegalization = true;
4725 // FIXME: Check flags on the node to see if we can use a finite call.
4726 unsigned Opc = Node->getOpcode();
4727 switch (Opc) {
4728 case ISD::ATOMIC_FENCE: {
4729 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4730 // FIXME: handle "fence singlethread" more efficiently.
4731 TargetLowering::ArgListTy Args;
4732
4733 TargetLowering::CallLoweringInfo CLI(DAG);
4734 CLI.setDebugLoc(dl)
4735 .setChain(Node->getOperand(Num: 0))
4736 .setLibCallee(
4737 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4738 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4739 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4740 ArgsList: std::move(Args));
4741
4742 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4743
4744 Results.push_back(Elt: CallResult.second);
4745 break;
4746 }
4747 // By default, atomic intrinsics are marked Legal and lowered. Targets
4748 // which don't support them directly, however, may want libcalls, in which
4749 // case they mark them Expand, and we get here.
4750 case ISD::ATOMIC_SWAP:
4751 case ISD::ATOMIC_LOAD_ADD:
4752 case ISD::ATOMIC_LOAD_SUB:
4753 case ISD::ATOMIC_LOAD_AND:
4754 case ISD::ATOMIC_LOAD_CLR:
4755 case ISD::ATOMIC_LOAD_OR:
4756 case ISD::ATOMIC_LOAD_XOR:
4757 case ISD::ATOMIC_LOAD_NAND:
4758 case ISD::ATOMIC_LOAD_MIN:
4759 case ISD::ATOMIC_LOAD_MAX:
4760 case ISD::ATOMIC_LOAD_UMIN:
4761 case ISD::ATOMIC_LOAD_UMAX:
4762 case ISD::ATOMIC_CMP_SWAP: {
4763 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4764 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4765 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4766 EVT RetVT = Node->getValueType(ResNo: 0);
4767 SmallVector<SDValue, 4> Ops;
4768 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported) {
4769 // If outline atomic available, prepare its arguments and expand.
4770 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4771 Ops.push_back(Elt: Node->getOperand(Num: 1));
4772
4773 } else {
4774 LC = RTLIB::getSYNC(Opc, VT);
4775 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4776 "Unexpected atomic op or value type!");
4777 // Arguments for expansion to sync libcall
4778 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4779 }
4780 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4781 Ops, CallOptions,
4782 dl: SDLoc(Node),
4783 Chain: Node->getOperand(Num: 0));
4784 Results.push_back(Elt: Tmp.first);
4785 Results.push_back(Elt: Tmp.second);
4786 break;
4787 }
4788 case ISD::TRAP: {
4789 // If this operation is not supported, lower it to 'abort()' call
4790 TargetLowering::ArgListTy Args;
4791 TargetLowering::CallLoweringInfo CLI(DAG);
4792 CLI.setDebugLoc(dl)
4793 .setChain(Node->getOperand(Num: 0))
4794 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4795 Target: DAG.getExternalSymbol(
4796 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4797 ArgsList: std::move(Args));
4798 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4799
4800 Results.push_back(Elt: CallResult.second);
4801 break;
4802 }
4803 case ISD::CLEAR_CACHE: {
4804 SDValue InputChain = Node->getOperand(Num: 0);
4805 SDValue StartVal = Node->getOperand(Num: 1);
4806 SDValue EndVal = Node->getOperand(Num: 2);
4807 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4808 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
4809 dl: SDLoc(Node), Chain: InputChain);
4810 Results.push_back(Elt: Tmp.second);
4811 break;
4812 }
4813 case ISD::FMINNUM:
4814 case ISD::STRICT_FMINNUM:
4815 ExpandFPLibCall(Node, Call_F32: RTLIB::FMIN_F32, Call_F64: RTLIB::FMIN_F64,
4816 Call_F80: RTLIB::FMIN_F80, Call_F128: RTLIB::FMIN_F128,
4817 Call_PPCF128: RTLIB::FMIN_PPCF128, Results);
4818 break;
4819 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4820 // libcall legalization for these nodes, but there is no default expasion for
4821 // these nodes either (see PR63267 for example).
4822 case ISD::FMAXNUM:
4823 case ISD::STRICT_FMAXNUM:
4824 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAX_F32, Call_F64: RTLIB::FMAX_F64,
4825 Call_F80: RTLIB::FMAX_F80, Call_F128: RTLIB::FMAX_F128,
4826 Call_PPCF128: RTLIB::FMAX_PPCF128, Results);
4827 break;
4828 case ISD::FMINIMUMNUM:
4829 ExpandFPLibCall(Node, Call_F32: RTLIB::FMINIMUM_NUM_F32, Call_F64: RTLIB::FMINIMUM_NUM_F64,
4830 Call_F80: RTLIB::FMINIMUM_NUM_F80, Call_F128: RTLIB::FMINIMUM_NUM_F128,
4831 Call_PPCF128: RTLIB::FMINIMUM_NUM_PPCF128, Results);
4832 break;
4833 case ISD::FMAXIMUMNUM:
4834 ExpandFPLibCall(Node, Call_F32: RTLIB::FMAXIMUM_NUM_F32, Call_F64: RTLIB::FMAXIMUM_NUM_F64,
4835 Call_F80: RTLIB::FMAXIMUM_NUM_F80, Call_F128: RTLIB::FMAXIMUM_NUM_F128,
4836 Call_PPCF128: RTLIB::FMAXIMUM_NUM_PPCF128, Results);
4837 break;
4838 case ISD::FSQRT:
4839 case ISD::STRICT_FSQRT: {
4840 // FIXME: Probably should define fast to respect nan/inf and only be
4841 // approximate functions.
4842 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
4843 Call_F32: {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
4844 Call_F64: {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
4845 Call_F80: {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
4846 Call_F128: {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
4847 Call_PPCF128: {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
4848 Results);
4849 break;
4850 }
4851 case ISD::FCBRT:
4852 ExpandFPLibCall(Node, Call_F32: RTLIB::CBRT_F32, Call_F64: RTLIB::CBRT_F64,
4853 Call_F80: RTLIB::CBRT_F80, Call_F128: RTLIB::CBRT_F128,
4854 Call_PPCF128: RTLIB::CBRT_PPCF128, Results);
4855 break;
4856 case ISD::FSIN:
4857 case ISD::STRICT_FSIN:
4858 ExpandFPLibCall(Node, Call_F32: RTLIB::SIN_F32, Call_F64: RTLIB::SIN_F64,
4859 Call_F80: RTLIB::SIN_F80, Call_F128: RTLIB::SIN_F128,
4860 Call_PPCF128: RTLIB::SIN_PPCF128, Results);
4861 break;
4862 case ISD::FCOS:
4863 case ISD::STRICT_FCOS:
4864 ExpandFPLibCall(Node, Call_F32: RTLIB::COS_F32, Call_F64: RTLIB::COS_F64,
4865 Call_F80: RTLIB::COS_F80, Call_F128: RTLIB::COS_F128,
4866 Call_PPCF128: RTLIB::COS_PPCF128, Results);
4867 break;
4868 case ISD::FTAN:
4869 case ISD::STRICT_FTAN:
4870 ExpandFPLibCall(Node, Call_F32: RTLIB::TAN_F32, Call_F64: RTLIB::TAN_F64, Call_F80: RTLIB::TAN_F80,
4871 Call_F128: RTLIB::TAN_F128, Call_PPCF128: RTLIB::TAN_PPCF128, Results);
4872 break;
4873 case ISD::FASIN:
4874 case ISD::STRICT_FASIN:
4875 ExpandFPLibCall(Node, Call_F32: RTLIB::ASIN_F32, Call_F64: RTLIB::ASIN_F64, Call_F80: RTLIB::ASIN_F80,
4876 Call_F128: RTLIB::ASIN_F128, Call_PPCF128: RTLIB::ASIN_PPCF128, Results);
4877 break;
4878 case ISD::FACOS:
4879 case ISD::STRICT_FACOS:
4880 ExpandFPLibCall(Node, Call_F32: RTLIB::ACOS_F32, Call_F64: RTLIB::ACOS_F64, Call_F80: RTLIB::ACOS_F80,
4881 Call_F128: RTLIB::ACOS_F128, Call_PPCF128: RTLIB::ACOS_PPCF128, Results);
4882 break;
4883 case ISD::FATAN:
4884 case ISD::STRICT_FATAN:
4885 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN_F32, Call_F64: RTLIB::ATAN_F64, Call_F80: RTLIB::ATAN_F80,
4886 Call_F128: RTLIB::ATAN_F128, Call_PPCF128: RTLIB::ATAN_PPCF128, Results);
4887 break;
4888 case ISD::FATAN2:
4889 case ISD::STRICT_FATAN2:
4890 ExpandFPLibCall(Node, Call_F32: RTLIB::ATAN2_F32, Call_F64: RTLIB::ATAN2_F64, Call_F80: RTLIB::ATAN2_F80,
4891 Call_F128: RTLIB::ATAN2_F128, Call_PPCF128: RTLIB::ATAN2_PPCF128, Results);
4892 break;
4893 case ISD::FSINH:
4894 case ISD::STRICT_FSINH:
4895 ExpandFPLibCall(Node, Call_F32: RTLIB::SINH_F32, Call_F64: RTLIB::SINH_F64, Call_F80: RTLIB::SINH_F80,
4896 Call_F128: RTLIB::SINH_F128, Call_PPCF128: RTLIB::SINH_PPCF128, Results);
4897 break;
4898 case ISD::FCOSH:
4899 case ISD::STRICT_FCOSH:
4900 ExpandFPLibCall(Node, Call_F32: RTLIB::COSH_F32, Call_F64: RTLIB::COSH_F64, Call_F80: RTLIB::COSH_F80,
4901 Call_F128: RTLIB::COSH_F128, Call_PPCF128: RTLIB::COSH_PPCF128, Results);
4902 break;
4903 case ISD::FTANH:
4904 case ISD::STRICT_FTANH:
4905 ExpandFPLibCall(Node, Call_F32: RTLIB::TANH_F32, Call_F64: RTLIB::TANH_F64, Call_F80: RTLIB::TANH_F80,
4906 Call_F128: RTLIB::TANH_F128, Call_PPCF128: RTLIB::TANH_PPCF128, Results);
4907 break;
4908 case ISD::FSINCOS:
4909 case ISD::FSINCOSPI: {
4910 EVT VT = Node->getValueType(ResNo: 0);
4911
4912 if (Node->getOpcode() == ISD::FSINCOS) {
4913 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(RetVT: VT);
4914 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
4915 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
4916 Results.push_back(Elt: Expanded);
4917 Results.push_back(Elt: Expanded.getValue(R: 1));
4918 break;
4919 }
4920 }
4921 }
4922
4923 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4924 ? RTLIB::getSINCOS(RetVT: VT)
4925 : RTLIB::getSINCOSPI(RetVT: VT);
4926 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);
4927 if (!Expanded) {
4928 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
4929 Node->getOperationName(G: &DAG));
4930 SDValue Poison = DAG.getPOISON(VT);
4931 Results.push_back(Elt: Poison);
4932 Results.push_back(Elt: Poison);
4933 }
4934
4935 break;
4936 }
4937 case ISD::FLOG:
4938 case ISD::STRICT_FLOG:
4939 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG_F32, Call_F64: RTLIB::LOG_F64, Call_F80: RTLIB::LOG_F80,
4940 Call_F128: RTLIB::LOG_F128, Call_PPCF128: RTLIB::LOG_PPCF128, Results);
4941 break;
4942 case ISD::FLOG2:
4943 case ISD::STRICT_FLOG2:
4944 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG2_F32, Call_F64: RTLIB::LOG2_F64, Call_F80: RTLIB::LOG2_F80,
4945 Call_F128: RTLIB::LOG2_F128, Call_PPCF128: RTLIB::LOG2_PPCF128, Results);
4946 break;
4947 case ISD::FLOG10:
4948 case ISD::STRICT_FLOG10:
4949 ExpandFPLibCall(Node, Call_F32: RTLIB::LOG10_F32, Call_F64: RTLIB::LOG10_F64, Call_F80: RTLIB::LOG10_F80,
4950 Call_F128: RTLIB::LOG10_F128, Call_PPCF128: RTLIB::LOG10_PPCF128, Results);
4951 break;
4952 case ISD::FEXP:
4953 case ISD::STRICT_FEXP:
4954 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP_F32, Call_F64: RTLIB::EXP_F64, Call_F80: RTLIB::EXP_F80,
4955 Call_F128: RTLIB::EXP_F128, Call_PPCF128: RTLIB::EXP_PPCF128, Results);
4956 break;
4957 case ISD::FEXP2:
4958 case ISD::STRICT_FEXP2:
4959 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP2_F32, Call_F64: RTLIB::EXP2_F64, Call_F80: RTLIB::EXP2_F80,
4960 Call_F128: RTLIB::EXP2_F128, Call_PPCF128: RTLIB::EXP2_PPCF128, Results);
4961 break;
4962 case ISD::FEXP10:
4963 ExpandFPLibCall(Node, Call_F32: RTLIB::EXP10_F32, Call_F64: RTLIB::EXP10_F64, Call_F80: RTLIB::EXP10_F80,
4964 Call_F128: RTLIB::EXP10_F128, Call_PPCF128: RTLIB::EXP10_PPCF128, Results);
4965 break;
4966 case ISD::FTRUNC:
4967 case ISD::STRICT_FTRUNC:
4968 ExpandFPLibCall(Node, Call_F32: RTLIB::TRUNC_F32, Call_F64: RTLIB::TRUNC_F64,
4969 Call_F80: RTLIB::TRUNC_F80, Call_F128: RTLIB::TRUNC_F128,
4970 Call_PPCF128: RTLIB::TRUNC_PPCF128, Results);
4971 break;
4972 case ISD::FFLOOR:
4973 case ISD::STRICT_FFLOOR:
4974 ExpandFPLibCall(Node, Call_F32: RTLIB::FLOOR_F32, Call_F64: RTLIB::FLOOR_F64,
4975 Call_F80: RTLIB::FLOOR_F80, Call_F128: RTLIB::FLOOR_F128,
4976 Call_PPCF128: RTLIB::FLOOR_PPCF128, Results);
4977 break;
4978 case ISD::FCEIL:
4979 case ISD::STRICT_FCEIL:
4980 ExpandFPLibCall(Node, Call_F32: RTLIB::CEIL_F32, Call_F64: RTLIB::CEIL_F64,
4981 Call_F80: RTLIB::CEIL_F80, Call_F128: RTLIB::CEIL_F128,
4982 Call_PPCF128: RTLIB::CEIL_PPCF128, Results);
4983 break;
4984 case ISD::FRINT:
4985 case ISD::STRICT_FRINT:
4986 ExpandFPLibCall(Node, Call_F32: RTLIB::RINT_F32, Call_F64: RTLIB::RINT_F64,
4987 Call_F80: RTLIB::RINT_F80, Call_F128: RTLIB::RINT_F128,
4988 Call_PPCF128: RTLIB::RINT_PPCF128, Results);
4989 break;
4990 case ISD::FNEARBYINT:
4991 case ISD::STRICT_FNEARBYINT:
4992 ExpandFPLibCall(Node, Call_F32: RTLIB::NEARBYINT_F32,
4993 Call_F64: RTLIB::NEARBYINT_F64,
4994 Call_F80: RTLIB::NEARBYINT_F80,
4995 Call_F128: RTLIB::NEARBYINT_F128,
4996 Call_PPCF128: RTLIB::NEARBYINT_PPCF128, Results);
4997 break;
4998 case ISD::FROUND:
4999 case ISD::STRICT_FROUND:
5000 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUND_F32,
5001 Call_F64: RTLIB::ROUND_F64,
5002 Call_F80: RTLIB::ROUND_F80,
5003 Call_F128: RTLIB::ROUND_F128,
5004 Call_PPCF128: RTLIB::ROUND_PPCF128, Results);
5005 break;
5006 case ISD::FROUNDEVEN:
5007 case ISD::STRICT_FROUNDEVEN:
5008 ExpandFPLibCall(Node, Call_F32: RTLIB::ROUNDEVEN_F32,
5009 Call_F64: RTLIB::ROUNDEVEN_F64,
5010 Call_F80: RTLIB::ROUNDEVEN_F80,
5011 Call_F128: RTLIB::ROUNDEVEN_F128,
5012 Call_PPCF128: RTLIB::ROUNDEVEN_PPCF128, Results);
5013 break;
5014 case ISD::FLDEXP:
5015 case ISD::STRICT_FLDEXP:
5016 ExpandFPLibCall(Node, Call_F32: RTLIB::LDEXP_F32, Call_F64: RTLIB::LDEXP_F64, Call_F80: RTLIB::LDEXP_F80,
5017 Call_F128: RTLIB::LDEXP_F128, Call_PPCF128: RTLIB::LDEXP_PPCF128, Results);
5018 break;
5019 case ISD::FMODF:
5020 case ISD::FFREXP: {
5021 EVT VT = Node->getValueType(ResNo: 0);
5022 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
5023 : RTLIB::getFREXP(RetVT: VT);
5024 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
5025 /*CallRetResNo=*/0);
5026 if (!Expanded)
5027 llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");
5028 break;
5029 }
5030 case ISD::FPOWI:
5031 case ISD::STRICT_FPOWI: {
5032 RTLIB::Libcall LC = RTLIB::getPOWI(RetVT: Node->getSimpleValueType(ResNo: 0));
5033 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
5034 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
5035 // Some targets don't have a powi libcall; use pow instead.
5036 if (Node->isStrictFPOpcode()) {
5037 SDValue Exponent =
5038 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
5039 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5040 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5041 SDValue FPOW =
5042 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
5043 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
5044 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
5045 Results.push_back(Elt: FPOW);
5046 Results.push_back(Elt: FPOW.getValue(R: 1));
5047 } else {
5048 SDValue Exponent =
5049 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
5050 Operand: Node->getOperand(Num: 1));
5051 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
5052 VT: Node->getValueType(ResNo: 0),
5053 N1: Node->getOperand(Num: 0), N2: Exponent));
5054 }
5055 break;
5056 }
5057 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
5058 bool ExponentHasSizeOfInt =
5059 DAG.getLibInfo().getIntSize() ==
5060 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
5061 if (!ExponentHasSizeOfInt) {
5062 // If the exponent does not match with sizeof(int) a libcall to
5063 // RTLIB::POWI would use the wrong type for the argument.
5064 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
5065 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
5066 break;
5067 }
5068 ExpandFPLibCall(Node, LC, Results);
5069 break;
5070 }
5071 case ISD::FPOW:
5072 case ISD::STRICT_FPOW:
5073 ExpandFPLibCall(Node, Call_F32: RTLIB::POW_F32, Call_F64: RTLIB::POW_F64, Call_F80: RTLIB::POW_F80,
5074 Call_F128: RTLIB::POW_F128, Call_PPCF128: RTLIB::POW_PPCF128, Results);
5075 break;
5076 case ISD::LROUND:
5077 case ISD::STRICT_LROUND:
5078 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
5079 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
5080 Call_F128: RTLIB::LROUND_F128,
5081 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
5082 break;
5083 case ISD::LLROUND:
5084 case ISD::STRICT_LLROUND:
5085 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
5086 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
5087 Call_F128: RTLIB::LLROUND_F128,
5088 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
5089 break;
5090 case ISD::LRINT:
5091 case ISD::STRICT_LRINT:
5092 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
5093 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
5094 Call_F128: RTLIB::LRINT_F128,
5095 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
5096 break;
5097 case ISD::LLRINT:
5098 case ISD::STRICT_LLRINT:
5099 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
5100 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
5101 Call_F128: RTLIB::LLRINT_F128,
5102 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
5103 break;
5104 case ISD::FDIV:
5105 case ISD::STRICT_FDIV: {
5106 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5107 Call_F32: {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5108 Call_F64: {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5109 Call_F80: {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5110 Call_F128: {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5111 Call_PPCF128: {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5112 break;
5113 }
5114 case ISD::FREM:
5115 case ISD::STRICT_FREM:
5116 ExpandFPLibCall(Node, Call_F32: RTLIB::REM_F32, Call_F64: RTLIB::REM_F64,
5117 Call_F80: RTLIB::REM_F80, Call_F128: RTLIB::REM_F128,
5118 Call_PPCF128: RTLIB::REM_PPCF128, Results);
5119 break;
5120 case ISD::FMA:
5121 case ISD::STRICT_FMA:
5122 ExpandFPLibCall(Node, Call_F32: RTLIB::FMA_F32, Call_F64: RTLIB::FMA_F64,
5123 Call_F80: RTLIB::FMA_F80, Call_F128: RTLIB::FMA_F128,
5124 Call_PPCF128: RTLIB::FMA_PPCF128, Results);
5125 break;
5126 case ISD::FADD:
5127 case ISD::STRICT_FADD: {
5128 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5129 Call_F32: {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5130 Call_F64: {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5131 Call_F80: {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5132 Call_F128: {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5133 Call_PPCF128: {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5134 break;
5135 }
5136 case ISD::FMUL:
5137 case ISD::STRICT_FMUL: {
5138 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5139 Call_F32: {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5140 Call_F64: {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5141 Call_F80: {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5142 Call_F128: {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5143 Call_PPCF128: {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5144 break;
5145 }
5146 case ISD::FP16_TO_FP:
5147 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5148 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
5149 }
5150 break;
5151 case ISD::STRICT_BF16_TO_FP:
5152 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5153 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5154 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
5155 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5156 Results.push_back(Elt: Tmp.first);
5157 Results.push_back(Elt: Tmp.second);
5158 }
5159 break;
5160 case ISD::STRICT_FP16_TO_FP: {
5161 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5162 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5163 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
5164 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5165 Results.push_back(Elt: Tmp.first);
5166 Results.push_back(Elt: Tmp.second);
5167 }
5168 break;
5169 }
5170 case ISD::FP_TO_FP16: {
5171 RTLIB::Libcall LC =
5172 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
5173 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5174 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5175 break;
5176 }
5177 case ISD::FP_TO_BF16: {
5178 RTLIB::Libcall LC =
5179 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
5180 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5181 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5182 break;
5183 }
5184 case ISD::STRICT_SINT_TO_FP:
5185 case ISD::STRICT_UINT_TO_FP:
5186 case ISD::SINT_TO_FP:
5187 case ISD::UINT_TO_FP: {
5188 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
5189 bool IsStrict = Node->isStrictFPOpcode();
5190 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
5191 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
5192 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
5193 EVT RVT = Node->getValueType(ResNo: 0);
5194 EVT NVT = EVT();
5195 SDLoc dl(Node);
5196
5197 // Even if the input is legal, no libcall may exactly match, eg. we don't
5198 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
5199 // eg: i13 -> fp. Then, look for an appropriate libcall.
5200 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5201 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5202 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5203 ++t) {
5204 NVT = (MVT::SimpleValueType)t;
5205 // The source needs to big enough to hold the operand.
5206 if (NVT.bitsGE(VT: SVT))
5207 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
5208 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
5209 }
5210 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5211
5212 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5213 // Sign/zero extend the argument if the libcall takes a larger type.
5214 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
5215 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
5216 CallOptions.setIsSigned(Signed);
5217 std::pair<SDValue, SDValue> Tmp =
5218 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
5219 Results.push_back(Elt: Tmp.first);
5220 if (IsStrict)
5221 Results.push_back(Elt: Tmp.second);
5222 break;
5223 }
5224 case ISD::FP_TO_SINT:
5225 case ISD::FP_TO_UINT:
5226 case ISD::STRICT_FP_TO_SINT:
5227 case ISD::STRICT_FP_TO_UINT: {
5228 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
5229 bool IsStrict = Node->isStrictFPOpcode();
5230 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
5231 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
5232
5233 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5234 EVT SVT = Op.getValueType();
5235 EVT RVT = Node->getValueType(ResNo: 0);
5236 EVT NVT = EVT();
5237 SDLoc dl(Node);
5238
5239 // Even if the result is legal, no libcall may exactly match, eg. we don't
5240 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
5241 // eg: fp -> i32. Then, look for an appropriate libcall.
5242 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5243 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5244 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5245 ++IntVT) {
5246 NVT = (MVT::SimpleValueType)IntVT;
5247 // The type needs to big enough to hold the result.
5248 if (NVT.bitsGE(VT: RVT))
5249 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
5250 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
5251 }
5252 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5253
5254 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5255 std::pair<SDValue, SDValue> Tmp =
5256 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
5257
5258 // Truncate the result if the libcall returns a larger type.
5259 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
5260 if (IsStrict)
5261 Results.push_back(Elt: Tmp.second);
5262 break;
5263 }
5264
5265 case ISD::FP_ROUND:
5266 case ISD::STRICT_FP_ROUND: {
5267 // X = FP_ROUND(Y, TRUNC)
5268 // TRUNC is a flag, which is always an integer that is zero or one.
5269 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5270 // is known to not change the value of Y.
5271 // We can only expand it into libcall if the TRUNC is 0.
5272 bool IsStrict = Node->isStrictFPOpcode();
5273 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5274 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5275 EVT VT = Node->getValueType(ResNo: 0);
5276 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5277 "Unable to expand as libcall if it is not normal rounding");
5278
5279 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5280 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5281
5282 std::pair<SDValue, SDValue> Tmp =
5283 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5284 Results.push_back(Elt: Tmp.first);
5285 if (IsStrict)
5286 Results.push_back(Elt: Tmp.second);
5287 break;
5288 }
5289 case ISD::FP_EXTEND: {
5290 Results.push_back(
5291 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5292 RetVT: Node->getValueType(ResNo: 0)),
5293 Node, isSigned: false).first);
5294 break;
5295 }
5296 case ISD::STRICT_FP_EXTEND:
5297 case ISD::STRICT_FP_TO_FP16:
5298 case ISD::STRICT_FP_TO_BF16: {
5299 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5300 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5301 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5302 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5303 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5304 else
5305 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5306 RetVT: Node->getValueType(ResNo: 0));
5307
5308 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5309
5310 std::pair<SDValue, SDValue> Tmp =
5311 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5312 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5313 Results.push_back(Elt: Tmp.first);
5314 Results.push_back(Elt: Tmp.second);
5315 break;
5316 }
5317 case ISD::FSUB:
5318 case ISD::STRICT_FSUB: {
5319 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5320 Call_F32: {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5321 Call_F64: {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5322 Call_F80: {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5323 Call_F128: {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5324 Call_PPCF128: {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5325 break;
5326 }
5327 case ISD::SREM:
5328 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5329 Call_I8: RTLIB::SREM_I8,
5330 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5331 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5332 break;
5333 case ISD::UREM:
5334 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5335 Call_I8: RTLIB::UREM_I8,
5336 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5337 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5338 break;
5339 case ISD::SDIV:
5340 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5341 Call_I8: RTLIB::SDIV_I8,
5342 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5343 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5344 break;
5345 case ISD::UDIV:
5346 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5347 Call_I8: RTLIB::UDIV_I8,
5348 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5349 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5350 break;
5351 case ISD::SDIVREM:
5352 case ISD::UDIVREM:
5353 // Expand into divrem libcall
5354 ExpandDivRemLibCall(Node, Results);
5355 break;
5356 case ISD::MUL:
5357 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5358 Call_I8: RTLIB::MUL_I8,
5359 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5360 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5361 break;
5362 case ISD::CTLZ_ZERO_UNDEF:
5363 Results.push_back(Elt: ExpandBitCountingLibCall(
5364 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5365 break;
5366 case ISD::CTPOP:
5367 Results.push_back(Elt: ExpandBitCountingLibCall(
5368 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5369 break;
5370 case ISD::RESET_FPENV: {
5371 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5372 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5373 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5374 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5375 SDValue Chain = Node->getOperand(Num: 0);
5376 Results.push_back(
5377 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, DLoc: dl));
5378 break;
5379 }
5380 case ISD::GET_FPENV_MEM: {
5381 SDValue Chain = Node->getOperand(Num: 0);
5382 SDValue EnvPtr = Node->getOperand(Num: 1);
5383 Results.push_back(
5384 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5385 break;
5386 }
5387 case ISD::SET_FPENV_MEM: {
5388 SDValue Chain = Node->getOperand(Num: 0);
5389 SDValue EnvPtr = Node->getOperand(Num: 1);
5390 Results.push_back(
5391 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, DLoc: dl));
5392 break;
5393 }
5394 case ISD::GET_FPMODE: {
5395 // Call fegetmode, which saves control modes into a stack slot. Then load
5396 // the value to return from the stack.
5397 EVT ModeVT = Node->getValueType(ResNo: 0);
5398 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5399 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5400 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5401 InChain: Node->getOperand(Num: 0), DLoc: dl);
5402 SDValue LdInst = DAG.getLoad(
5403 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5404 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5405 Results.push_back(Elt: LdInst);
5406 Results.push_back(Elt: LdInst.getValue(R: 1));
5407 break;
5408 }
5409 case ISD::SET_FPMODE: {
5410 // Move control modes to stack slot and then call fesetmode with the pointer
5411 // to the slot as argument.
5412 SDValue Mode = Node->getOperand(Num: 1);
5413 EVT ModeVT = Mode.getValueType();
5414 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5415 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5416 SDValue StInst = DAG.getStore(
5417 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5418 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5419 Results.push_back(
5420 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, DLoc: dl));
5421 break;
5422 }
5423 case ISD::RESET_FPMODE: {
5424 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5425 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5426 // target must provide custom lowering.
5427 const DataLayout &DL = DAG.getDataLayout();
5428 EVT PtrTy = TLI.getPointerTy(DL);
5429 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5430 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5431 InChain: Node->getOperand(Num: 0), DLoc: dl));
5432 break;
5433 }
5434 }
5435
5436 // Replace the original node with the legalized result.
5437 if (!Results.empty()) {
5438 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5439 ReplaceNode(Old: Node, New: Results.data());
5440 } else
5441 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5442}
5443
5444// Determine the vector type to use in place of an original scalar element when
5445// promoting equally sized vectors.
5446static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5447 MVT EltVT, MVT NewEltVT) {
5448 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5449 MVT MidVT = OldEltsPerNewElt == 1
5450 ? NewEltVT
5451 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5452 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5453 return MidVT;
5454}
5455
5456void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5457 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5458 SmallVector<SDValue, 8> Results;
5459 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5460 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5461 Node->getOpcode() == ISD::SINT_TO_FP ||
5462 Node->getOpcode() == ISD::SETCC ||
5463 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5464 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5465 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5466 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5467 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5468 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM) {
5469 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5470 }
5471 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5472 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5473 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5474 Node->getOpcode() == ISD::STRICT_FSETCC ||
5475 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5476 Node->getOpcode() == ISD::STRICT_LRINT ||
5477 Node->getOpcode() == ISD::STRICT_LLRINT ||
5478 Node->getOpcode() == ISD::STRICT_LROUND ||
5479 Node->getOpcode() == ISD::STRICT_LLROUND ||
5480 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5481 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5482 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5483 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5484 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5485 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5486 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5487 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5488 if (Node->getOpcode() == ISD::BR_CC ||
5489 Node->getOpcode() == ISD::SELECT_CC)
5490 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5491 // Preserve fast math flags
5492 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5493 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5494 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5495 SDLoc dl(Node);
5496 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5497 switch (Node->getOpcode()) {
5498 case ISD::CTTZ:
5499 case ISD::CTTZ_ZERO_UNDEF:
5500 case ISD::CTLZ:
5501 case ISD::CTPOP: {
5502 // Zero extend the argument unless its cttz, then use any_extend.
5503 if (Node->getOpcode() == ISD::CTTZ ||
5504 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
5505 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5506 else
5507 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5508
5509 unsigned NewOpc = Node->getOpcode();
5510 if (NewOpc == ISD::CTTZ) {
5511 // The count is the same in the promoted type except if the original
5512 // value was zero. This can be handled by setting the bit just off
5513 // the top of the original type.
5514 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5515 BitNo: OVT.getSizeInBits());
5516 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5517 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5518 NewOpc = ISD::CTTZ_ZERO_UNDEF;
5519 }
5520 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
5521 // already the correct result.
5522 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5523 if (NewOpc == ISD::CTLZ) {
5524 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5525 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5526 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5527 OVT.getSizeInBits(), DL: dl, VT: NVT));
5528 }
5529 Results.push_back(
5530 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5531 break;
5532 }
5533 case ISD::CTLZ_ZERO_UNDEF: {
5534 // We know that the argument is unlikely to be zero, hence we can take a
5535 // different approach as compared to ISD::CTLZ
5536
5537 // Any Extend the argument
5538 auto AnyExtendedNode =
5539 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5540
5541 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5542 auto ShiftConstant = DAG.getShiftAmountConstant(
5543 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5544 auto LeftShiftResult =
5545 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5546
5547 // Perform the larger operation
5548 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5549 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5550 break;
5551 }
5552 case ISD::BITREVERSE:
5553 case ISD::BSWAP: {
5554 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5555 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5556 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5557 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5558 N2: DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl));
5559
5560 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5561 break;
5562 }
5563 case ISD::FP_TO_UINT:
5564 case ISD::STRICT_FP_TO_UINT:
5565 case ISD::FP_TO_SINT:
5566 case ISD::STRICT_FP_TO_SINT:
5567 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5568 break;
5569 case ISD::FP_TO_UINT_SAT:
5570 case ISD::FP_TO_SINT_SAT:
5571 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5572 break;
5573 case ISD::UINT_TO_FP:
5574 case ISD::STRICT_UINT_TO_FP:
5575 case ISD::SINT_TO_FP:
5576 case ISD::STRICT_SINT_TO_FP:
5577 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5578 break;
5579 case ISD::VAARG: {
5580 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5581 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5582
5583 unsigned TruncOp;
5584 if (OVT.isVector()) {
5585 TruncOp = ISD::BITCAST;
5586 } else {
5587 assert(OVT.isInteger()
5588 && "VAARG promotion is supported only for vectors or integer types");
5589 TruncOp = ISD::TRUNCATE;
5590 }
5591
5592 // Perform the larger operation, then convert back
5593 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5594 Align: Node->getConstantOperandVal(Num: 3));
5595 Chain = Tmp1.getValue(R: 1);
5596
5597 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5598
5599 // Modified the chain result - switch anything that used the old chain to
5600 // use the new one.
5601 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5602 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5603 if (UpdatedNodes) {
5604 UpdatedNodes->insert(X: Tmp2.getNode());
5605 UpdatedNodes->insert(X: Chain.getNode());
5606 }
5607 ReplacedNode(N: Node);
5608 break;
5609 }
5610 case ISD::MUL:
5611 case ISD::SDIV:
5612 case ISD::SREM:
5613 case ISD::UDIV:
5614 case ISD::UREM:
5615 case ISD::SMIN:
5616 case ISD::SMAX:
5617 case ISD::UMIN:
5618 case ISD::UMAX:
5619 case ISD::AND:
5620 case ISD::OR:
5621 case ISD::XOR: {
5622 unsigned ExtOp, TruncOp;
5623 if (OVT.isVector()) {
5624 ExtOp = ISD::BITCAST;
5625 TruncOp = ISD::BITCAST;
5626 } else {
5627 assert(OVT.isInteger() && "Cannot promote logic operation");
5628
5629 switch (Node->getOpcode()) {
5630 default:
5631 ExtOp = ISD::ANY_EXTEND;
5632 break;
5633 case ISD::SDIV:
5634 case ISD::SREM:
5635 case ISD::SMIN:
5636 case ISD::SMAX:
5637 ExtOp = ISD::SIGN_EXTEND;
5638 break;
5639 case ISD::UDIV:
5640 case ISD::UREM:
5641 ExtOp = ISD::ZERO_EXTEND;
5642 break;
5643 case ISD::UMIN:
5644 case ISD::UMAX:
5645 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5646 ExtOp = ISD::SIGN_EXTEND;
5647 else
5648 ExtOp = ISD::ZERO_EXTEND;
5649 break;
5650 }
5651 TruncOp = ISD::TRUNCATE;
5652 }
5653 // Promote each of the values to the new type.
5654 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5655 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5656 // Perform the larger operation, then convert back
5657 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5658 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5659 break;
5660 }
5661 case ISD::UMUL_LOHI:
5662 case ISD::SMUL_LOHI: {
5663 // Promote to a multiply in a wider integer type.
5664 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5665 : ISD::SIGN_EXTEND;
5666 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5667 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5668 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5669
5670 unsigned OriginalSize = OVT.getScalarSizeInBits();
5671 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5672 N2: DAG.getShiftAmountConstant(Val: OriginalSize, VT: NVT, DL: dl));
5673 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5674 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5675 break;
5676 }
5677 case ISD::SELECT: {
5678 unsigned ExtOp, TruncOp;
5679 if (Node->getValueType(ResNo: 0).isVector() ||
5680 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5681 ExtOp = ISD::BITCAST;
5682 TruncOp = ISD::BITCAST;
5683 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5684 ExtOp = ISD::ANY_EXTEND;
5685 TruncOp = ISD::TRUNCATE;
5686 } else {
5687 ExtOp = ISD::FP_EXTEND;
5688 TruncOp = ISD::FP_ROUND;
5689 }
5690 Tmp1 = Node->getOperand(Num: 0);
5691 // Promote each of the values to the new type.
5692 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5693 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5694 // Perform the larger operation, then round down.
5695 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5696 if (TruncOp != ISD::FP_ROUND)
5697 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5698 else
5699 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5700 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5701 Results.push_back(Elt: Tmp1);
5702 break;
5703 }
5704 case ISD::VECTOR_SHUFFLE: {
5705 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5706
5707 // Cast the two input vectors.
5708 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5709 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5710
5711 // Convert the shuffle mask to the right # elements.
5712 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5713 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5714 Results.push_back(Elt: Tmp1);
5715 break;
5716 }
5717 case ISD::VECTOR_SPLICE_LEFT:
5718 case ISD::VECTOR_SPLICE_RIGHT: {
5719 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5720 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5721 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5722 N3: Node->getOperand(Num: 2));
5723 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5724 break;
5725 }
5726 case ISD::SELECT_CC: {
5727 SDValue Cond = Node->getOperand(Num: 4);
5728 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5729 // Type of the comparison operands.
5730 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5731 assert(CVT == OVT && "not handled");
5732
5733 unsigned ExtOp = ISD::FP_EXTEND;
5734 if (NVT.isInteger()) {
5735 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5736 }
5737
5738 // Promote the comparison operands, if needed.
5739 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5740 Tmp1 = Node->getOperand(Num: 0);
5741 Tmp2 = Node->getOperand(Num: 1);
5742 } else {
5743 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5744 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5745 }
5746 // Cast the true/false operands.
5747 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5748 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5749
5750 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5751 Flags: Node->getFlags());
5752
5753 // Cast the result back to the original type.
5754 if (ExtOp != ISD::FP_EXTEND)
5755 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5756 else
5757 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5758 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5759
5760 Results.push_back(Elt: Tmp1);
5761 break;
5762 }
5763 case ISD::SETCC:
5764 case ISD::STRICT_FSETCC:
5765 case ISD::STRICT_FSETCCS: {
5766 unsigned ExtOp = ISD::FP_EXTEND;
5767 if (NVT.isInteger()) {
5768 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Node->getOperand(Num: 2))->get();
5769 if (isSignedIntSetCC(Code: CCCode) ||
5770 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5771 ExtOp = ISD::SIGN_EXTEND;
5772 else
5773 ExtOp = ISD::ZERO_EXTEND;
5774 }
5775 if (Node->isStrictFPOpcode()) {
5776 SDValue InChain = Node->getOperand(Num: 0);
5777 std::tie(args&: Tmp1, args: std::ignore) =
5778 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5779 std::tie(args&: Tmp2, args: std::ignore) =
5780 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5781 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5782 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5783 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
5784 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5785 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5786 Flags: Node->getFlags()));
5787 Results.push_back(Elt: Results.back().getValue(R: 1));
5788 break;
5789 }
5790 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5791 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5792 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5793 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5794 break;
5795 }
5796 case ISD::BR_CC: {
5797 unsigned ExtOp = ISD::FP_EXTEND;
5798 if (NVT.isInteger()) {
5799 ISD::CondCode CCCode =
5800 cast<CondCodeSDNode>(Val: Node->getOperand(Num: 1))->get();
5801 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5802 }
5803 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5804 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5805 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5806 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5807 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5808 break;
5809 }
5810 case ISD::FADD:
5811 case ISD::FSUB:
5812 case ISD::FMUL:
5813 case ISD::FDIV:
5814 case ISD::FREM:
5815 case ISD::FMINNUM:
5816 case ISD::FMAXNUM:
5817 case ISD::FMINIMUM:
5818 case ISD::FMAXIMUM:
5819 case ISD::FMINIMUMNUM:
5820 case ISD::FMAXIMUMNUM:
5821 case ISD::FPOW:
5822 case ISD::FATAN2:
5823 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5824 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5825 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5826 Results.push_back(
5827 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5828 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5829 break;
5830
5831 case ISD::STRICT_FMINIMUM:
5832 case ISD::STRICT_FMAXIMUM: {
5833 SDValue InChain = Node->getOperand(Num: 0);
5834 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
5835 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5836 N2: Node->getOperand(Num: 1));
5837 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5838 N2: Node->getOperand(Num: 2));
5839 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
5840 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
5841 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
5842 N1: InChain, N2: Tmp3,
5843 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5844 Results.push_back(Elt: Tmp4);
5845 Results.push_back(Elt: Tmp4.getValue(R: 1));
5846 break;
5847 }
5848
5849 case ISD::STRICT_FADD:
5850 case ISD::STRICT_FSUB:
5851 case ISD::STRICT_FMUL:
5852 case ISD::STRICT_FDIV:
5853 case ISD::STRICT_FMINNUM:
5854 case ISD::STRICT_FMAXNUM:
5855 case ISD::STRICT_FREM:
5856 case ISD::STRICT_FPOW:
5857 case ISD::STRICT_FATAN2:
5858 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5859 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5860 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5861 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5862 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5863 N2: Tmp2.getValue(R: 1));
5864 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5865 Ops: {Tmp3, Tmp1, Tmp2});
5866 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5867 Ops: {Tmp1.getValue(R: 1), Tmp1,
5868 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5869 Results.push_back(Elt: Tmp1);
5870 Results.push_back(Elt: Tmp1.getValue(R: 1));
5871 break;
5872 case ISD::FMA:
5873 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5874 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5875 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5876 Results.push_back(
5877 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5878 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5879 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5880 break;
5881 case ISD::STRICT_FMA:
5882 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5883 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5884 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5885 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5886 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5887 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
5888 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5889 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
5890 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5891 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
5892 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5893 Ops: {Tmp4.getValue(R: 1), Tmp4,
5894 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5895 Results.push_back(Elt: Tmp4);
5896 Results.push_back(Elt: Tmp4.getValue(R: 1));
5897 break;
5898 case ISD::FCOPYSIGN:
5899 case ISD::FLDEXP:
5900 case ISD::FPOWI: {
5901 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5902 Tmp2 = Node->getOperand(Num: 1);
5903 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5904
5905 // fcopysign doesn't change anything but the sign bit, so
5906 // (fp_round (fcopysign (fpext a), b))
5907 // is as precise as
5908 // (fp_round (fpext a))
5909 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5910 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5911 Results.push_back(
5912 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5913 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5914 break;
5915 }
5916 case ISD::STRICT_FLDEXP: {
5917 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5918 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5919 Tmp2 = Node->getOperand(Num: 2);
5920 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
5921 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
5922 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5923 Ops: {Tmp3.getValue(R: 1), Tmp3,
5924 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5925 Results.push_back(Elt: Tmp4);
5926 Results.push_back(Elt: Tmp4.getValue(R: 1));
5927 break;
5928 }
5929 case ISD::STRICT_FPOWI:
5930 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5931 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5932 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5933 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
5934 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5935 Ops: {Tmp2.getValue(R: 1), Tmp2,
5936 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5937 Results.push_back(Elt: Tmp3);
5938 Results.push_back(Elt: Tmp3.getValue(R: 1));
5939 break;
5940 case ISD::FFREXP: {
5941 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5942 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5943
5944 Results.push_back(
5945 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5946 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5947
5948 Results.push_back(Elt: Tmp2.getValue(R: 1));
5949 break;
5950 }
5951 case ISD::FMODF:
5952 case ISD::FSINCOS:
5953 case ISD::FSINCOSPI: {
5954 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5955 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
5956 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
5957 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5958 Results.push_back(
5959 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
5960 break;
5961 }
5962 case ISD::FFLOOR:
5963 case ISD::FCEIL:
5964 case ISD::FRINT:
5965 case ISD::FNEARBYINT:
5966 case ISD::FROUND:
5967 case ISD::FROUNDEVEN:
5968 case ISD::FTRUNC:
5969 case ISD::FNEG:
5970 case ISD::FSQRT:
5971 case ISD::FSIN:
5972 case ISD::FCOS:
5973 case ISD::FTAN:
5974 case ISD::FASIN:
5975 case ISD::FACOS:
5976 case ISD::FATAN:
5977 case ISD::FSINH:
5978 case ISD::FCOSH:
5979 case ISD::FTANH:
5980 case ISD::FLOG:
5981 case ISD::FLOG2:
5982 case ISD::FLOG10:
5983 case ISD::FABS:
5984 case ISD::FEXP:
5985 case ISD::FEXP2:
5986 case ISD::FEXP10:
5987 case ISD::FCANONICALIZE:
5988 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5989 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5990 Results.push_back(
5991 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5992 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5993 break;
5994 case ISD::STRICT_FFLOOR:
5995 case ISD::STRICT_FCEIL:
5996 case ISD::STRICT_FRINT:
5997 case ISD::STRICT_FNEARBYINT:
5998 case ISD::STRICT_FROUND:
5999 case ISD::STRICT_FROUNDEVEN:
6000 case ISD::STRICT_FTRUNC:
6001 case ISD::STRICT_FSQRT:
6002 case ISD::STRICT_FSIN:
6003 case ISD::STRICT_FCOS:
6004 case ISD::STRICT_FTAN:
6005 case ISD::STRICT_FASIN:
6006 case ISD::STRICT_FACOS:
6007 case ISD::STRICT_FATAN:
6008 case ISD::STRICT_FSINH:
6009 case ISD::STRICT_FCOSH:
6010 case ISD::STRICT_FTANH:
6011 case ISD::STRICT_FLOG:
6012 case ISD::STRICT_FLOG2:
6013 case ISD::STRICT_FLOG10:
6014 case ISD::STRICT_FEXP:
6015 case ISD::STRICT_FEXP2:
6016 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6017 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6018 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6019 Ops: {Tmp1.getValue(R: 1), Tmp1});
6020 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6021 Ops: {Tmp2.getValue(R: 1), Tmp2,
6022 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6023 Results.push_back(Elt: Tmp3);
6024 Results.push_back(Elt: Tmp3.getValue(R: 1));
6025 break;
6026 case ISD::LLROUND:
6027 case ISD::LROUND:
6028 case ISD::LRINT:
6029 case ISD::LLRINT:
6030 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6031 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
6032 Results.push_back(Elt: Tmp2);
6033 break;
6034 case ISD::STRICT_LLROUND:
6035 case ISD::STRICT_LROUND:
6036 case ISD::STRICT_LRINT:
6037 case ISD::STRICT_LLRINT:
6038 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6039 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6040 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6041 Ops: {Tmp1.getValue(R: 1), Tmp1});
6042 Results.push_back(Elt: Tmp2);
6043 Results.push_back(Elt: Tmp2.getValue(R: 1));
6044 break;
6045 case ISD::BUILD_VECTOR: {
6046 MVT EltVT = OVT.getVectorElementType();
6047 MVT NewEltVT = NVT.getVectorElementType();
6048
6049 // Handle bitcasts to a different vector type with the same total bit size
6050 //
6051 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
6052 // =>
6053 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
6054
6055 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6056 "Invalid promote type for build_vector");
6057 assert(NewEltVT.bitsLE(EltVT) && "not handled");
6058
6059 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6060
6061 SmallVector<SDValue, 8> NewOps;
6062 for (const SDValue &Op : Node->op_values())
6063 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
6064
6065 SDLoc SL(Node);
6066 SDValue Concat =
6067 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
6068 DL: SL, VT: NVT, Ops: NewOps);
6069 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6070 Results.push_back(Elt: CvtVec);
6071 break;
6072 }
6073 case ISD::EXTRACT_VECTOR_ELT: {
6074 MVT EltVT = OVT.getVectorElementType();
6075 MVT NewEltVT = NVT.getVectorElementType();
6076
6077 // Handle bitcasts to a different vector type with the same total bit size.
6078 //
6079 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
6080 // =>
6081 // v4i32:castx = bitcast x:v2i64
6082 //
6083 // i64 = bitcast
6084 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
6085 // (i32 (extract_vector_elt castx, (2 * y + 1)))
6086 //
6087
6088 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6089 "Invalid promote type for extract_vector_elt");
6090 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6091
6092 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6093 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6094
6095 SDValue Idx = Node->getOperand(Num: 1);
6096 EVT IdxVT = Idx.getValueType();
6097 SDLoc SL(Node);
6098 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
6099 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6100
6101 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6102
6103 SmallVector<SDValue, 8> NewOps;
6104 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6105 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6106 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6107
6108 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6109 N1: CastVec, N2: TmpIdx);
6110 NewOps.push_back(Elt);
6111 }
6112
6113 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
6114 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
6115 break;
6116 }
6117 case ISD::INSERT_VECTOR_ELT: {
6118 MVT EltVT = OVT.getVectorElementType();
6119 MVT NewEltVT = NVT.getVectorElementType();
6120
6121 // Handle bitcasts to a different vector type with the same total bit size
6122 //
6123 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
6124 // =>
6125 // v4i32:castx = bitcast x:v2i64
6126 // v2i32:casty = bitcast y:i64
6127 //
6128 // v2i64 = bitcast
6129 // (v4i32 insert_vector_elt
6130 // (v4i32 insert_vector_elt v4i32:castx,
6131 // (extract_vector_elt casty, 0), 2 * z),
6132 // (extract_vector_elt casty, 1), (2 * z + 1))
6133
6134 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6135 "Invalid promote type for insert_vector_elt");
6136 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6137
6138 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6139 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6140
6141 SDValue Val = Node->getOperand(Num: 1);
6142 SDValue Idx = Node->getOperand(Num: 2);
6143 EVT IdxVT = Idx.getValueType();
6144 SDLoc SL(Node);
6145
6146 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
6147 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6148
6149 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6150 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6151
6152 SDValue NewVec = CastVec;
6153 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6154 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6155 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6156
6157 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6158 N1: CastVal, N2: IdxOffset);
6159
6160 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
6161 N1: NewVec, N2: Elt, N3: InEltIdx);
6162 }
6163
6164 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
6165 break;
6166 }
6167 case ISD::SCALAR_TO_VECTOR: {
6168 MVT EltVT = OVT.getVectorElementType();
6169 MVT NewEltVT = NVT.getVectorElementType();
6170
6171 // Handle bitcasts to different vector type with the same total bit size.
6172 //
6173 // e.g. v2i64 = scalar_to_vector x:i64
6174 // =>
6175 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
6176 //
6177
6178 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6179 SDValue Val = Node->getOperand(Num: 0);
6180 SDLoc SL(Node);
6181
6182 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6183 SDValue Undef = DAG.getUNDEF(VT: MidVT);
6184
6185 SmallVector<SDValue, 8> NewElts;
6186 NewElts.push_back(Elt: CastVal);
6187 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
6188 NewElts.push_back(Elt: Undef);
6189
6190 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
6191 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6192 Results.push_back(Elt: CvtVec);
6193 break;
6194 }
6195 case ISD::ATOMIC_SWAP:
6196 case ISD::ATOMIC_STORE: {
6197 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6198 SDLoc SL(Node);
6199 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
6200 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6201 "unexpected promotion type");
6202 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6203 "unexpected atomic_swap with illegal type");
6204
6205 SDValue Op0 = AM->getBasePtr();
6206 SDValue Op1 = CastVal;
6207
6208 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
6209 // but really it should merge with ISD::STORE.
6210 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6211 std::swap(a&: Op0, b&: Op1);
6212
6213 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
6214 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
6215
6216 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6217 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6218 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6219 } else
6220 Results.push_back(Elt: NewAtomic);
6221 break;
6222 }
6223 case ISD::ATOMIC_LOAD: {
6224 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6225 SDLoc SL(Node);
6226 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6227 "unexpected promotion type");
6228 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6229 "unexpected atomic_load with illegal type");
6230
6231 SDValue NewAtomic =
6232 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
6233 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
6234 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6235 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6236 break;
6237 }
6238 case ISD::SPLAT_VECTOR: {
6239 SDValue Scalar = Node->getOperand(Num: 0);
6240 MVT ScalarType = Scalar.getSimpleValueType();
6241 MVT NewScalarType = NVT.getVectorElementType();
6242 if (ScalarType.isInteger()) {
6243 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6244 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6245 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
6246 break;
6247 }
6248 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6249 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6250 Results.push_back(
6251 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6252 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6253 break;
6254 }
6255 case ISD::VECREDUCE_FMAX:
6256 case ISD::VECREDUCE_FMIN:
6257 case ISD::VECREDUCE_FMAXIMUM:
6258 case ISD::VECREDUCE_FMINIMUM:
6259 case ISD::VP_REDUCE_FMAX:
6260 case ISD::VP_REDUCE_FMIN:
6261 case ISD::VP_REDUCE_FMAXIMUM:
6262 case ISD::VP_REDUCE_FMINIMUM:
6263 Results.push_back(Elt: PromoteReduction(Node));
6264 break;
6265 }
6266
6267 // Replace the original node with the legalized result.
6268 if (!Results.empty()) {
6269 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6270 ReplaceNode(Old: Node, New: Results.data());
6271 } else
6272 LLVM_DEBUG(dbgs() << "Could not promote node\n");
6273}
6274
6275/// This is the entry point for the file.
6276void SelectionDAG::Legalize() {
6277 AssignTopologicalOrder();
6278
6279 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6280 // Use a delete listener to remove nodes which were deleted during
6281 // legalization from LegalizeNodes. This is needed to handle the situation
6282 // where a new node is allocated by the object pool to the same address of a
6283 // previously deleted node.
6284 DAGNodeDeletedListener DeleteListener(
6285 *this,
6286 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
6287
6288 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6289
6290 // Visit all the nodes. We start in topological order, so that we see
6291 // nodes with their original operands intact. Legalization can produce
6292 // new nodes which may themselves need to be legalized. Iterate until all
6293 // nodes have been legalized.
6294 while (true) {
6295 bool AnyLegalized = false;
6296 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6297 --NI;
6298
6299 SDNode *N = &*NI;
6300 if (N->use_empty() && N != getRoot().getNode()) {
6301 ++NI;
6302 DeleteNode(N);
6303 continue;
6304 }
6305
6306 if (LegalizedNodes.insert(Ptr: N).second) {
6307 AnyLegalized = true;
6308 Legalizer.LegalizeOp(Node: N);
6309
6310 if (N->use_empty() && N != getRoot().getNode()) {
6311 ++NI;
6312 DeleteNode(N);
6313 }
6314 }
6315 }
6316 if (!AnyLegalized)
6317 break;
6318
6319 }
6320
6321 // Remove dead nodes now.
6322 RemoveDeadNodes();
6323}
6324
6325bool SelectionDAG::LegalizeOp(SDNode *N,
6326 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6327 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6328 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6329
6330 // Directly insert the node in question, and legalize it. This will recurse
6331 // as needed through operands.
6332 LegalizedNodes.insert(Ptr: N);
6333 Legalizer.LegalizeOp(Node: N);
6334
6335 return LegalizedNodes.count(Ptr: N);
6336}
6337