1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/FloatingPointMode.h"
17#include "llvm/ADT/SetVector.h"
18#include "llvm/ADT/SmallPtrSet.h"
19#include "llvm/ADT/SmallSet.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/Analysis/ConstantFolding.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/CodeGen/ISDOpcodes.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineMemOperand.h"
29#include "llvm/CodeGen/RuntimeLibcallUtil.h"
30#include "llvm/CodeGen/SelectionDAG.h"
31#include "llvm/CodeGen/SelectionDAGNodes.h"
32#include "llvm/CodeGen/TargetFrameLowering.h"
33#include "llvm/CodeGen/TargetLowering.h"
34#include "llvm/CodeGen/TargetSubtargetInfo.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/CodeGenTypes/MachineValueType.h"
37#include "llvm/IR/CallingConv.h"
38#include "llvm/IR/Constants.h"
39#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/DerivedTypes.h"
41#include "llvm/IR/Function.h"
42#include "llvm/IR/Metadata.h"
43#include "llvm/IR/Type.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Compiler.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/MathExtras.h"
49#include "llvm/Support/raw_ostream.h"
50#include "llvm/Target/TargetMachine.h"
51#include "llvm/Target/TargetOptions.h"
52#include <cassert>
53#include <cstdint>
54#include <tuple>
55#include <utility>
56
57using namespace llvm;
58
59#define DEBUG_TYPE "legalizedag"
60
61namespace {
62
63/// Keeps track of state when getting the sign of a floating-point value as an
64/// integer.
65struct FloatSignAsInt {
66 EVT FloatVT;
67 SDValue Chain;
68 SDValue FloatPtr;
69 SDValue IntPtr;
70 MachinePointerInfo IntPointerInfo;
71 MachinePointerInfo FloatPointerInfo;
72 SDValue IntValue;
73 APInt SignMask;
74 uint8_t SignBit;
75};
76
77//===----------------------------------------------------------------------===//
78/// This takes an arbitrary SelectionDAG as input and
79/// hacks on it until the target machine can handle it. This involves
80/// eliminating value sizes the machine cannot handle (promoting small sizes to
81/// large sizes or splitting up large values into small values) as well as
82/// eliminating operations the machine cannot handle.
83///
84/// This code also does a small amount of optimization and recognition of idioms
85/// as part of its processing. For example, if a target does not support a
86/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
87/// will attempt merge setcc and brc instructions into brcc's.
88class SelectionDAGLegalize {
89 const TargetMachine &TM;
90 const TargetLowering &TLI;
91 SelectionDAG &DAG;
92
93 /// The set of nodes which have already been legalized. We hold a
94 /// reference to it in order to update as necessary on node deletion.
95 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
96
97 /// A set of all the nodes updated during legalization.
98 SmallSetVector<SDNode *, 16> *UpdatedNodes;
99
100 EVT getSetCCResultType(EVT VT) const {
101 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
102 }
103
104 // Libcall insertion helpers.
105
106public:
107 SelectionDAGLegalize(SelectionDAG &DAG,
108 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
109 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
110 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
111 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
112
113 /// Legalizes the given operation.
114 void LegalizeOp(SDNode *Node);
115
116private:
117 SDValue OptimizeFloatStore(StoreSDNode *ST);
118
119 void LegalizeLoadOps(SDNode *Node);
120 void LegalizeStoreOps(SDNode *Node);
121
122 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
123
124 /// Return a vector shuffle operation which
125 /// performs the same shuffe in terms of order or result bytes, but on a type
126 /// whose vector element type is narrower than the original shuffle type.
127 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
128 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
129 SDValue N1, SDValue N2,
130 ArrayRef<int> Mask) const;
131
132 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 TargetLowering::ArgListTy &&Args,
134 bool IsSigned, EVT RetVT);
135 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
136
137 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 SmallVectorImpl<SDValue> &Results);
139
140 void
141 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
142 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
143 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
144 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
145 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
146 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
147 SmallVectorImpl<SDValue> &Results);
148
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, RTLIB::Libcall Call_I8,
150 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
151 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
152 void ExpandArgFPLibCall(SDNode *Node,
153 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
154 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
155 RTLIB::Libcall Call_PPCF128,
156 SmallVectorImpl<SDValue> &Results);
157 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
158 RTLIB::Libcall CallI64,
159 RTLIB::Libcall CallI128);
160 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161
162 SDValue ExpandSincosStretLibCall(SDNode *Node) const;
163
164 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
165 const SDLoc &dl);
166 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
167 const SDLoc &dl, SDValue ChainIn);
168 SDValue ExpandBUILD_VECTOR(SDNode *Node);
169 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
170 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
171 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
172 SmallVectorImpl<SDValue> &Results);
173 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
174 SDValue Value) const;
175 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
176 SDValue NewIntValue) const;
177 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
178 SDValue ExpandFABS(SDNode *Node) const;
179 SDValue ExpandFNEG(SDNode *Node) const;
180 SDValue expandLdexp(SDNode *Node) const;
181 SDValue expandFrexp(SDNode *Node) const;
182
183 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
184 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
185 SmallVectorImpl<SDValue> &Results);
186 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
187 SmallVectorImpl<SDValue> &Results);
188 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
189
190 /// Implements vector reduce operation promotion.
191 ///
192 /// All vector operands are promoted to a vector type with larger element
193 /// type, and the start value is promoted to a larger scalar type. Then the
194 /// result is truncated back to the original scalar type.
195 SDValue PromoteReduction(SDNode *Node);
196
197 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
198
199 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
200 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
201 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
202 SDValue ExpandConcatVectors(SDNode *Node);
203
204 SDValue ExpandConstantFP(ConstantFPSDNode *CFP);
205 SDValue ExpandConstant(ConstantSDNode *CP);
206
207 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
208 bool ExpandNode(SDNode *Node);
209 void ConvertNodeToLibcall(SDNode *Node);
210 void PromoteNode(SDNode *Node);
211
212public:
213 // Node replacement helpers
214
215 void ReplacedNode(SDNode *N) {
216 LegalizedNodes.erase(Ptr: N);
217 if (UpdatedNodes)
218 UpdatedNodes->insert(X: N);
219 }
220
221 void ReplaceNode(SDNode *Old, SDNode *New) {
222 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
223 dbgs() << " with: "; New->dump(&DAG));
224
225 assert(Old->getNumValues() == New->getNumValues() &&
226 "Replacing one node with another that produces a different number "
227 "of values!");
228 DAG.ReplaceAllUsesWith(From: Old, To: New);
229 if (UpdatedNodes)
230 UpdatedNodes->insert(X: New);
231 ReplacedNode(N: Old);
232 }
233
234 void ReplaceNode(SDValue Old, SDValue New) {
235 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
236 dbgs() << " with: "; New->dump(&DAG));
237
238 DAG.ReplaceAllUsesWith(From: Old, To: New);
239 if (UpdatedNodes)
240 UpdatedNodes->insert(X: New.getNode());
241 ReplacedNode(N: Old.getNode());
242 }
243
244 void ReplaceNode(SDNode *Old, const SDValue *New) {
245 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
246
247 DAG.ReplaceAllUsesWith(From: Old, To: New);
248 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
249 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
250 New[i]->dump(&DAG));
251 if (UpdatedNodes)
252 UpdatedNodes->insert(X: New[i].getNode());
253 }
254 ReplacedNode(N: Old);
255 }
256
257 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
258 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
259 dbgs() << " with: "; New->dump(&DAG));
260
261 DAG.ReplaceAllUsesOfValueWith(From: Old, To: New);
262 if (UpdatedNodes)
263 UpdatedNodes->insert(X: New.getNode());
264 ReplacedNode(N: Old.getNode());
265 }
266};
267
268} // end anonymous namespace
269
270// Helper function that generates an MMO that considers the alignment of the
271// stack, and the size of the stack object
272static MachineMemOperand *getStackAlignedMMO(SDValue StackPtr,
273 MachineFunction &MF,
274 bool isObjectScalable) {
275 auto &MFI = MF.getFrameInfo();
276 int FI = cast<FrameIndexSDNode>(Val&: StackPtr)->getIndex();
277 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
278 LocationSize ObjectSize = isObjectScalable
279 ? LocationSize::beforeOrAfterPointer()
280 : LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: FI));
281 return MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
282 Size: ObjectSize, BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
283}
284
285/// Return a vector shuffle operation which
286/// performs the same shuffle in terms of order or result bytes, but on a type
287/// whose vector element type is narrower than the original shuffle type.
288/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
289SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
290 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
291 ArrayRef<int> Mask) const {
292 unsigned NumMaskElts = VT.getVectorNumElements();
293 unsigned NumDestElts = NVT.getVectorNumElements();
294 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
295
296 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
297
298 if (NumEltsGrowth == 1)
299 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask);
300
301 SmallVector<int, 8> NewMask;
302 for (unsigned i = 0; i != NumMaskElts; ++i) {
303 int Idx = Mask[i];
304 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
305 if (Idx < 0)
306 NewMask.push_back(Elt: -1);
307 else
308 NewMask.push_back(Elt: Idx * NumEltsGrowth + j);
309 }
310 }
311 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
312 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
313 return DAG.getVectorShuffle(VT: NVT, dl, N1, N2, Mask: NewMask);
314}
315
316/// Expands the ConstantFP node to a load from the constant pool.
317SDValue SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP) {
318 bool Extend = false;
319 SDLoc dl(CFP);
320
321 // If a FP immediate is precise when represented as a float and if the
322 // target can do an extending load from float to double, we put it into
323 // the constant pool as a float, even if it's is statically typed as a
324 // double. This shrinks FP constants and canonicalizes them for targets where
325 // an FP extending load is the same cost as a normal load (such as on the x87
326 // fp stack or PPC FP unit).
327 EVT VT = CFP->getValueType(ResNo: 0);
328 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
329
330 APFloat APF = CFP->getValueAPF();
331 EVT OrigVT = VT;
332 EVT SVT = VT;
333
334 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
335 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
336 if (!APF.isSignaling()) {
337 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
338 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
339 if (ConstantFPSDNode::isValueValidForType(VT: SVT, Val: APF) &&
340 // Only do this if the target has a native EXTLOAD instruction from
341 // smaller type.
342 TLI.isLoadLegal(
343 ValVT: OrigVT, MemVT: SVT,
344 Alignment: Align(DAG.getDataLayout().getPrefTypeAlign(
345 Ty: SVT.getTypeForEVT(Context&: *DAG.getContext()))),
346 AddrSpace: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction())
347 .getAddrSpace(),
348 ExtType: ISD::EXTLOAD, Atomic: false) &&
349 TLI.ShouldShrinkFPConstant(OrigVT)) {
350 Type *SType = SVT.getTypeForEVT(Context&: *DAG.getContext());
351 LLVMC = cast<ConstantFP>(Val: ConstantFoldCastOperand(
352 Opcode: Instruction::FPTrunc, C: LLVMC, DestTy: SType, DL: DAG.getDataLayout()));
353 VT = SVT;
354 Extend = true;
355 }
356 }
357 }
358
359 SDValue CPIdx =
360 DAG.getConstantPool(C: LLVMC, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
361 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
362 if (Extend) {
363 SDValue Result = DAG.getExtLoad(
364 ExtType: ISD::EXTLOAD, dl, VT: OrigVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
365 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: VT,
366 Alignment);
367 return Result;
368 }
369 SDValue Result = DAG.getLoad(
370 VT: OrigVT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
371 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
372 return Result;
373}
374
375/// Expands the Constant node to a load from the constant pool.
376SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
377 SDLoc dl(CP);
378 EVT VT = CP->getValueType(ResNo: 0);
379 SDValue CPIdx = DAG.getConstantPool(C: CP->getConstantIntValue(),
380 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
381 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
382 SDValue Result = DAG.getLoad(
383 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
384 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), Alignment);
385 return Result;
386}
387
388SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
389 SDValue Vec = Op.getOperand(i: 0);
390 SDValue Val = Op.getOperand(i: 1);
391 SDValue Idx = Op.getOperand(i: 2);
392 SDLoc dl(Op);
393
394 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Val&: Idx)) {
395 // SCALAR_TO_VECTOR requires that the type of the value being inserted
396 // match the element type of the vector being created, except for
397 // integers in which case the inserted value can be over width.
398 EVT EltVT = Vec.getValueType().getVectorElementType();
399 if (Val.getValueType() == EltVT ||
400 (EltVT.isInteger() && Val.getValueType().bitsGE(VT: EltVT))) {
401 SDValue ScVec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl,
402 VT: Vec.getValueType(), Operand: Val);
403
404 unsigned NumElts = Vec.getValueType().getVectorNumElements();
405 // We generate a shuffle of InVec and ScVec, so the shuffle mask
406 // should be 0,1,2,3,4,5... with the appropriate element replaced with
407 // elt 0 of the RHS.
408 SmallVector<int, 8> ShufOps;
409 for (unsigned i = 0; i != NumElts; ++i)
410 ShufOps.push_back(Elt: i != InsertPos->getZExtValue() ? i : NumElts);
411
412 return DAG.getVectorShuffle(VT: Vec.getValueType(), dl, N1: Vec, N2: ScVec, Mask: ShufOps);
413 }
414 }
415 return ExpandInsertToVectorThroughStack(Op);
416}
417
418SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
419 if (!ISD::isNormalStore(N: ST))
420 return SDValue();
421
422 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
423 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
424 // FIXME: move this to the DAG Combiner! Note that we can't regress due
425 // to phase ordering between legalized code and the dag combiner. This
426 // probably means that we need to integrate dag combiner and legalizer
427 // together.
428 // We generally can't do this one for long doubles.
429 SDValue Chain = ST->getChain();
430 SDValue Ptr = ST->getBasePtr();
431 SDValue Value = ST->getValue();
432 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
433 AAMDNodes AAInfo = ST->getAAInfo();
434 SDLoc dl(ST);
435
436 // Don't optimise TargetConstantFP
437 if (Value.getOpcode() == ISD::TargetConstantFP)
438 return SDValue();
439
440 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Value)) {
441 if (CFP->getValueType(ResNo: 0) == MVT::f32 &&
442 TLI.isTypeLegal(VT: MVT::i32)) {
443 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().
444 bitcastToAPInt().zextOrTrunc(width: 32),
445 DL: SDLoc(CFP), VT: MVT::i32);
446 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
447 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
448 }
449
450 if (CFP->getValueType(ResNo: 0) == MVT::f64 &&
451 !TLI.isFPImmLegal(Imm: CFP->getValueAPF(), VT: MVT::f64)) {
452 // If this target supports 64-bit registers, do a single 64-bit store.
453 if (TLI.isTypeLegal(VT: MVT::i64)) {
454 SDValue Con = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
455 zextOrTrunc(width: 64), DL: SDLoc(CFP), VT: MVT::i64);
456 return DAG.getStore(Chain, dl, Val: Con, Ptr, PtrInfo: ST->getPointerInfo(),
457 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
458 }
459
460 if (TLI.isTypeLegal(VT: MVT::i32) && !ST->isVolatile()) {
461 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
462 // stores. If the target supports neither 32- nor 64-bits, this
463 // xform is certainly not worth it.
464 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
465 SDValue Lo = DAG.getConstant(Val: IntVal.trunc(width: 32), DL: dl, VT: MVT::i32);
466 SDValue Hi = DAG.getConstant(Val: IntVal.lshr(shiftAmt: 32).trunc(width: 32), DL: dl, VT: MVT::i32);
467 if (DAG.getDataLayout().isBigEndian())
468 std::swap(a&: Lo, b&: Hi);
469
470 Lo = DAG.getStore(Chain, dl, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
471 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
472 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
473 Hi = DAG.getStore(Chain, dl, Val: Hi, Ptr,
474 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
475 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
476
477 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
478 }
479 }
480 }
481 return SDValue();
482}
483
484void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
485 StoreSDNode *ST = cast<StoreSDNode>(Val: Node);
486 SDValue Chain = ST->getChain();
487 SDValue Ptr = ST->getBasePtr();
488 SDLoc dl(Node);
489
490 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
491 MMOMetadata Metadata = ST->getMMOMetadataForSubAccess();
492
493 if (!ST->isTruncatingStore()) {
494 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
495 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
496 ReplaceNode(Old: ST, New: OptStore);
497 return;
498 }
499
500 SDValue Value = ST->getValue();
501 MVT VT = Value.getSimpleValueType();
502 switch (TLI.getOperationAction(Op: ISD::STORE, VT)) {
503 default: llvm_unreachable("This action is not supported yet!");
504 case TargetLowering::Legal: {
505 // If this is an unaligned store and the target doesn't support it,
506 // expand it.
507 EVT MemVT = ST->getMemoryVT();
508 const DataLayout &DL = DAG.getDataLayout();
509 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
510 MMO: *ST->getMemOperand())) {
511 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
512 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
513 ReplaceNode(Old: SDValue(ST, 0), New: Result);
514 } else
515 LLVM_DEBUG(dbgs() << "Legal store\n");
516 break;
517 }
518 case TargetLowering::Custom: {
519 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
520 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
521 if (Res && Res != SDValue(Node, 0))
522 ReplaceNode(Old: SDValue(Node, 0), New: Res);
523 return;
524 }
525 case TargetLowering::Promote: {
526 MVT NVT = TLI.getTypeToPromoteTo(Op: ISD::STORE, VT);
527 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
528 "Can only promote stores to same size type");
529 Value = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Value);
530 SDValue Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
531 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
532 ReplaceNode(Old: SDValue(Node, 0), New: Result);
533 break;
534 }
535 case TargetLowering::Expand:
536 assert(VT.isVector() && "Cannot expand a scalar store");
537 ReplaceNode(Old: SDValue(Node, 0), New: TLI.scalarizeVectorStore(ST, DAG));
538 break;
539 }
540 return;
541 }
542
543 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
544 SDValue Value = ST->getValue();
545 EVT StVT = ST->getMemoryVT();
546 TypeSize StWidth = StVT.getSizeInBits();
547 TypeSize StSize = StVT.getStoreSizeInBits();
548 auto &DL = DAG.getDataLayout();
549
550 if (StWidth != StSize) {
551 // Promote to a byte-sized store with upper bits zero if not
552 // storing an integral number of bytes. For example, promote
553 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
554 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StSize.getFixedValue());
555 Value = DAG.getZeroExtendInReg(Op: Value, DL: dl, VT: StVT);
556 SDValue Result =
557 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: NVT,
558 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
559 ReplaceNode(Old: SDValue(Node, 0), New: Result);
560 } else if (!StVT.isVector() && !isPowerOf2_64(Value: StWidth.getFixedValue())) {
561 // If not storing a power-of-2 number of bits, expand as two stores.
562 assert(!StVT.isVector() && "Unsupported truncstore!");
563 unsigned StWidthBits = StWidth.getFixedValue();
564 unsigned LogStWidth = Log2_32(Value: StWidthBits);
565 assert(LogStWidth < 32);
566 unsigned RoundWidth = 1 << LogStWidth;
567 assert(RoundWidth < StWidthBits);
568 unsigned ExtraWidth = StWidthBits - RoundWidth;
569 assert(ExtraWidth < RoundWidth);
570 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
571 "Store size not an integral number of bytes!");
572 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
573 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
574 SDValue Lo, Hi;
575 unsigned IncrementSize;
576
577 if (DL.isLittleEndian()) {
578 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
579 // Store the bottom RoundWidth bits.
580 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
581 SVT: RoundVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
582
583 // Store the remaining ExtraWidth bits.
584 IncrementSize = RoundWidth / 8;
585 Ptr =
586 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
587 Hi = DAG.getNode(
588 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
589 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Value.getValueType(), DL: dl));
590 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr,
591 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
592 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
593 } else {
594 // Big endian - avoid unaligned stores.
595 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
596 // Store the top RoundWidth bits.
597 Hi = DAG.getNode(
598 Opcode: ISD::SRL, DL: dl, VT: Value.getValueType(), N1: Value,
599 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Value.getValueType(), DL: dl));
600 Hi = DAG.getTruncStore(Chain, dl, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo(), SVT: RoundVT,
601 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
602
603 // Store the remaining ExtraWidth bits.
604 IncrementSize = RoundWidth / 8;
605 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr,
606 N2: DAG.getConstant(Val: IncrementSize, DL: dl,
607 VT: Ptr.getValueType()));
608 Lo = DAG.getTruncStore(Chain, dl, Val: Value, Ptr,
609 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize),
610 SVT: ExtraVT, Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
611 }
612
613 // The order of the stores doesn't matter.
614 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo, N2: Hi);
615 ReplaceNode(Old: SDValue(Node, 0), New: Result);
616 } else {
617 switch (TLI.getTruncStoreAction(ValVT: ST->getValue().getValueType(), MemVT: StVT,
618 Alignment: ST->getAlign(), AddrSpace: ST->getAddressSpace())) {
619 default:
620 llvm_unreachable("This action is not supported yet!");
621 case TargetLowering::Legal: {
622 EVT MemVT = ST->getMemoryVT();
623 // If this is an unaligned store and the target doesn't support it,
624 // expand it.
625 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
626 MMO: *ST->getMemOperand())) {
627 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
628 ReplaceNode(Old: SDValue(ST, 0), New: Result);
629 }
630 break;
631 }
632 case TargetLowering::Custom: {
633 SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG);
634 if (Res && Res != SDValue(Node, 0))
635 ReplaceNode(Old: SDValue(Node, 0), New: Res);
636 return;
637 }
638 case TargetLowering::Expand:
639 assert(!StVT.isVector() &&
640 "Vector Stores are handled in LegalizeVectorOps");
641
642 SDValue Result;
643
644 // TRUNCSTORE:i16 i32 -> STORE i16
645 if (TLI.isTypeLegal(VT: StVT)) {
646 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: StVT, Operand: Value);
647 Result = DAG.getStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(),
648 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
649 } else {
650 // The in-memory type isn't legal. Truncate to the type it would promote
651 // to, and then do a truncstore.
652 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl,
653 VT: TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StVT),
654 Operand: Value);
655 Result =
656 DAG.getTruncStore(Chain, dl, Val: Value, Ptr, PtrInfo: ST->getPointerInfo(), SVT: StVT,
657 Alignment: ST->getBaseAlign(), MMOFlags, Metadata);
658 }
659
660 ReplaceNode(Old: SDValue(Node, 0), New: Result);
661 break;
662 }
663 }
664}
665
666void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
667 LoadSDNode *LD = cast<LoadSDNode>(Val: Node);
668 SDValue Chain = LD->getChain(); // The chain.
669 SDValue Ptr = LD->getBasePtr(); // The base pointer.
670 SDValue Value; // The value returned by the load op.
671 SDLoc dl(Node);
672
673 ISD::LoadExtType ExtType = LD->getExtensionType();
674 if (ExtType == ISD::NON_EXTLOAD) {
675 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
676 MVT VT = Node->getSimpleValueType(ResNo: 0);
677 SDValue RVal = SDValue(Node, 0);
678 SDValue RChain = SDValue(Node, 1);
679
680 switch (TLI.getOperationAction(Op: Node->getOpcode(), VT)) {
681 default: llvm_unreachable("This action is not supported yet!");
682 case TargetLowering::Legal: {
683 EVT MemVT = LD->getMemoryVT();
684 const DataLayout &DL = DAG.getDataLayout();
685 // If this is an unaligned load and the target doesn't support it,
686 // expand it.
687 if (!TLI.allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL, VT: MemVT,
688 MMO: *LD->getMemOperand())) {
689 std::tie(args&: RVal, args&: RChain) = TLI.expandUnalignedLoad(LD, DAG);
690 }
691 break;
692 }
693 case TargetLowering::Custom:
694 if (SDValue Res = TLI.LowerOperation(Op: RVal, DAG)) {
695 RVal = Res;
696 RChain = Res.getValue(R: 1);
697 }
698 break;
699
700 case TargetLowering::Promote: {
701 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT);
702 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
703 "Can only promote loads to same size type");
704
705 // If the range metadata type does not match the legalized memory
706 // operation type, remove the range metadata.
707 if (const MDNode *MD = LD->getRanges()) {
708 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
709 if (Lower->getBitWidth() != NVT.getScalarSizeInBits() ||
710 !NVT.isInteger())
711 LD->getMemOperand()->clearRanges();
712 }
713 SDValue Res = DAG.getLoad(VT: NVT, dl, Chain, Ptr, MMO: LD->getMemOperand());
714 RVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Res);
715 RChain = Res.getValue(R: 1);
716 break;
717 }
718 case TargetLowering::Expand:
719 assert(VT.isVector() && "Cannot expand a scalar load");
720 std::tie(args&: RVal, args&: RChain) = TLI.scalarizeVectorLoad(LD, DAG);
721 break;
722 }
723 if (RChain.getNode() != Node) {
724 assert(RVal.getNode() != Node && "Load must be completely replaced");
725 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: RVal);
726 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: RChain);
727 if (UpdatedNodes) {
728 UpdatedNodes->insert(X: RVal.getNode());
729 UpdatedNodes->insert(X: RChain.getNode());
730 }
731 ReplacedNode(N: Node);
732 }
733 return;
734 }
735
736 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
737 EVT SrcVT = LD->getMemoryVT();
738 TypeSize SrcWidth = SrcVT.getSizeInBits();
739 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
740 MMOMetadata Metadata = LD->getMMOMetadataForSubAccess();
741
742 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
743 // Some targets pretend to have an i1 loading operation, and actually
744 // load an i8. This trick is correct for ZEXTLOAD because the top 7
745 // bits are guaranteed to be zero; it helps the optimizers understand
746 // that these bits are zero. It is also useful for EXTLOAD, since it
747 // tells the optimizers that those bits are undefined. It would be
748 // nice to have an effective generic way of getting these benefits...
749 // Until such a way is found, don't insist on promoting i1 here.
750 (SrcVT != MVT::i1 ||
751 TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: MVT::i1, Alignment: LD->getAlign(),
752 AddrSpace: LD->getAddressSpace(), ExtType,
753 Atomic: false) == TargetLowering::Promote)) {
754 // Promote to a byte-sized load if not loading an integral number of
755 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
756 unsigned NewWidth = SrcVT.getStoreSizeInBits();
757 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewWidth);
758 SDValue Ch;
759
760 // The extra bits are guaranteed to be zero, since we stored them that
761 // way. A zext load from NVT thus automatically gives zext from SrcVT.
762
763 ISD::LoadExtType NewExtType =
764 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
765
766 SDValue Result = DAG.getExtLoad(ExtType: NewExtType, dl, VT: Node->getValueType(ResNo: 0),
767 Chain, Ptr, PtrInfo: LD->getPointerInfo(), MemVT: NVT,
768 Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
769
770 Ch = Result.getValue(R: 1); // The chain.
771
772 if (ExtType == ISD::SEXTLOAD)
773 // Having the top bits zero doesn't help when sign extending.
774 Result = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
775 VT: Result.getValueType(),
776 N1: Result, N2: DAG.getValueType(SrcVT));
777 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
778 // All the top bits are guaranteed to be zero - inform the optimizers.
779 Result = DAG.getNode(Opcode: ISD::AssertZext, DL: dl,
780 VT: Result.getValueType(), N1: Result,
781 N2: DAG.getValueType(SrcVT));
782
783 Value = Result;
784 Chain = Ch;
785 } else if (!isPowerOf2_64(Value: SrcWidth.getKnownMinValue())) {
786 // If not loading a power-of-2 number of bits, expand as two loads.
787 assert(!SrcVT.isVector() && "Unsupported extload!");
788 unsigned SrcWidthBits = SrcWidth.getFixedValue();
789 unsigned LogSrcWidth = Log2_32(Value: SrcWidthBits);
790 assert(LogSrcWidth < 32);
791 unsigned RoundWidth = 1 << LogSrcWidth;
792 assert(RoundWidth < SrcWidthBits);
793 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
794 assert(ExtraWidth < RoundWidth);
795 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
796 "Load size not an integral number of bytes!");
797 EVT RoundVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RoundWidth);
798 EVT ExtraVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ExtraWidth);
799 SDValue Lo, Hi, Ch;
800 unsigned IncrementSize;
801 auto &DL = DAG.getDataLayout();
802
803 if (DL.isLittleEndian()) {
804 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
805 // Load the bottom RoundWidth bits.
806 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
807 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
808 MMOFlags, Metadata);
809
810 // Load the remaining ExtraWidth bits.
811 IncrementSize = RoundWidth / 8;
812 Ptr =
813 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
814 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
815 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
816 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
817
818 // Build a factor node to remember that this load is independent of
819 // the other one.
820 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
821 N2: Hi.getValue(R: 1));
822
823 // Move the top bits to the right place.
824 Hi = DAG.getNode(
825 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
826 N2: DAG.getShiftAmountConstant(Val: RoundWidth, VT: Hi.getValueType(), DL: dl));
827
828 // Join the hi and lo parts.
829 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
830 } else {
831 // Big endian - avoid unaligned loads.
832 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
833 // Load the top RoundWidth bits.
834 Hi = DAG.getExtLoad(ExtType, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
835 PtrInfo: LD->getPointerInfo(), MemVT: RoundVT, Alignment: LD->getBaseAlign(),
836 MMOFlags, Metadata);
837
838 // Load the remaining ExtraWidth bits.
839 IncrementSize = RoundWidth / 8;
840 Ptr =
841 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize), DL: dl);
842 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: Node->getValueType(ResNo: 0), Chain, Ptr,
843 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
844 MemVT: ExtraVT, Alignment: LD->getBaseAlign(), MMOFlags, Metadata);
845
846 // Build a factor node to remember that this load is independent of
847 // the other one.
848 Ch = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
849 N2: Hi.getValue(R: 1));
850
851 // Move the top bits to the right place.
852 Hi = DAG.getNode(
853 Opcode: ISD::SHL, DL: dl, VT: Hi.getValueType(), N1: Hi,
854 N2: DAG.getShiftAmountConstant(Val: ExtraWidth, VT: Hi.getValueType(), DL: dl));
855
856 // Join the hi and lo parts.
857 Value = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Lo, N2: Hi);
858 }
859
860 Chain = Ch;
861 } else {
862 bool isCustom = false;
863 switch (TLI.getLoadAction(ValVT: Node->getValueType(ResNo: 0), MemVT: SrcVT.getSimpleVT(),
864 Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(), ExtType,
865 Atomic: false)) {
866 default:
867 llvm_unreachable("This action is not supported yet!");
868 case TargetLowering::Custom:
869 isCustom = true;
870 [[fallthrough]];
871 case TargetLowering::Legal:
872 Value = SDValue(Node, 0);
873 Chain = SDValue(Node, 1);
874
875 if (isCustom) {
876 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
877 Value = Res;
878 Chain = Res.getValue(R: 1);
879 }
880 } else {
881 // If this is an unaligned load and the target doesn't support it,
882 // expand it.
883 EVT MemVT = LD->getMemoryVT();
884 const DataLayout &DL = DAG.getDataLayout();
885 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL, VT: MemVT,
886 MMO: *LD->getMemOperand())) {
887 std::tie(args&: Value, args&: Chain) = TLI.expandUnalignedLoad(LD, DAG);
888 }
889 }
890 break;
891
892 case TargetLowering::Expand: {
893 EVT DestVT = Node->getValueType(ResNo: 0);
894 if (!TLI.isLoadLegal(ValVT: DestVT, MemVT: SrcVT, Alignment: LD->getAlign(), AddrSpace: LD->getAddressSpace(),
895 ExtType: ISD::EXTLOAD, Atomic: false)) {
896 // If the source type is not legal, see if there is a legal extload to
897 // an intermediate type that we can then extend further.
898 EVT LoadVT =
899 TLI.getRegisterType(Context&: *DAG.getContext(), VT: SrcVT.getSimpleVT());
900 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
901 (TLI.isTypeLegal(VT: SrcVT) || // Same as SrcVT == LoadVT?
902 TLI.isLoadLegal(ValVT: LoadVT, MemVT: SrcVT, Alignment: LD->getAlign(),
903 AddrSpace: LD->getAddressSpace(), ExtType, Atomic: false))) {
904 // If we are loading a legal type, this is a non-extload followed by a
905 // full extend.
906 ISD::LoadExtType MidExtType =
907 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
908
909 SDValue Load = DAG.getExtLoad(ExtType: MidExtType, dl, VT: LoadVT, Chain, Ptr,
910 MemVT: SrcVT, MMO: LD->getMemOperand());
911 unsigned ExtendOp =
912 ISD::getExtForLoadExtType(IsFP: SrcVT.isFloatingPoint(), ExtType);
913 Value = DAG.getNode(Opcode: ExtendOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Load);
914 Chain = Load.getValue(R: 1);
915 break;
916 }
917
918 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
919 // normal undefined upper bits behavior to allow using an in-reg extend
920 // with the illegal FP type, so load as an integer and do the
921 // from-integer conversion.
922 EVT SVT = SrcVT.getScalarType();
923 if (SVT == MVT::f16 || SVT == MVT::bf16) {
924 EVT ISrcVT = SrcVT.changeTypeToInteger();
925 EVT IDestVT = DestVT.changeTypeToInteger();
926 EVT ILoadVT =
927 TLI.getRegisterType(Context&: *DAG.getContext(), VT: IDestVT.getSimpleVT());
928
929 SDValue Result = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: ILoadVT, Chain,
930 Ptr, MemVT: ISrcVT, MMO: LD->getMemOperand());
931 Value =
932 DAG.getNode(Opcode: SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
933 DL: dl, VT: DestVT, Operand: Result);
934 Chain = Result.getValue(R: 1);
935 break;
936 }
937 }
938
939 assert(!SrcVT.isVector() &&
940 "Vector Loads are handled in LegalizeVectorOps");
941
942 // FIXME: This does not work for vectors on most targets. Sign-
943 // and zero-extend operations are currently folded into extending
944 // loads, whether they are legal or not, and then we end up here
945 // without any support for legalizing them.
946 assert(ExtType != ISD::EXTLOAD &&
947 "EXTLOAD should always be supported!");
948 // Turn the unsupported load into an EXTLOAD followed by an
949 // explicit zero/sign extend inreg.
950 SDValue Result = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl,
951 VT: Node->getValueType(ResNo: 0),
952 Chain, Ptr, MemVT: SrcVT,
953 MMO: LD->getMemOperand());
954 SDValue ValRes;
955 if (ExtType == ISD::SEXTLOAD)
956 ValRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl,
957 VT: Result.getValueType(),
958 N1: Result, N2: DAG.getValueType(SrcVT));
959 else
960 ValRes = DAG.getZeroExtendInReg(Op: Result, DL: dl, VT: SrcVT);
961 Value = ValRes;
962 Chain = Result.getValue(R: 1);
963 break;
964 }
965 }
966 }
967
968 // Since loads produce two values, make sure to remember that we legalized
969 // both of them.
970 if (Chain.getNode() != Node) {
971 assert(Value.getNode() != Node && "Load must be completely replaced");
972 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Value);
973 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
974 if (UpdatedNodes) {
975 UpdatedNodes->insert(X: Value.getNode());
976 UpdatedNodes->insert(X: Chain.getNode());
977 }
978 ReplacedNode(N: Node);
979 }
980}
981
982/// Return a legal replacement for the given operation, with all legal operands.
983void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
984 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
985
986 // Allow illegal target nodes and illegal registers.
987 if (Node->getOpcode() == ISD::TargetConstant ||
988 Node->getOpcode() == ISD::Register)
989 return;
990
991#ifndef NDEBUG
992 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
993 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
994 TargetLowering::TypeLegal &&
995 "Unexpected illegal type!");
996
997 for (const SDValue &Op : Node->op_values())
998 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
999 TargetLowering::TypeLegal ||
1000 Op.getOpcode() == ISD::TargetConstant ||
1001 Op.getOpcode() == ISD::Register) &&
1002 "Unexpected illegal type!");
1003#endif
1004
1005 // Figure out the correct action; the way to query this varies by opcode
1006 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1007 bool SimpleFinishLegalizing = true;
1008 switch (Node->getOpcode()) {
1009 case ISD::INTRINSIC_W_CHAIN:
1010 case ISD::INTRINSIC_WO_CHAIN:
1011 case ISD::INTRINSIC_VOID:
1012 case ISD::STACKSAVE:
1013 case ISD::STACKADDRESS:
1014 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1015 break;
1016 case ISD::GET_DYNAMIC_AREA_OFFSET:
1017 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1018 VT: Node->getValueType(ResNo: 0));
1019 break;
1020 case ISD::VAARG:
1021 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1022 VT: Node->getValueType(ResNo: 0));
1023 if (Action != TargetLowering::Promote)
1024 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::Other);
1025 break;
1026 case ISD::SET_FPENV:
1027 case ISD::SET_FPMODE:
1028 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1029 VT: Node->getOperand(Num: 1).getValueType());
1030 break;
1031 case ISD::FP_TO_FP16:
1032 case ISD::FP_TO_BF16:
1033 case ISD::SINT_TO_FP:
1034 case ISD::UINT_TO_FP:
1035 case ISD::EXTRACT_VECTOR_ELT:
1036 case ISD::LROUND:
1037 case ISD::LLROUND:
1038 case ISD::LRINT:
1039 case ISD::LLRINT:
1040 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1041 VT: Node->getOperand(Num: 0).getValueType());
1042 break;
1043 case ISD::STRICT_FP_TO_FP16:
1044 case ISD::STRICT_FP_TO_BF16:
1045 case ISD::STRICT_SINT_TO_FP:
1046 case ISD::STRICT_UINT_TO_FP:
1047 case ISD::STRICT_LRINT:
1048 case ISD::STRICT_LLRINT:
1049 case ISD::STRICT_LROUND:
1050 case ISD::STRICT_LLROUND:
1051 // These pseudo-ops are the same as the other STRICT_ ops except
1052 // they are registered with setOperationAction() using the input type
1053 // instead of the output type.
1054 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1055 VT: Node->getOperand(Num: 1).getValueType());
1056 break;
1057 case ISD::SIGN_EXTEND_INREG: {
1058 EVT InnerType = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
1059 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: InnerType);
1060 break;
1061 }
1062 case ISD::ATOMIC_STORE:
1063 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1064 VT: Node->getOperand(Num: 1).getValueType());
1065 break;
1066 case ISD::ATOMIC_SWAP:
1067 case ISD::ATOMIC_LOAD_ADD:
1068 case ISD::ATOMIC_LOAD_SUB:
1069 case ISD::ATOMIC_LOAD_AND:
1070 case ISD::ATOMIC_LOAD_CLR:
1071 case ISD::ATOMIC_LOAD_OR:
1072 case ISD::ATOMIC_LOAD_XOR:
1073 case ISD::ATOMIC_LOAD_NAND:
1074 case ISD::ATOMIC_LOAD_MIN:
1075 case ISD::ATOMIC_LOAD_MAX:
1076 case ISD::ATOMIC_LOAD_UMIN:
1077 case ISD::ATOMIC_LOAD_UMAX:
1078 case ISD::ATOMIC_CMP_SWAP:
1079 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1080 VT: cast<AtomicSDNode>(Val: Node)->getMemoryVT());
1081 break;
1082 case ISD::SELECT_CC:
1083 case ISD::STRICT_FSETCC:
1084 case ISD::STRICT_FSETCCS:
1085 case ISD::SETCC:
1086 case ISD::SETCCCARRY:
1087 case ISD::BR_CC: {
1088 unsigned Opc = Node->getOpcode();
1089 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1090 : Opc == ISD::STRICT_FSETCC ? 3
1091 : Opc == ISD::STRICT_FSETCCS ? 3
1092 : Opc == ISD::SETCCCARRY ? 3
1093 : Opc == ISD::SETCC ? 2
1094 : 1;
1095 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1096 : Opc == ISD::STRICT_FSETCC ? 1
1097 : Opc == ISD::STRICT_FSETCCS ? 1
1098 : 0;
1099 MVT OpVT = Node->getOperand(Num: CompareOperand).getSimpleValueType();
1100 ISD::CondCode CCCode = Node->getOperandCondCode(Num: CCOperand);
1101 Action = TLI.getCondCodeAction(CC: CCCode, VT: OpVT);
1102 if (Action == TargetLowering::Legal) {
1103 if (Node->getOpcode() == ISD::SELECT_CC)
1104 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1105 VT: Node->getValueType(ResNo: 0));
1106 else
1107 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: OpVT);
1108 }
1109 break;
1110 }
1111 case ISD::LOAD:
1112 case ISD::STORE:
1113 // FIXME: Model these properly. LOAD and STORE are complicated, and
1114 // STORE expects the unlegalized operand in some cases.
1115 SimpleFinishLegalizing = false;
1116 break;
1117 case ISD::CALLSEQ_START:
1118 case ISD::CALLSEQ_END:
1119 // FIXME: This shouldn't be necessary. These nodes have special properties
1120 // dealing with the recursive nature of legalization. Removing this
1121 // special case should be done as part of making LegalizeDAG non-recursive.
1122 SimpleFinishLegalizing = false;
1123 break;
1124 case ISD::EXTRACT_ELEMENT:
1125 case ISD::GET_ROUNDING:
1126 case ISD::MERGE_VALUES:
1127 case ISD::EH_RETURN:
1128 case ISD::FRAME_TO_ARGS_OFFSET:
1129 case ISD::EH_DWARF_CFA:
1130 case ISD::EH_SJLJ_SETJMP:
1131 case ISD::EH_SJLJ_LONGJMP:
1132 case ISD::EH_SJLJ_SETUP_DISPATCH:
1133 // These operations lie about being legal: when they claim to be legal,
1134 // they should actually be expanded.
1135 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1136 if (Action == TargetLowering::Legal)
1137 Action = TargetLowering::Expand;
1138 break;
1139 case ISD::INIT_TRAMPOLINE:
1140 case ISD::ADJUST_TRAMPOLINE:
1141 case ISD::FRAMEADDR:
1142 case ISD::RETURNADDR:
1143 case ISD::ADDROFRETURNADDR:
1144 case ISD::SPONENTRY:
1145 // These operations lie about being legal: when they claim to be legal,
1146 // they should actually be custom-lowered.
1147 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1148 if (Action == TargetLowering::Legal)
1149 Action = TargetLowering::Custom;
1150 break;
1151 case ISD::CLEAR_CACHE:
1152 // This operation is typically going to be LibCall unless the target wants
1153 // something differrent.
1154 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1155 break;
1156 case ISD::READCYCLECOUNTER:
1157 case ISD::READSTEADYCOUNTER:
1158 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1159 // legalization might have expanded that to several smaller types.
1160 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: MVT::i64);
1161 break;
1162 case ISD::READ_REGISTER:
1163 case ISD::WRITE_REGISTER:
1164 // Named register is legal in the DAG, but blocked by register name
1165 // selection if not implemented by target (to chose the correct register)
1166 // They'll be converted to Copy(To/From)Reg.
1167 Action = TargetLowering::Legal;
1168 break;
1169 case ISD::UBSANTRAP:
1170 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1171 if (Action == TargetLowering::Expand) {
1172 // replace ISD::UBSANTRAP with ISD::TRAP
1173 SDValue NewVal;
1174 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1175 N: Node->getOperand(Num: 0));
1176 ReplaceNode(Old: Node, New: NewVal.getNode());
1177 LegalizeOp(Node: NewVal.getNode());
1178 return;
1179 }
1180 break;
1181 case ISD::DEBUGTRAP:
1182 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1183 if (Action == TargetLowering::Expand) {
1184 // replace ISD::DEBUGTRAP with ISD::TRAP
1185 SDValue NewVal;
1186 NewVal = DAG.getNode(Opcode: ISD::TRAP, DL: SDLoc(Node), VTList: Node->getVTList(),
1187 N: Node->getOperand(Num: 0));
1188 ReplaceNode(Old: Node, New: NewVal.getNode());
1189 LegalizeOp(Node: NewVal.getNode());
1190 return;
1191 }
1192 break;
1193 case ISD::SADDSAT:
1194 case ISD::UADDSAT:
1195 case ISD::SSUBSAT:
1196 case ISD::USUBSAT:
1197 case ISD::SSHLSAT:
1198 case ISD::USHLSAT:
1199 case ISD::SCMP:
1200 case ISD::UCMP:
1201 case ISD::FP_TO_SINT_SAT:
1202 case ISD::FP_TO_UINT_SAT:
1203 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1204 break;
1205 case ISD::SMULFIX:
1206 case ISD::SMULFIXSAT:
1207 case ISD::UMULFIX:
1208 case ISD::UMULFIXSAT:
1209 case ISD::SDIVFIX:
1210 case ISD::SDIVFIXSAT:
1211 case ISD::UDIVFIX:
1212 case ISD::UDIVFIXSAT: {
1213 unsigned Scale = Node->getConstantOperandVal(Num: 2);
1214 Action = TLI.getFixedPointOperationAction(Op: Node->getOpcode(),
1215 VT: Node->getValueType(ResNo: 0), Scale);
1216 break;
1217 }
1218 case ISD::MSCATTER:
1219 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1220 VT: cast<MaskedScatterSDNode>(Val: Node)->getValue().getValueType());
1221 break;
1222 case ISD::MSTORE:
1223 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1224 VT: cast<MaskedStoreSDNode>(Val: Node)->getValue().getValueType());
1225 break;
1226 case ISD::VP_SCATTER:
1227 Action = TLI.getOperationAction(
1228 Op: Node->getOpcode(),
1229 VT: cast<VPScatterSDNode>(Val: Node)->getValue().getValueType());
1230 break;
1231 case ISD::VP_STORE:
1232 Action = TLI.getOperationAction(
1233 Op: Node->getOpcode(),
1234 VT: cast<VPStoreSDNode>(Val: Node)->getValue().getValueType());
1235 break;
1236 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1237 Action = TLI.getOperationAction(
1238 Op: Node->getOpcode(),
1239 VT: cast<VPStridedStoreSDNode>(Val: Node)->getValue().getValueType());
1240 break;
1241 case ISD::VECREDUCE_FADD:
1242 case ISD::VECREDUCE_FMUL:
1243 case ISD::VECREDUCE_ADD:
1244 case ISD::VECREDUCE_MUL:
1245 case ISD::VECREDUCE_AND:
1246 case ISD::VECREDUCE_OR:
1247 case ISD::VECREDUCE_XOR:
1248 case ISD::VECREDUCE_SMAX:
1249 case ISD::VECREDUCE_SMIN:
1250 case ISD::VECREDUCE_UMAX:
1251 case ISD::VECREDUCE_UMIN:
1252 case ISD::VECREDUCE_FMAX:
1253 case ISD::VECREDUCE_FMIN:
1254 case ISD::VECREDUCE_FMAXIMUM:
1255 case ISD::VECREDUCE_FMINIMUM:
1256 case ISD::VECREDUCE_FMAXIMUMNUM:
1257 case ISD::VECREDUCE_FMINIMUMNUM:
1258 case ISD::IS_FPCLASS:
1259 Action = TLI.getOperationAction(
1260 Op: Node->getOpcode(), VT: Node->getOperand(Num: 0).getValueType());
1261 break;
1262 case ISD::VECREDUCE_SEQ_FADD:
1263 case ISD::VECREDUCE_SEQ_FMUL:
1264 case ISD::VP_REDUCE_FADD:
1265 case ISD::VP_REDUCE_FMUL:
1266 case ISD::VP_REDUCE_ADD:
1267 case ISD::VP_REDUCE_MUL:
1268 case ISD::VP_REDUCE_AND:
1269 case ISD::VP_REDUCE_OR:
1270 case ISD::VP_REDUCE_XOR:
1271 case ISD::VP_REDUCE_SMAX:
1272 case ISD::VP_REDUCE_SMIN:
1273 case ISD::VP_REDUCE_UMAX:
1274 case ISD::VP_REDUCE_UMIN:
1275 case ISD::VP_REDUCE_FMAX:
1276 case ISD::VP_REDUCE_FMIN:
1277 case ISD::VP_REDUCE_FMAXIMUM:
1278 case ISD::VP_REDUCE_FMINIMUM:
1279 case ISD::VP_REDUCE_SEQ_FADD:
1280 case ISD::VP_REDUCE_SEQ_FMUL:
1281 Action = TLI.getOperationAction(
1282 Op: Node->getOpcode(), VT: Node->getOperand(Num: 1).getValueType());
1283 break;
1284 case ISD::CTTZ_ELTS:
1285 case ISD::CTTZ_ELTS_ZERO_POISON:
1286 case ISD::VP_CTTZ_ELTS:
1287 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
1288 Action = TLI.getOperationAction(Op: Node->getOpcode(),
1289 VT: Node->getOperand(Num: 0).getValueType());
1290 break;
1291 case ISD::VECTOR_INTERLEAVE:
1292 case ISD::VECTOR_DEINTERLEAVE:
1293 Action = TLI.getVectorInterleaveAction(
1294 Opc: Node->getOpcode(), Factor: Node->getNumOperands(), VT: Node->getValueType(ResNo: 0));
1295 break;
1296 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1297 Action = TLI.getOperationAction(
1298 Op: Node->getOpcode(),
1299 VT: cast<MaskedHistogramSDNode>(Val: Node)->getIndex().getValueType());
1300 break;
1301 default:
1302 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1303 Action = TLI.getCustomOperationAction(Op&: *Node);
1304 } else {
1305 Action = TLI.getOperationAction(Op: Node->getOpcode(), VT: Node->getValueType(ResNo: 0));
1306 }
1307 break;
1308 }
1309
1310 if (SimpleFinishLegalizing) {
1311 SDNode *NewNode = Node;
1312 switch (Node->getOpcode()) {
1313 default: break;
1314 case ISD::SHL:
1315 case ISD::SRL:
1316 case ISD::SRA:
1317 case ISD::ROTL:
1318 case ISD::ROTR:
1319 case ISD::SSHLSAT:
1320 case ISD::USHLSAT: {
1321 // Legalizing shifts/rotates requires adjusting the shift amount
1322 // to the appropriate width.
1323 SDValue Op0 = Node->getOperand(Num: 0);
1324 SDValue Op1 = Node->getOperand(Num: 1);
1325 if (!Op1.getValueType().isVector()) {
1326 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op1);
1327 // The getShiftAmountOperand() may create a new operand node or
1328 // return the existing one. If new operand is created we need
1329 // to update the parent node.
1330 // Do not try to legalize SAO here! It will be automatically legalized
1331 // in the next round.
1332 if (SAO != Op1)
1333 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: SAO);
1334 }
1335 break;
1336 }
1337 case ISD::FSHL:
1338 case ISD::FSHR:
1339 case ISD::SRL_PARTS:
1340 case ISD::SRA_PARTS:
1341 case ISD::SHL_PARTS: {
1342 // Legalizing shifts/rotates requires adjusting the shift amount
1343 // to the appropriate width.
1344 SDValue Op0 = Node->getOperand(Num: 0);
1345 SDValue Op1 = Node->getOperand(Num: 1);
1346 SDValue Op2 = Node->getOperand(Num: 2);
1347 if (!Op2.getValueType().isVector()) {
1348 SDValue SAO = DAG.getShiftAmountOperand(LHSTy: Op0.getValueType(), Op: Op2);
1349 // The getShiftAmountOperand() may create a new operand node or
1350 // return the existing one. If new operand is created we need
1351 // to update the parent node.
1352 if (SAO != Op2)
1353 NewNode = DAG.UpdateNodeOperands(N: Node, Op1: Op0, Op2: Op1, Op3: SAO);
1354 }
1355 break;
1356 }
1357 }
1358
1359 if (NewNode != Node) {
1360 ReplaceNode(Old: Node, New: NewNode);
1361 Node = NewNode;
1362 }
1363 switch (Action) {
1364 case TargetLowering::Legal:
1365 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1366 return;
1367 case TargetLowering::Custom:
1368 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1369 // FIXME: The handling for custom lowering with multiple results is
1370 // a complete mess.
1371 if (SDValue Res = TLI.LowerOperation(Op: SDValue(Node, 0), DAG)) {
1372 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1373 return;
1374
1375 if (Node->getNumValues() == 1) {
1376 // Verify the new types match the original. Glue is waived because
1377 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1378 assert((Res.getValueType() == Node->getValueType(0) ||
1379 Node->getValueType(0) == MVT::Glue) &&
1380 "Type mismatch for custom legalized operation");
1381 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1382 // We can just directly replace this node with the lowered value.
1383 ReplaceNode(Old: SDValue(Node, 0), New: Res);
1384 return;
1385 }
1386
1387 SmallVector<SDValue, 8> ResultVals;
1388 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1389 // Verify the new types match the original. Glue is waived because
1390 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1391 assert((Res->getValueType(i) == Node->getValueType(i) ||
1392 Node->getValueType(i) == MVT::Glue) &&
1393 "Type mismatch for custom legalized operation");
1394 ResultVals.push_back(Elt: Res.getValue(R: i));
1395 }
1396 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1397 ReplaceNode(Old: Node, New: ResultVals.data());
1398 return;
1399 }
1400 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1401 [[fallthrough]];
1402 case TargetLowering::Expand:
1403 if (ExpandNode(Node))
1404 return;
1405 [[fallthrough]];
1406 case TargetLowering::LibCall:
1407 ConvertNodeToLibcall(Node);
1408 return;
1409 case TargetLowering::Promote:
1410 PromoteNode(Node);
1411 return;
1412 }
1413 }
1414
1415 switch (Node->getOpcode()) {
1416 default:
1417#ifndef NDEBUG
1418 dbgs() << "NODE: ";
1419 Node->dump( &DAG);
1420 dbgs() << "\n";
1421#endif
1422 llvm_unreachable("Do not know how to legalize this operator!");
1423
1424 case ISD::CALLSEQ_START:
1425 case ISD::CALLSEQ_END:
1426 break;
1427 case ISD::LOAD:
1428 return LegalizeLoadOps(Node);
1429 case ISD::STORE:
1430 return LegalizeStoreOps(Node);
1431 }
1432}
1433
1434SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1435 SDValue Vec = Op.getOperand(i: 0);
1436 SDValue Idx = Op.getOperand(i: 1);
1437 SDLoc dl(Op);
1438
1439 // Before we generate a new store to a temporary stack slot, see if there is
1440 // already one that we can use. There often is because when we scalarize
1441 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1442 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1443 // the vector. If all are expanded here, we don't want one store per vector
1444 // element.
1445
1446 // Caches for hasPredecessorHelper
1447 SmallPtrSet<const SDNode *, 32> Visited;
1448 SmallVector<const SDNode *, 16> Worklist;
1449 Visited.insert(Ptr: Op.getNode());
1450 Worklist.push_back(Elt: Idx.getNode());
1451 SDValue StackPtr, Ch;
1452 for (SDNode *User : Vec.getNode()->users()) {
1453 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: User)) {
1454 if (ST->isIndexed() || ST->isTruncatingStore() ||
1455 ST->getValue() != Vec)
1456 continue;
1457
1458 // Make sure that nothing else could have stored into the destination of
1459 // this store.
1460 if (!ST->getChain().reachesChainWithoutSideEffects(Dest: DAG.getEntryNode()))
1461 continue;
1462
1463 // If the index is dependent on the store we will introduce a cycle when
1464 // creating the load (the load uses the index, and by replacing the chain
1465 // we will make the index dependent on the load). Also, the store might be
1466 // dependent on the extractelement and introduce a cycle when creating
1467 // the load.
1468 if (SDNode::hasPredecessorHelper(N: ST, Visited, Worklist) ||
1469 ST->hasPredecessor(N: Op.getNode()))
1470 continue;
1471
1472 StackPtr = ST->getBasePtr();
1473 Ch = SDValue(ST, 0);
1474 break;
1475 }
1476 }
1477
1478 EVT VecVT = Vec.getValueType();
1479
1480 if (!Ch.getNode()) {
1481 // Store the value to a temporary stack slot, then LOAD the returned part.
1482 StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1483 MachineMemOperand *StoreMMO = getStackAlignedMMO(
1484 StackPtr, MF&: DAG.getMachineFunction(), isObjectScalable: VecVT.isScalableVector());
1485 Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, MMO: StoreMMO);
1486 }
1487
1488 SDValue NewLoad;
1489 Align ElementAlignment =
1490 std::min(a: cast<StoreSDNode>(Val&: Ch)->getAlign(),
1491 b: DAG.getDataLayout().getPrefTypeAlign(
1492 Ty: Op.getValueType().getTypeForEVT(Context&: *DAG.getContext())));
1493
1494 if (Op.getValueType().isVector()) {
1495 StackPtr = TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT,
1496 SubVecVT: Op.getValueType(), Index: Idx);
1497 NewLoad = DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr,
1498 PtrInfo: MachinePointerInfo(), Alignment: ElementAlignment);
1499 } else {
1500 StackPtr = TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1501 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: Op.getValueType(), Chain: Ch, Ptr: StackPtr,
1502 PtrInfo: MachinePointerInfo(), MemVT: VecVT.getVectorElementType(),
1503 Alignment: ElementAlignment);
1504 }
1505
1506 // Replace the chain going out of the store, by the one out of the load.
1507 DAG.ReplaceAllUsesOfValueWith(From: Ch, To: SDValue(NewLoad.getNode(), 1));
1508
1509 // We introduced a cycle though, so update the loads operands, making sure
1510 // to use the original store's chain as an incoming chain.
1511 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->ops());
1512 NewLoadOperands[0] = Ch;
1513 NewLoad =
1514 SDValue(DAG.UpdateNodeOperands(N: NewLoad.getNode(), Ops: NewLoadOperands), 0);
1515 return NewLoad;
1516}
1517
1518SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1519 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1520
1521 SDValue Vec = Op.getOperand(i: 0);
1522 SDValue Part = Op.getOperand(i: 1);
1523 SDValue Idx = Op.getOperand(i: 2);
1524 SDLoc dl(Op);
1525
1526 // Store the value to a temporary stack slot, then LOAD the returned part.
1527 EVT VecVT = Vec.getValueType();
1528 EVT PartVT = Part.getValueType();
1529 SDValue StackPtr = DAG.CreateStackTemporary(VT: VecVT);
1530 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1531 MachinePointerInfo PtrInfo =
1532 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1533
1534 // First store the whole vector.
1535 Align BaseVecAlignment =
1536 DAG.getMachineFunction().getFrameInfo().getObjectAlign(ObjectIdx: FI);
1537 SDValue Ch = DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Vec, Ptr: StackPtr, PtrInfo,
1538 Alignment: BaseVecAlignment);
1539
1540 // Freeze the index so we don't poison the clamping code we're about to emit.
1541 Idx = DAG.getFreeze(V: Idx);
1542
1543 Type *PartTy = PartVT.getTypeForEVT(Context&: *DAG.getContext());
1544 Align PartAlignment = DAG.getDataLayout().getPrefTypeAlign(Ty: PartTy);
1545
1546 // Then store the inserted part.
1547 if (PartVT.isVector()) {
1548 SDValue SubStackPtr =
1549 TLI.getVectorSubVecPointer(DAG, VecPtr: StackPtr, VecVT, SubVecVT: PartVT, Index: Idx);
1550
1551 // Store the subvector.
1552 Ch = DAG.getStore(
1553 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1554 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1555 Alignment: PartAlignment);
1556 } else {
1557 SDValue SubStackPtr =
1558 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Idx);
1559
1560 // Store the scalar value.
1561 Ch = DAG.getTruncStore(
1562 Chain: Ch, dl, Val: Part, Ptr: SubStackPtr,
1563 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()),
1564 SVT: VecVT.getVectorElementType(), Alignment: PartAlignment);
1565 }
1566
1567 assert(cast<StoreSDNode>(Ch)->getAlign() == PartAlignment &&
1568 "ElementAlignment does not match!");
1569
1570 // Finally, load the updated vector.
1571 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Ch, Ptr: StackPtr, PtrInfo,
1572 Alignment: BaseVecAlignment);
1573}
1574
1575SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1576 assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
1577 SDLoc DL(Node);
1578 SmallVector<SDValue, 16> Ops;
1579 unsigned NumOperands = Node->getNumOperands();
1580 MVT VectorIdxType = TLI.getVectorIdxTy(DL: DAG.getDataLayout());
1581 EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
1582 unsigned NumSubElem = VectorValueType.getVectorNumElements();
1583 EVT ElementValueType = TLI.getTypeToTransformTo(
1584 Context&: *DAG.getContext(), VT: VectorValueType.getVectorElementType());
1585 for (unsigned I = 0; I < NumOperands; ++I) {
1586 SDValue SubOp = Node->getOperand(Num: I);
1587 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1588 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ElementValueType,
1589 N1: SubOp,
1590 N2: DAG.getConstant(Val: Idx, DL, VT: VectorIdxType)));
1591 }
1592 }
1593 return DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL, Ops);
1594}
1595
1596SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1597 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1598 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1599 "Unexpected opcode!");
1600
1601 // We can't handle this case efficiently. Allocate a sufficiently
1602 // aligned object on the stack, store each operand into it, then load
1603 // the result as a vector.
1604 // Create the stack frame object.
1605 EVT VT = Node->getValueType(ResNo: 0);
1606 EVT MemVT = isa<BuildVectorSDNode>(Val: Node) ? VT.getVectorElementType()
1607 : Node->getOperand(Num: 0).getValueType();
1608 SDLoc dl(Node);
1609 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1610 int FI = cast<FrameIndexSDNode>(Val: FIPtr.getNode())->getIndex();
1611 MachinePointerInfo PtrInfo =
1612 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
1613
1614 // Emit a store of each element to the stack slot.
1615 SmallVector<SDValue, 8> Stores;
1616 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1617 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1618
1619 // If the destination vector element type of a BUILD_VECTOR is narrower than
1620 // the source element type, only store the bits necessary.
1621 bool Truncate = isa<BuildVectorSDNode>(Val: Node) &&
1622 MemVT.bitsLT(VT: Node->getOperand(Num: 0).getValueType());
1623
1624 // Store (in the right endianness) the elements to memory.
1625 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1626 // Ignore undef elements.
1627 if (Node->getOperand(Num: i).isUndef()) continue;
1628
1629 unsigned Offset = TypeByteSize*i;
1630
1631 SDValue Idx =
1632 DAG.getMemBasePlusOffset(Base: FIPtr, Offset: TypeSize::getFixed(ExactSize: Offset), DL: dl);
1633
1634 if (Truncate)
1635 Stores.push_back(Elt: DAG.getTruncStore(Chain: DAG.getEntryNode(), dl,
1636 Val: Node->getOperand(Num: i), Ptr: Idx,
1637 PtrInfo: PtrInfo.getWithOffset(O: Offset), SVT: MemVT));
1638 else
1639 Stores.push_back(Elt: DAG.getStore(Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: i),
1640 Ptr: Idx, PtrInfo: PtrInfo.getWithOffset(O: Offset)));
1641 }
1642
1643 SDValue StoreChain;
1644 if (!Stores.empty()) // Not all undef elements?
1645 StoreChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
1646 else
1647 StoreChain = DAG.getEntryNode();
1648
1649 // Result is a load from the stack slot.
1650 return DAG.getLoad(VT, dl, Chain: StoreChain, Ptr: FIPtr, PtrInfo);
1651}
1652
1653/// Bitcast a floating-point value to an integer value. Only bitcast the part
1654/// containing the sign bit if the target has no integer value capable of
1655/// holding all bits of the floating-point value.
1656void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1657 const SDLoc &DL,
1658 SDValue Value) const {
1659 EVT FloatVT = Value.getValueType();
1660 unsigned NumBits = FloatVT.getScalarSizeInBits();
1661 State.FloatVT = FloatVT;
1662 EVT IVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
1663 // Convert to an integer of the same size.
1664 if (TLI.isTypeLegal(VT: IVT)) {
1665 State.IntValue = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Value);
1666 State.SignMask = APInt::getSignMask(BitWidth: NumBits);
1667 State.SignBit = NumBits - 1;
1668 return;
1669 }
1670
1671 auto &DataLayout = DAG.getDataLayout();
1672 // Store the float to memory, then load the sign part out as an integer.
1673 MVT LoadTy = TLI.getRegisterType(Context&: *DAG.getContext(), VT: MVT::i8);
1674 // First create a temporary that is aligned for both the load and store.
1675 SDValue StackPtr = DAG.CreateStackTemporary(VT1: FloatVT, VT2: LoadTy);
1676 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
1677 // Then store the float to it.
1678 State.FloatPtr = StackPtr;
1679 MachineFunction &MF = DAG.getMachineFunction();
1680 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1681 State.Chain = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: Value, Ptr: State.FloatPtr,
1682 PtrInfo: State.FloatPointerInfo);
1683
1684 SDValue IntPtr;
1685 if (DataLayout.isBigEndian()) {
1686 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1687 // Load out a legal integer with the same sign bit as the float.
1688 IntPtr = StackPtr;
1689 State.IntPointerInfo = State.FloatPointerInfo;
1690 } else {
1691 // Advance the pointer so that the loaded byte will contain the sign bit.
1692 unsigned ByteOffset = (NumBits / 8) - 1;
1693 IntPtr =
1694 DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByteOffset), DL);
1695 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1696 Offset: ByteOffset);
1697 }
1698
1699 State.IntPtr = IntPtr;
1700 State.IntValue = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: LoadTy, Chain: State.Chain, Ptr: IntPtr,
1701 PtrInfo: State.IntPointerInfo, MemVT: MVT::i8);
1702 State.SignMask = APInt::getOneBitSet(numBits: LoadTy.getScalarSizeInBits(), BitNo: 7);
1703 State.SignBit = 7;
1704}
1705
1706/// Replace the integer value produced by getSignAsIntValue() with a new value
1707/// and cast the result back to a floating-point type.
1708SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1709 const SDLoc &DL,
1710 SDValue NewIntValue) const {
1711 if (!State.Chain)
1712 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: State.FloatVT, Operand: NewIntValue);
1713
1714 // Override the part containing the sign bit in the value stored on the stack.
1715 SDValue Chain = DAG.getTruncStore(Chain: State.Chain, dl: DL, Val: NewIntValue, Ptr: State.IntPtr,
1716 PtrInfo: State.IntPointerInfo, SVT: MVT::i8);
1717 return DAG.getLoad(VT: State.FloatVT, dl: DL, Chain, Ptr: State.FloatPtr,
1718 PtrInfo: State.FloatPointerInfo);
1719}
1720
1721SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1722 SDLoc DL(Node);
1723 SDValue Mag = Node->getOperand(Num: 0);
1724 SDValue Sign = Node->getOperand(Num: 1);
1725
1726 if (Sign.getValueType().isVector())
1727 return DAG.UnrollVectorOp(N: Node);
1728
1729 // Get sign bit into an integer value.
1730 FloatSignAsInt SignAsInt;
1731 getSignAsIntValue(State&: SignAsInt, DL, Value: Sign);
1732
1733 EVT IntVT = SignAsInt.IntValue.getValueType();
1734 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1735 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: SignAsInt.IntValue,
1736 N2: SignMask);
1737
1738 // If FABS is legal transform
1739 // FCOPYSIGN(x, y) => SignBit(y) ? -FABS(x) : FABS(x)
1740 EVT FloatVT = Mag.getValueType();
1741 if (TLI.isOperationLegalOrCustom(Op: ISD::FABS, VT: FloatVT) &&
1742 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT: FloatVT)) {
1743 SDValue AbsValue = DAG.getNode(Opcode: ISD::FABS, DL, VT: FloatVT, Operand: Mag);
1744 SDValue NegValue = DAG.getNode(Opcode: ISD::FNEG, DL, VT: FloatVT, Operand: AbsValue);
1745 SDValue Cond = DAG.getSetCC(DL, VT: getSetCCResultType(VT: IntVT), LHS: SignBit,
1746 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETNE);
1747 return DAG.getSelect(DL, VT: FloatVT, Cond, LHS: NegValue, RHS: AbsValue);
1748 }
1749
1750 // Transform Mag value to integer, and clear the sign bit.
1751 FloatSignAsInt MagAsInt;
1752 getSignAsIntValue(State&: MagAsInt, DL, Value: Mag);
1753 EVT MagVT = MagAsInt.IntValue.getValueType();
1754 SDValue ClearSignMask = DAG.getConstant(Val: ~MagAsInt.SignMask, DL, VT: MagVT);
1755 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: MagVT, N1: MagAsInt.IntValue,
1756 N2: ClearSignMask);
1757
1758 // Get the signbit at the right position for MagAsInt.
1759 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1760 EVT ShiftVT = IntVT;
1761 if (SignBit.getScalarValueSizeInBits() <
1762 ClearedSign.getScalarValueSizeInBits()) {
1763 SignBit = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MagVT, Operand: SignBit);
1764 ShiftVT = MagVT;
1765 }
1766 if (ShiftAmount > 0) {
1767 SDValue ShiftCnst = DAG.getConstant(Val: ShiftAmount, DL, VT: ShiftVT);
1768 SignBit = DAG.getNode(Opcode: ISD::SRL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1769 } else if (ShiftAmount < 0) {
1770 SDValue ShiftCnst = DAG.getConstant(Val: -ShiftAmount, DL, VT: ShiftVT);
1771 SignBit = DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftVT, N1: SignBit, N2: ShiftCnst);
1772 }
1773 if (SignBit.getScalarValueSizeInBits() >
1774 ClearedSign.getScalarValueSizeInBits()) {
1775 SignBit = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MagVT, Operand: SignBit);
1776 }
1777
1778 // Store the part with the modified sign and convert back to float.
1779 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: MagVT, N1: ClearedSign, N2: SignBit,
1780 Flags: SDNodeFlags::Disjoint);
1781
1782 return modifySignAsInt(State: MagAsInt, DL, NewIntValue: CopiedSign);
1783}
1784
1785SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1786 // Get the sign bit as an integer.
1787 SDLoc DL(Node);
1788 if (Node->getValueType(ResNo: 0).isVector())
1789 return DAG.UnrollVectorOp(N: Node);
1790
1791 FloatSignAsInt SignAsInt;
1792 getSignAsIntValue(State&: SignAsInt, DL, Value: Node->getOperand(Num: 0));
1793 EVT IntVT = SignAsInt.IntValue.getValueType();
1794
1795 // Flip the sign.
1796 SDValue SignMask = DAG.getConstant(Val: SignAsInt.SignMask, DL, VT: IntVT);
1797 SDValue SignFlip =
1798 DAG.getNode(Opcode: ISD::XOR, DL, VT: IntVT, N1: SignAsInt.IntValue, N2: SignMask);
1799
1800 // Convert back to float.
1801 return modifySignAsInt(State: SignAsInt, DL, NewIntValue: SignFlip);
1802}
1803
1804SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1805 SDLoc DL(Node);
1806 SDValue Value = Node->getOperand(Num: 0);
1807
1808 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1809 EVT FloatVT = Value.getValueType();
1810 if (TLI.isOperationLegalOrCustom(Op: ISD::FCOPYSIGN, VT: FloatVT)) {
1811 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT: FloatVT);
1812 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT: FloatVT, N1: Value, N2: Zero);
1813 }
1814
1815 if (FloatVT.isVector())
1816 return DAG.UnrollVectorOp(N: Node);
1817
1818 // Transform value to integer, clear the sign bit and transform back.
1819 FloatSignAsInt ValueAsInt;
1820 getSignAsIntValue(State&: ValueAsInt, DL, Value);
1821 EVT IntVT = ValueAsInt.IntValue.getValueType();
1822 SDValue ClearSignMask = DAG.getConstant(Val: ~ValueAsInt.SignMask, DL, VT: IntVT);
1823 SDValue ClearedSign = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ValueAsInt.IntValue,
1824 N2: ClearSignMask);
1825 return modifySignAsInt(State: ValueAsInt, DL, NewIntValue: ClearedSign);
1826}
1827
1828void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1829 SmallVectorImpl<SDValue> &Results) {
1830 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1831 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1832 " not tell us which reg is the stack pointer!");
1833 SDLoc dl(Node);
1834 EVT VT = Node->getValueType(ResNo: 0);
1835 SDValue Tmp1 = SDValue(Node, 0);
1836 SDValue Tmp2 = SDValue(Node, 1);
1837 SDValue Tmp3 = Node->getOperand(Num: 2);
1838 SDValue Chain = Tmp1.getOperand(i: 0);
1839
1840 // Chain the dynamic stack allocation so that it doesn't modify the stack
1841 // pointer when other instructions are using the stack.
1842 Chain = DAG.getCALLSEQ_START(Chain, InSize: 0, OutSize: 0, DL: dl);
1843
1844 SDValue Size = Tmp2.getOperand(i: 1);
1845 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: SPReg, VT);
1846 Chain = SP.getValue(R: 1);
1847 Align Alignment = cast<ConstantSDNode>(Val&: Tmp3)->getAlignValue();
1848 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1849 unsigned Opc =
1850 TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp ?
1851 ISD::ADD : ISD::SUB;
1852
1853 Align StackAlign = TFL->getStackAlign();
1854 Tmp1 = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SP, N2: Size); // Value
1855 if (Alignment > StackAlign)
1856 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp1,
1857 N2: DAG.getSignedConstant(Val: -Alignment.value(), DL: dl, VT));
1858 Chain = DAG.getCopyToReg(Chain, dl, Reg: SPReg, N: Tmp1); // Output chain
1859
1860 Tmp2 = DAG.getCALLSEQ_END(Chain, Size1: 0, Size2: 0, Glue: SDValue(), DL: dl);
1861
1862 Results.push_back(Elt: Tmp1);
1863 Results.push_back(Elt: Tmp2);
1864}
1865
1866SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1867 EVT DestVT, const SDLoc &dl) {
1868 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, ChainIn: DAG.getEntryNode());
1869}
1870
1871SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1872 EVT DestVT, const SDLoc &dl,
1873 SDValue Chain) {
1874 EVT SrcVT = SrcOp.getValueType();
1875 Type *DestType = DestVT.getTypeForEVT(Context&: *DAG.getContext());
1876 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(Ty: DestType);
1877 // Don't convert with stack if the load/store is expensive.
1878 if ((SrcVT.bitsGT(VT: SlotVT) && !TLI.isTruncStoreLegalOrCustom(
1879 ValVT: SrcVT, MemVT: SlotVT, Alignment: DestAlign,
1880 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace())) ||
1881 (SlotVT.bitsLT(VT: DestVT) &&
1882 !TLI.isLoadLegalOrCustom(ValVT: DestVT, MemVT: SlotVT, Alignment: DestAlign,
1883 AddrSpace: DAG.getDataLayout().getAllocaAddrSpace(),
1884 ExtType: ISD::EXTLOAD, Atomic: false)))
1885 return SDValue();
1886
1887 return DAG.emitStackConvert(SrcOp, SlotVT, DestVT, DL: dl, Chain);
1888}
1889
1890SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1891 SDLoc dl(Node);
1892 // Create a vector sized/aligned stack slot, store the value to element #0,
1893 // then load the whole vector back out.
1894 SDValue StackPtr = DAG.CreateStackTemporary(VT: Node->getValueType(ResNo: 0));
1895
1896 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(Val&: StackPtr);
1897 int SPFI = StackPtrFI->getIndex();
1898
1899 SDValue Ch = DAG.getTruncStore(
1900 Chain: DAG.getEntryNode(), dl, Val: Node->getOperand(Num: 0), Ptr: StackPtr,
1901 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI),
1902 SVT: Node->getValueType(ResNo: 0).getVectorElementType());
1903 return DAG.getLoad(
1904 VT: Node->getValueType(ResNo: 0), dl, Chain: Ch, Ptr: StackPtr,
1905 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
1906}
1907
1908static bool
1909ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1910 const TargetLowering &TLI, SDValue &Res) {
1911 unsigned NumElems = Node->getNumOperands();
1912 SDLoc dl(Node);
1913 EVT VT = Node->getValueType(ResNo: 0);
1914
1915 // Try to group the scalars into pairs, shuffle the pairs together, then
1916 // shuffle the pairs of pairs together, etc. until the vector has
1917 // been built. This will work only if all of the necessary shuffle masks
1918 // are legal.
1919
1920 // We do this in two phases; first to check the legality of the shuffles,
1921 // and next, assuming that all shuffles are legal, to create the new nodes.
1922 for (int Phase = 0; Phase < 2; ++Phase) {
1923 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1924 NewIntermedVals;
1925 for (unsigned i = 0; i < NumElems; ++i) {
1926 SDValue V = Node->getOperand(Num: i);
1927 if (V.isUndef())
1928 continue;
1929
1930 SDValue Vec;
1931 if (Phase)
1932 Vec = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: V);
1933 IntermedVals.push_back(Elt: std::make_pair(x&: Vec, y: SmallVector<int, 16>(1, i)));
1934 }
1935
1936 while (IntermedVals.size() > 2) {
1937 NewIntermedVals.clear();
1938 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1939 // This vector and the next vector are shuffled together (simply to
1940 // append the one to the other).
1941 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1942
1943 SmallVector<int, 16> FinalIndices;
1944 FinalIndices.reserve(N: IntermedVals[i].second.size() +
1945 IntermedVals[i+1].second.size());
1946
1947 int k = 0;
1948 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1949 ++j, ++k) {
1950 ShuffleVec[k] = j;
1951 FinalIndices.push_back(Elt: IntermedVals[i].second[j]);
1952 }
1953 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1954 ++j, ++k) {
1955 ShuffleVec[k] = NumElems + j;
1956 FinalIndices.push_back(Elt: IntermedVals[i+1].second[j]);
1957 }
1958
1959 SDValue Shuffle;
1960 if (Phase)
1961 Shuffle = DAG.getVectorShuffle(VT, dl, N1: IntermedVals[i].first,
1962 N2: IntermedVals[i+1].first,
1963 Mask: ShuffleVec);
1964 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1965 return false;
1966 NewIntermedVals.push_back(
1967 Elt: std::make_pair(x&: Shuffle, y: std::move(FinalIndices)));
1968 }
1969
1970 // If we had an odd number of defined values, then append the last
1971 // element to the array of new vectors.
1972 if ((IntermedVals.size() & 1) != 0)
1973 NewIntermedVals.push_back(Elt: IntermedVals.back());
1974
1975 IntermedVals.swap(RHS&: NewIntermedVals);
1976 }
1977
1978 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1979 "Invalid number of intermediate vectors");
1980 SDValue Vec1 = IntermedVals[0].first;
1981 SDValue Vec2;
1982 if (IntermedVals.size() > 1)
1983 Vec2 = IntermedVals[1].first;
1984 else if (Phase)
1985 Vec2 = DAG.getPOISON(VT);
1986
1987 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1988 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1989 ShuffleVec[IntermedVals[0].second[i]] = i;
1990 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1991 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1992
1993 if (Phase)
1994 Res = DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
1995 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1996 return false;
1997 }
1998
1999 return true;
2000}
2001
2002/// Expand a BUILD_VECTOR node on targets that don't
2003/// support the operation, but do support the resultant vector type.
2004SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2005 unsigned NumElems = Node->getNumOperands();
2006 SDValue Value1, Value2;
2007 SDLoc dl(Node);
2008 EVT VT = Node->getValueType(ResNo: 0);
2009 EVT OpVT = Node->getOperand(Num: 0).getValueType();
2010 EVT EltVT = VT.getVectorElementType();
2011
2012 // If the only non-undef value is the low element, turn this into a
2013 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
2014 bool isOnlyLowElement = true;
2015 bool MoreThanTwoValues = false;
2016 bool isConstant = true;
2017 for (unsigned i = 0; i < NumElems; ++i) {
2018 SDValue V = Node->getOperand(Num: i);
2019 if (V.isUndef())
2020 continue;
2021 if (i > 0)
2022 isOnlyLowElement = false;
2023 if (!isa<ConstantFPSDNode>(Val: V) && !isa<ConstantSDNode>(Val: V))
2024 isConstant = false;
2025
2026 if (!Value1.getNode()) {
2027 Value1 = V;
2028 } else if (!Value2.getNode()) {
2029 if (V != Value1)
2030 Value2 = V;
2031 } else if (V != Value1 && V != Value2) {
2032 MoreThanTwoValues = true;
2033 }
2034 }
2035
2036 if (!Value1.getNode())
2037 return DAG.getUNDEF(VT);
2038
2039 if (isOnlyLowElement)
2040 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Node->getOperand(Num: 0));
2041
2042 // If all elements are constants, create a load from the constant pool.
2043 if (isConstant) {
2044 SmallVector<Constant*, 16> CV;
2045 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2046 if (ConstantFPSDNode *V =
2047 dyn_cast<ConstantFPSDNode>(Val: Node->getOperand(Num: i))) {
2048 CV.push_back(Elt: const_cast<ConstantFP *>(V->getConstantFPValue()));
2049 } else if (ConstantSDNode *V =
2050 dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: i))) {
2051 if (OpVT==EltVT)
2052 CV.push_back(Elt: const_cast<ConstantInt *>(V->getConstantIntValue()));
2053 else {
2054 // If OpVT and EltVT don't match, EltVT is not legal and the
2055 // element values have been promoted/truncated earlier. Undo this;
2056 // we don't want a v16i8 to become a v16i32 for example.
2057 const ConstantInt *CI = V->getConstantIntValue();
2058 CV.push_back(Elt: ConstantInt::get(Ty: EltVT.getTypeForEVT(Context&: *DAG.getContext()),
2059 V: CI->getZExtValue(), /*IsSigned=*/false,
2060 /*ImplicitTrunc=*/true));
2061 }
2062 } else {
2063 assert(Node->getOperand(i).isUndef());
2064 Type *OpNTy = EltVT.getTypeForEVT(Context&: *DAG.getContext());
2065 CV.push_back(Elt: UndefValue::get(T: OpNTy));
2066 }
2067 }
2068 Constant *CP = ConstantVector::get(V: CV);
2069 SDValue CPIdx =
2070 DAG.getConstantPool(C: CP, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2071 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2072 return DAG.getLoad(
2073 VT, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2074 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2075 Alignment);
2076 }
2077
2078 SmallSet<SDValue, 16> DefinedValues;
2079 for (unsigned i = 0; i < NumElems; ++i) {
2080 if (Node->getOperand(Num: i).isUndef())
2081 continue;
2082 DefinedValues.insert(V: Node->getOperand(Num: i));
2083 }
2084
2085 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues: DefinedValues.size())) {
2086 if (!MoreThanTwoValues) {
2087 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2088 for (unsigned i = 0; i < NumElems; ++i) {
2089 SDValue V = Node->getOperand(Num: i);
2090 if (V.isUndef())
2091 continue;
2092 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2093 }
2094 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(ResNo: 0))) {
2095 // Get the splatted value into the low element of a vector register.
2096 SDValue Vec1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value1);
2097 SDValue Vec2;
2098 if (Value2.getNode())
2099 Vec2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT, Operand: Value2);
2100 else
2101 Vec2 = DAG.getPOISON(VT);
2102
2103 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2104 return DAG.getVectorShuffle(VT, dl, N1: Vec1, N2: Vec2, Mask: ShuffleVec);
2105 }
2106 } else {
2107 SDValue Res;
2108 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2109 return Res;
2110 }
2111 }
2112
2113 // Otherwise, we can't handle this case efficiently.
2114 return ExpandVectorBuildThroughStack(Node);
2115}
2116
2117SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2118 SDLoc DL(Node);
2119 EVT VT = Node->getValueType(ResNo: 0);
2120 SDValue SplatVal = Node->getOperand(Num: 0);
2121
2122 return DAG.getSplatBuildVector(VT, DL, Op: SplatVal);
2123}
2124
2125// Expand a node into a call to a libcall, returning the value as the first
2126// result and the chain as the second. If the result value does not fit into a
2127// register, return the lo part and set the hi part to the by-reg argument in
2128// the first. If it does fit into a single register, return the result and
2129// leave the Hi part unset.
2130std::pair<SDValue, SDValue>
2131SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2132 TargetLowering::ArgListTy &&Args,
2133 bool IsSigned, EVT RetVT) {
2134 EVT CodePtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
2135 SDValue Callee;
2136 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2137 if (LCImpl != RTLIB::Unsupported)
2138 Callee = DAG.getExternalSymbol(LCImpl, VT: CodePtrTy);
2139 else {
2140 Callee = DAG.getPOISON(VT: CodePtrTy);
2141 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2142 Node->getOperationName(G: &DAG));
2143 }
2144
2145 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2146
2147 // By default, the input chain to this libcall is the entry node of the
2148 // function. If the libcall is going to be emitted as a tail call then
2149 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2150 // node which is being folded has a non-entry input chain.
2151 SDValue InChain = DAG.getEntryNode();
2152
2153 // isTailCall may be true since the callee does not reference caller stack
2154 // frame. Check if it's in the right position and that the return types match.
2155 SDValue TCChain = InChain;
2156 const Function &F = DAG.getMachineFunction().getFunction();
2157 bool isTailCall =
2158 TLI.isInTailCallPosition(DAG, Node, Chain&: TCChain) &&
2159 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()) &&
2160 // Lowering doesn't support tail calling inside a function with
2161 // a swifterror argument yet.
2162 !DAG.hasSwiftErrorArg();
2163 if (isTailCall)
2164 InChain = TCChain;
2165
2166 TargetLowering::CallLoweringInfo CLI(DAG);
2167 bool signExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned);
2168 CLI.setDebugLoc(SDLoc(Node))
2169 .setChain(InChain)
2170 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LCImpl), ResultType: RetTy,
2171 Target: Callee, ArgsList: std::move(Args))
2172 .setTailCall(isTailCall)
2173 .setSExtResult(signExtend)
2174 .setZExtResult(!signExtend)
2175 .setIsPostTypeLegalization(true);
2176
2177 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2178
2179 if (!CallInfo.second.getNode()) {
2180 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2181 // It's a tailcall, return the chain (which is the DAG root).
2182 return {DAG.getRoot(), DAG.getRoot()};
2183 }
2184
2185 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2186 return CallInfo;
2187}
2188
2189std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2190 bool isSigned) {
2191 TargetLowering::ArgListTy Args;
2192 for (const SDValue &Op : Node->op_values()) {
2193 EVT ArgVT = Op.getValueType();
2194 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2195 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2196 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: isSigned);
2197 Entry.IsZExt = !Entry.IsSExt;
2198 Args.push_back(x: Entry);
2199 }
2200
2201 return ExpandLibCall(LC, Node, Args: std::move(Args), IsSigned: isSigned,
2202 RetVT: Node->getValueType(ResNo: 0));
2203}
2204
2205void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2206 RTLIB::Libcall LC,
2207 SmallVectorImpl<SDValue> &Results) {
2208 if (LC == RTLIB::UNKNOWN_LIBCALL)
2209 llvm_unreachable("Can't create an unknown libcall!");
2210
2211 if (Node->isStrictFPOpcode()) {
2212 EVT RetVT = Node->getValueType(ResNo: 0);
2213 RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2214 if (LCImpl == RTLIB::Unsupported) {
2215 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2216 Node->getOperationName(G: &DAG));
2217 Results.push_back(Elt: DAG.getPOISON(VT: RetVT));
2218 Results.push_back(Elt: Node->getOperand(Num: 0));
2219 return;
2220 }
2221 SmallVector<SDValue, 4> Ops(drop_begin(RangeOrContainer: Node->ops()));
2222 TargetLowering::MakeLibCallOptions CallOptions;
2223 CallOptions.IsPostTypeLegalization = true;
2224 // FIXME: This doesn't support tail calls.
2225 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2226 DAG, LibcallImpl: LCImpl, RetVT, Ops, CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
2227 Results.push_back(Elt: Tmp.first);
2228 Results.push_back(Elt: Tmp.second);
2229 } else {
2230 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2231 SDValue Tmp = ExpandLibCall(LC, Node, isSigned: IsSignedArgument).first;
2232 Results.push_back(Elt: Tmp);
2233 }
2234}
2235
2236/// Expand the node to a libcall based on the result type.
2237void SelectionDAGLegalize::ExpandFastFPLibCall(
2238 SDNode *Node, bool IsFast,
2239 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2240 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2241 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2242 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2243 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2244 SmallVectorImpl<SDValue> &Results) {
2245
2246 EVT VT = Node->getSimpleValueType(ResNo: 0);
2247
2248 RTLIB::Libcall LC;
2249
2250 // FIXME: Probably should define fast to respect nan/inf and only be
2251 // approximate functions.
2252
2253 if (IsFast) {
2254 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.first, Call_F64: Call_F64.first, Call_F80: Call_F80.first,
2255 Call_F128: Call_F128.first, Call_PPCF128: Call_PPCF128.first);
2256 }
2257
2258 if (!IsFast || DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
2259 // Fall back if we don't have a fast implementation.
2260 LC = RTLIB::getFPLibCall(VT, Call_F32: Call_F32.second, Call_F64: Call_F64.second,
2261 Call_F80: Call_F80.second, Call_F128: Call_F128.second,
2262 Call_PPCF128: Call_PPCF128.second);
2263 }
2264
2265 ExpandFPLibCall(Node, LC, Results);
2266}
2267
2268SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2269 RTLIB::Libcall Call_I8,
2270 RTLIB::Libcall Call_I16,
2271 RTLIB::Libcall Call_I32,
2272 RTLIB::Libcall Call_I64,
2273 RTLIB::Libcall Call_I128) {
2274 RTLIB::Libcall LC;
2275 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2276 default: llvm_unreachable("Unexpected request for libcall!");
2277 case MVT::i8: LC = Call_I8; break;
2278 case MVT::i16: LC = Call_I16; break;
2279 case MVT::i32: LC = Call_I32; break;
2280 case MVT::i64: LC = Call_I64; break;
2281 case MVT::i128: LC = Call_I128; break;
2282 }
2283 return ExpandLibCall(LC, Node, isSigned).first;
2284}
2285
2286/// Expand the node to a libcall based on first argument type (for instance
2287/// lround and its variant).
2288void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2289 RTLIB::Libcall Call_F32,
2290 RTLIB::Libcall Call_F64,
2291 RTLIB::Libcall Call_F80,
2292 RTLIB::Libcall Call_F128,
2293 RTLIB::Libcall Call_PPCF128,
2294 SmallVectorImpl<SDValue> &Results) {
2295 EVT InVT = Node->getOperand(Num: Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2296 RTLIB::Libcall LC = RTLIB::getFPLibCall(VT: InVT.getSimpleVT(),
2297 Call_F32, Call_F64, Call_F80,
2298 Call_F128, Call_PPCF128);
2299 ExpandFPLibCall(Node, LC, Results);
2300}
2301
2302SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2303 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2304 RTLIB::Libcall CallI128) {
2305 RTLIB::Libcall LC;
2306 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2307 default:
2308 llvm_unreachable("Unexpected request for libcall!");
2309 case MVT::i32:
2310 LC = CallI32;
2311 break;
2312 case MVT::i64:
2313 LC = CallI64;
2314 break;
2315 case MVT::i128:
2316 LC = CallI128;
2317 break;
2318 }
2319
2320 // Bit-counting libcalls have one unsigned argument and return `int`.
2321 // Note that `int` may be illegal on this target; ExpandLibCall will
2322 // take care of promoting it to a legal type.
2323 SDValue Op = Node->getOperand(Num: 0);
2324 EVT IntVT =
2325 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DAG.getLibInfo().getIntSize());
2326
2327 EVT ArgVT = Op.getValueType();
2328 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2329 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2330 Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, /*IsSigned=*/false);
2331 Arg.IsZExt = !Arg.IsSExt;
2332
2333 SDValue Res = ExpandLibCall(LC, Node, Args: TargetLowering::ArgListTy{Arg},
2334 /*IsSigned=*/true, RetVT: IntVT)
2335 .first;
2336
2337 // If ExpandLibCall created a tail call, the result was already
2338 // of the correct type. Otherwise, we need to sign extend it.
2339 if (Res.getValueType() != MVT::Other)
2340 Res = DAG.getSExtOrTrunc(Op: Res, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0));
2341 return Res;
2342}
2343
2344/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2345void
2346SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2347 SmallVectorImpl<SDValue> &Results) {
2348 unsigned Opcode = Node->getOpcode();
2349 bool isSigned = Opcode == ISD::SDIVREM;
2350
2351 RTLIB::Libcall LC;
2352 switch (Node->getSimpleValueType(ResNo: 0).SimpleTy) {
2353 default: llvm_unreachable("Unexpected request for libcall!");
2354 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2355 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2356 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2357 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2358 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2359 }
2360
2361 // The input chain to this libcall is the entry node of the function.
2362 // Legalizing the call will automatically add the previous call to the
2363 // dependence.
2364 SDValue InChain = DAG.getEntryNode();
2365
2366 EVT RetVT = Node->getValueType(ResNo: 0);
2367 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
2368
2369 TargetLowering::ArgListTy Args;
2370 for (const SDValue &Op : Node->op_values()) {
2371 EVT ArgVT = Op.getValueType();
2372 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
2373 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2374 Entry.IsSExt = isSigned;
2375 Entry.IsZExt = !isSigned;
2376 Args.push_back(x: Entry);
2377 }
2378
2379 // Also pass the return address of the remainder.
2380 SDValue FIPtr = DAG.CreateStackTemporary(VT: RetVT);
2381 TargetLowering::ArgListEntry Entry(
2382 FIPtr, PointerType::getUnqual(C&: RetTy->getContext()));
2383 Entry.IsSExt = isSigned;
2384 Entry.IsZExt = !isSigned;
2385 Args.push_back(x: Entry);
2386
2387 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
2388 if (LibcallImpl == RTLIB::Unsupported) {
2389 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
2390 Node->getOperationName(G: &DAG));
2391 SDValue Poison = DAG.getPOISON(VT: RetVT);
2392 Results.push_back(Elt: Poison);
2393 Results.push_back(Elt: Poison);
2394 return;
2395 }
2396
2397 SDValue Callee =
2398 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2399
2400 SDLoc dl(Node);
2401 TargetLowering::CallLoweringInfo CLI(DAG);
2402 CLI.setDebugLoc(dl)
2403 .setChain(InChain)
2404 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
2405 ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
2406 .setSExtResult(isSigned)
2407 .setZExtResult(!isSigned);
2408
2409 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2410
2411 // Remainder is loaded back from the stack frame.
2412 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
2413 MachinePointerInfo PtrInfo =
2414 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
2415
2416 SDValue Rem = DAG.getLoad(VT: RetVT, dl, Chain: CallInfo.second, Ptr: FIPtr, PtrInfo);
2417 Results.push_back(Elt: CallInfo.first);
2418 Results.push_back(Elt: Rem);
2419}
2420
2421/// Return true if sincos or __sincos_stret libcall is available.
2422static bool isSinCosLibcallAvailable(SDNode *Node,
2423 const LibcallLoweringInfo &Libcalls) {
2424 MVT::SimpleValueType VT = Node->getSimpleValueType(ResNo: 0).SimpleTy;
2425 return Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS(VT)) != RTLIB::Unsupported ||
2426 Libcalls.getLibcallImpl(Call: RTLIB::getSINCOS_STRET(VT)) !=
2427 RTLIB::Unsupported;
2428}
2429
2430/// Only issue sincos libcall if both sin and cos are needed.
2431static bool useSinCos(SDNode *Node) {
2432 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2433 ? ISD::FCOS : ISD::FSIN;
2434
2435 SDValue Op0 = Node->getOperand(Num: 0);
2436 for (const SDNode *User : Op0.getNode()->users()) {
2437 if (User == Node)
2438 continue;
2439 // The other user might have been turned into sincos already.
2440 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2441 return true;
2442 }
2443 return false;
2444}
2445
2446SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2447 // For iOS, we want to call an alternative entry point: __sincos_stret,
2448 // which returns the values in two S / D registers.
2449 SDLoc dl(Node);
2450 SDValue Arg = Node->getOperand(Num: 0);
2451 EVT ArgVT = Arg.getValueType();
2452 RTLIB::Libcall LC = RTLIB::getSINCOS_STRET(VT: ArgVT);
2453 RTLIB::LibcallImpl SincosStret = DAG.getLibcalls().getLibcallImpl(Call: LC);
2454 if (SincosStret == RTLIB::Unsupported)
2455 return SDValue();
2456
2457 /// There are 3 different ABI cases to handle:
2458 /// - Direct return of separate fields in registers
2459 /// - Single return as vector elements
2460 /// - sret struct
2461
2462 const RTLIB::RuntimeLibcallsInfo &CallsInfo =
2463 DAG.getLibcalls().getRuntimeLibcallsInfo();
2464
2465 const DataLayout &DL = DAG.getDataLayout();
2466
2467 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2468 Ctx&: *DAG.getContext(), TT: TM.getTargetTriple(), DL, LibcallImpl: SincosStret);
2469
2470 Type *SincosStretRetTy = FuncTy->getReturnType();
2471 CallingConv::ID CallConv = CallsInfo.getLibcallImplCallingConv(Call: SincosStret);
2472
2473 SDValue Callee =
2474 DAG.getExternalSymbol(LCImpl: SincosStret, VT: TLI.getProgramPointerTy(DL));
2475
2476 TargetLowering::ArgListTy Args;
2477 SDValue SRet;
2478
2479 int FrameIdx;
2480 if (FuncTy->getParamType(i: 0)->isPointerTy()) {
2481 // Uses sret
2482 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
2483
2484 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(ArgNo: 0);
2485 Type *StructTy = PtrAttrs.getStructRetType();
2486 const uint64_t ByteSize = DL.getTypeAllocSize(Ty: StructTy);
2487 const Align StackAlign = DL.getPrefTypeAlign(Ty: StructTy);
2488
2489 FrameIdx = MFI.CreateStackObject(Size: ByteSize, Alignment: StackAlign, isSpillSlot: false);
2490 SRet = DAG.getFrameIndex(FI: FrameIdx, VT: TLI.getFrameIndexTy(DL));
2491
2492 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(i: 0));
2493 Entry.IsSRet = true;
2494 Entry.IndirectType = StructTy;
2495 Entry.Alignment = StackAlign;
2496
2497 Args.push_back(x: Entry);
2498 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 1));
2499 } else {
2500 Args.emplace_back(args&: Arg, args: FuncTy->getParamType(i: 0));
2501 }
2502
2503 TargetLowering::CallLoweringInfo CLI(DAG);
2504 CLI.setDebugLoc(dl)
2505 .setChain(DAG.getEntryNode())
2506 .setLibCallee(CC: CallConv, ResultType: SincosStretRetTy, Target: Callee, ArgsList: std::move(Args))
2507 .setIsPostTypeLegalization();
2508
2509 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2510
2511 if (SRet) {
2512 MachinePointerInfo PtrInfo =
2513 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
2514 SDValue LoadSin = DAG.getLoad(VT: ArgVT, dl, Chain: CallResult.second, Ptr: SRet, PtrInfo);
2515
2516 TypeSize StoreSize = ArgVT.getStoreSize();
2517
2518 // Address of cos field.
2519 SDValue Add = DAG.getObjectPtrOffset(SL: dl, Ptr: SRet, Offset: StoreSize);
2520 SDValue LoadCos = DAG.getLoad(VT: ArgVT, dl, Chain: LoadSin.getValue(R: 1), Ptr: Add,
2521 PtrInfo: PtrInfo.getWithOffset(O: StoreSize));
2522
2523 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2524 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: LoadSin.getValue(R: 0),
2525 N2: LoadCos.getValue(R: 0));
2526 }
2527
2528 if (!CallResult.first.getValueType().isVector())
2529 return CallResult.first;
2530
2531 SDValue SinVal =
2532 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2533 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
2534 SDValue CosVal =
2535 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: ArgVT, N1: CallResult.first,
2536 N2: DAG.getVectorIdxConstant(Val: 1, DL: dl));
2537 SDVTList Tys = DAG.getVTList(VT1: ArgVT, VT2: ArgVT);
2538 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: Tys, N1: SinVal, N2: CosVal);
2539}
2540
2541SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2542 SDLoc dl(Node);
2543 EVT VT = Node->getValueType(ResNo: 0);
2544 SDValue X = Node->getOperand(Num: 0);
2545 SDValue N = Node->getOperand(Num: 1);
2546 EVT ExpVT = N.getValueType();
2547 EVT AsIntVT = VT.changeTypeToInteger();
2548 if (AsIntVT == EVT()) // TODO: How to handle f80?
2549 return SDValue();
2550
2551 // The expansion works through the integer-equivalent type; if that is not
2552 // legal, bail out and let the caller use a libcall (or diagnose a missing
2553 // one).
2554 if (!TLI.isTypeLegal(VT: AsIntVT))
2555 return SDValue();
2556
2557 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2558 return SDValue();
2559
2560 SDNodeFlags NSW;
2561 NSW.setNoSignedWrap(true);
2562 SDNodeFlags NUW_NSW;
2563 NUW_NSW.setNoUnsignedWrap(true);
2564 NUW_NSW.setNoSignedWrap(true);
2565
2566 EVT SetCCVT =
2567 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ExpVT);
2568 const fltSemantics &FltSem = VT.getFltSemantics();
2569
2570 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2571 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2572 const int Precision = APFloat::semanticsPrecision(FltSem);
2573
2574 const SDValue MaxExp = DAG.getSignedConstant(Val: MaxExpVal, DL: dl, VT: ExpVT);
2575 const SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2576
2577 const SDValue DoubleMaxExp = DAG.getSignedConstant(Val: 2 * MaxExpVal, DL: dl, VT: ExpVT);
2578
2579 const APFloat One(FltSem, "1.0");
2580 APFloat ScaleUpK = scalbn(X: One, Exp: MaxExpVal, RM: APFloat::rmNearestTiesToEven);
2581
2582 // Offset by precision to avoid denormal range.
2583 APFloat ScaleDownK =
2584 scalbn(X: One, Exp: MinExpVal + Precision, RM: APFloat::rmNearestTiesToEven);
2585
2586 // TODO: Should really introduce control flow and use a block for the >
2587 // MaxExp, < MinExp cases
2588
2589 // First, handle exponents Exp > MaxExp and scale down.
2590 SDValue NGtMaxExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MaxExp, Cond: ISD::SETGT);
2591
2592 SDValue DecN0 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: N, N2: MaxExp, Flags: NSW);
2593 SDValue ClampMaxVal = DAG.getConstant(Val: 3 * MaxExpVal, DL: dl, VT: ExpVT);
2594 SDValue ClampN_Big = DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT: ExpVT, N1: N, N2: ClampMaxVal);
2595 SDValue DecN1 =
2596 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: ExpVT, N1: ClampN_Big, N2: DoubleMaxExp, Flags: NSW);
2597
2598 SDValue ScaleUpTwice =
2599 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: DoubleMaxExp, Cond: ISD::SETUGT);
2600
2601 const SDValue ScaleUpVal = DAG.getConstantFP(Val: ScaleUpK, DL: dl, VT);
2602 SDValue ScaleUp0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleUpVal);
2603 SDValue ScaleUp1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleUp0, N2: ScaleUpVal);
2604
2605 SDValue SelectN_Big =
2606 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleUpTwice, N2: DecN1, N3: DecN0);
2607 SDValue SelectX_Big =
2608 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleUpTwice, N2: ScaleUp1, N3: ScaleUp0);
2609
2610 // Now handle exponents Exp < MinExp
2611 SDValue NLtMinExp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N, RHS: MinExp, Cond: ISD::SETLT);
2612
2613 SDValue Increment0 = DAG.getConstant(Val: -(MinExpVal + Precision), DL: dl, VT: ExpVT);
2614 SDValue Increment1 = DAG.getConstant(Val: -2 * (MinExpVal + Precision), DL: dl, VT: ExpVT);
2615
2616 SDValue IncN0 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: N, N2: Increment0, Flags: NUW_NSW);
2617
2618 SDValue ClampMinVal =
2619 DAG.getSignedConstant(Val: 3 * MinExpVal + 2 * Precision, DL: dl, VT: ExpVT);
2620 SDValue ClampN_Small = DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT: ExpVT, N1: N, N2: ClampMinVal);
2621 SDValue IncN1 =
2622 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: ClampN_Small, N2: Increment1, Flags: NSW);
2623
2624 const SDValue ScaleDownVal = DAG.getConstantFP(Val: ScaleDownK, DL: dl, VT);
2625 SDValue ScaleDown0 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: X, N2: ScaleDownVal);
2626 SDValue ScaleDown1 = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: ScaleDown0, N2: ScaleDownVal);
2627
2628 SDValue ScaleDownTwice = DAG.getSetCC(
2629 DL: dl, VT: SetCCVT, LHS: N,
2630 RHS: DAG.getSignedConstant(Val: 2 * MinExpVal + Precision, DL: dl, VT: ExpVT), Cond: ISD::SETULT);
2631
2632 SDValue SelectN_Small =
2633 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: ScaleDownTwice, N2: IncN1, N3: IncN0);
2634 SDValue SelectX_Small =
2635 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: ScaleDownTwice, N2: ScaleDown1, N3: ScaleDown0);
2636
2637 // Now combine the two out of range exponent handling cases with the base
2638 // case.
2639 SDValue NewX = DAG.getNode(
2640 Opcode: ISD::SELECT, DL: dl, VT, N1: NGtMaxExp, N2: SelectX_Big,
2641 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: NLtMinExp, N2: SelectX_Small, N3: X));
2642
2643 SDValue NewN = DAG.getNode(
2644 Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NGtMaxExp, N2: SelectN_Big,
2645 N3: DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: NLtMinExp, N2: SelectN_Small, N3: N));
2646
2647 SDValue BiasedN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NewN, N2: MaxExp, Flags: NSW);
2648
2649 SDValue ExponentShiftAmt =
2650 DAG.getShiftAmountConstant(Val: Precision - 1, VT: ExpVT, DL: dl);
2651 SDValue CastExpToValTy = DAG.getZExtOrTrunc(Op: BiasedN, DL: dl, VT: AsIntVT);
2652
2653 SDValue AsInt = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: AsIntVT, N1: CastExpToValTy,
2654 N2: ExponentShiftAmt, Flags: NUW_NSW);
2655 SDValue AsFP = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: AsInt);
2656 return DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: NewX, N2: AsFP);
2657}
2658
2659SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2660 SDLoc dl(Node);
2661 SDValue Val = Node->getOperand(Num: 0);
2662 EVT VT = Val.getValueType();
2663 EVT ExpVT = Node->getValueType(ResNo: 1);
2664 EVT AsIntVT = VT.changeTypeToInteger();
2665 if (AsIntVT == EVT()) // TODO: How to handle f80?
2666 return SDValue();
2667
2668 // The expansion works through the integer-equivalent type; if that is not
2669 // legal, bail out and let the caller use a libcall (or diagnose a missing
2670 // one).
2671 if (!TLI.isTypeLegal(VT: AsIntVT))
2672 return SDValue();
2673
2674 const fltSemantics &FltSem = VT.getFltSemantics();
2675 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2676 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2677 const unsigned BitSize = VT.getScalarSizeInBits();
2678
2679 // TODO: Could introduce control flow and skip over the denormal handling.
2680
2681 // scale_up = fmul value, scalbn(1.0, precision + 1)
2682 // extracted_exp = (bitcast value to uint) >> precision - 1
2683 // biased_exp = extracted_exp + min_exp
2684 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2685 //
2686 // is_denormal = val < smallest_normalized
2687 // computed_fract = is_denormal ? scale_up : extracted_fract
2688 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2689 //
2690 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2691 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2692
2693 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2694 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: true).bitcastToAPInt(), DL: dl,
2695 VT: AsIntVT);
2696
2697 SDValue SmallestNormalizedInt = DAG.getConstant(
2698 Val: APFloat::getSmallestNormalized(Sem: FltSem, Negative: false).bitcastToAPInt(), DL: dl,
2699 VT: AsIntVT);
2700
2701 // Masks out the exponent bits.
2702 SDValue ExpMask =
2703 DAG.getConstant(Val: APFloat::getInf(Sem: FltSem).bitcastToAPInt(), DL: dl, VT: AsIntVT);
2704
2705 // Mask out the exponent part of the value.
2706 //
2707 // e.g, for f32 FractSignMaskVal = 0x807fffff
2708 APInt FractSignMaskVal = APInt::getBitsSet(numBits: BitSize, loBit: 0, hiBit: Precision - 1);
2709 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2710
2711 APInt SignMaskVal = APInt::getSignedMaxValue(numBits: BitSize);
2712 SDValue SignMask = DAG.getConstant(Val: SignMaskVal, DL: dl, VT: AsIntVT);
2713
2714 SDValue FractSignMask = DAG.getConstant(Val: FractSignMaskVal, DL: dl, VT: AsIntVT);
2715
2716 const APFloat One(FltSem, "1.0");
2717 // Scale a possible denormal input.
2718 // e.g., for f64, 0x1p+54
2719 APFloat ScaleUpKVal =
2720 scalbn(X: One, Exp: Precision + 1, RM: APFloat::rmNearestTiesToEven);
2721
2722 SDValue ScaleUpK = DAG.getConstantFP(Val: ScaleUpKVal, DL: dl, VT);
2723 SDValue ScaleUp = DAG.getNode(Opcode: ISD::FMUL, DL: dl, VT, N1: Val, N2: ScaleUpK);
2724
2725 EVT SetCCVT =
2726 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
2727
2728 SDValue AsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: Val);
2729
2730 SDValue Abs = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: AsInt, N2: SignMask);
2731
2732 SDValue AddNegSmallestNormal =
2733 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AsIntVT, N1: Abs, N2: NegSmallestNormalizedInt);
2734 SDValue DenormOrZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AddNegSmallestNormal,
2735 RHS: NegSmallestNormalizedInt, Cond: ISD::SETULE);
2736
2737 SDValue IsDenormal =
2738 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: SmallestNormalizedInt, Cond: ISD::SETULT);
2739
2740 SDValue MinExp = DAG.getSignedConstant(Val: MinExpVal, DL: dl, VT: ExpVT);
2741 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ExpVT);
2742
2743 SDValue ScaledAsInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: AsIntVT, Operand: ScaleUp);
2744 SDValue ScaledSelect =
2745 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ScaledAsInt, N3: AsInt);
2746
2747 SDValue ExpMaskScaled =
2748 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledAsInt, N2: ExpMask);
2749
2750 SDValue ScaledValue =
2751 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: AsIntVT, N1: IsDenormal, N2: ExpMaskScaled, N3: Abs);
2752
2753 // Extract the exponent bits.
2754 SDValue ExponentShiftAmt =
2755 DAG.getShiftAmountConstant(Val: Precision - 1, VT: AsIntVT, DL: dl);
2756 SDValue ShiftedExp =
2757 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: AsIntVT, N1: ScaledValue, N2: ExponentShiftAmt);
2758 SDValue Exp = DAG.getSExtOrTrunc(Op: ShiftedExp, DL: dl, VT: ExpVT);
2759
2760 SDValue NormalBiasedExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: Exp, N2: MinExp);
2761 SDValue DenormalOffset = DAG.getConstant(Val: -Precision - 1, DL: dl, VT: ExpVT);
2762 SDValue DenormalExpBias =
2763 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: IsDenormal, N2: DenormalOffset, N3: Zero);
2764
2765 SDValue MaskedFractAsInt =
2766 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AsIntVT, N1: ScaledSelect, N2: FractSignMask);
2767 const APFloat Half(FltSem, "0.5");
2768 SDValue FPHalf = DAG.getConstant(Val: Half.bitcastToAPInt(), DL: dl, VT: AsIntVT);
2769 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: AsIntVT, N1: MaskedFractAsInt, N2: FPHalf);
2770 SDValue MaskedFract = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Or);
2771
2772 SDValue ComputedExp =
2773 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExpVT, N1: NormalBiasedExp, N2: DenormalExpBias);
2774
2775 SDValue Result0 =
2776 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: DenormOrZero, N2: Val, N3: MaskedFract);
2777
2778 SDValue Result1 =
2779 DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: ExpVT, N1: DenormOrZero, N2: Zero, N3: ComputedExp);
2780
2781 return DAG.getMergeValues(Ops: {Result0, Result1}, dl);
2782}
2783
2784/// This function is responsible for legalizing a
2785/// INT_TO_FP operation of the specified operand when the target requests that
2786/// we expand it. At this point, we know that the result and operand types are
2787/// legal for the target.
2788SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2789 SDValue &Chain) {
2790 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2791 Node->getOpcode() == ISD::SINT_TO_FP);
2792 EVT DestVT = Node->getValueType(ResNo: 0);
2793 SDLoc dl(Node);
2794 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2795 SDValue Op0 = Node->getOperand(Num: OpNo);
2796 EVT SrcVT = Op0.getValueType();
2797
2798 // TODO: Should any fast-math-flags be set for the created nodes?
2799 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2800 if (SrcVT == MVT::i32 && TLI.isTypeLegal(VT: MVT::f64) &&
2801 (DestVT.bitsLE(VT: MVT::f64) ||
2802 TLI.isOperationLegal(Op: Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2803 : ISD::FP_EXTEND,
2804 VT: DestVT))) {
2805 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2806 "expansion\n");
2807
2808 // Get the stack frame index of a 8 byte buffer.
2809 SDValue StackSlot = DAG.CreateStackTemporary(VT: MVT::f64);
2810
2811 SDValue Lo = Op0;
2812 // if signed map to unsigned space
2813 if (isSigned) {
2814 // Invert sign bit (signed to unsigned mapping).
2815 Lo = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Lo,
2816 N2: DAG.getConstant(Val: 0x80000000u, DL: dl, VT: MVT::i32));
2817 }
2818 // Initial hi portion of constructed double.
2819 SDValue Hi = DAG.getConstant(Val: 0x43300000u, DL: dl, VT: MVT::i32);
2820
2821 // If this a big endian target, swap the lo and high data.
2822 if (DAG.getDataLayout().isBigEndian())
2823 std::swap(a&: Lo, b&: Hi);
2824
2825 SDValue MemChain = DAG.getEntryNode();
2826
2827 // Store the lo of the constructed double.
2828 SDValue Store1 = DAG.getStore(Chain: MemChain, dl, Val: Lo, Ptr: StackSlot,
2829 PtrInfo: MachinePointerInfo());
2830 // Store the hi of the constructed double.
2831 SDValue HiPtr =
2832 DAG.getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL: dl);
2833 SDValue Store2 =
2834 DAG.getStore(Chain: MemChain, dl, Val: Hi, Ptr: HiPtr, PtrInfo: MachinePointerInfo());
2835 MemChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
2836
2837 // load the constructed double
2838 SDValue Load =
2839 DAG.getLoad(VT: MVT::f64, dl, Chain: MemChain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
2840 // FP constant to bias correct the final result
2841 SDValue Bias = DAG.getConstantFP(
2842 Val: isSigned ? llvm::bit_cast<double>(from: 0x4330000080000000ULL)
2843 : llvm::bit_cast<double>(from: 0x4330000000000000ULL),
2844 DL: dl, VT: MVT::f64);
2845 // Subtract the bias and get the final result.
2846 SDValue Sub;
2847 SDValue Result;
2848 if (Node->isStrictFPOpcode()) {
2849 Sub = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: {MVT::f64, MVT::Other},
2850 Ops: {Node->getOperand(Num: 0), Load, Bias});
2851 Chain = Sub.getValue(R: 1);
2852 if (DestVT != Sub.getValueType()) {
2853 std::pair<SDValue, SDValue> ResultPair;
2854 ResultPair =
2855 DAG.getStrictFPExtendOrRound(Op: Sub, Chain, DL: dl, VT: DestVT);
2856 Result = ResultPair.first;
2857 Chain = ResultPair.second;
2858 }
2859 else
2860 Result = Sub;
2861 } else {
2862 Sub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::f64, N1: Load, N2: Bias);
2863 Result = DAG.getFPExtendOrRound(Op: Sub, DL: dl, VT: DestVT);
2864 }
2865 return Result;
2866 }
2867
2868 if (isSigned)
2869 return SDValue();
2870
2871 // TODO: Generalize this for use with other types.
2872 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2873 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2874 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2875 // For unsigned conversions, convert them to signed conversions using the
2876 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2877 // should be valid for i32->f32 as well.
2878
2879 // More generally this transform should be valid if there are 3 more bits
2880 // in the integer type than the significand. Rounding uses the first bit
2881 // after the width of the significand and the OR of all bits after that. So
2882 // we need to be able to OR the shifted out bit into one of the bits that
2883 // participate in the OR.
2884
2885 // TODO: This really should be implemented using a branch rather than a
2886 // select. We happen to get lucky and machinesink does the right
2887 // thing most of the time. This would be a good candidate for a
2888 // pseudo-op, or, even better, for whole-function isel.
2889 EVT SetCCVT = getSetCCResultType(VT: SrcVT);
2890
2891 SDValue SignBitTest = DAG.getSetCC(
2892 DL: dl, VT: SetCCVT, LHS: Op0, RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2893
2894 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: 1, VT: SrcVT, DL: dl);
2895 SDValue Shr = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Op0, N2: ShiftConst);
2896 SDValue AndConst = DAG.getConstant(Val: 1, DL: dl, VT: SrcVT);
2897 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Op0, N2: AndConst);
2898 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: And, N2: Shr);
2899
2900 SDValue Slow, Fast;
2901 if (Node->isStrictFPOpcode()) {
2902 // In strict mode, we must avoid spurious exceptions, and therefore
2903 // must make sure to only emit a single STRICT_SINT_TO_FP.
2904 SDValue InCvt = DAG.getSelect(DL: dl, VT: SrcVT, Cond: SignBitTest, LHS: Or, RHS: Op0);
2905 // The STRICT_SINT_TO_FP inherits the exception mode from the
2906 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2907 // never raise any exception.
2908 SDNodeFlags Flags;
2909 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2910 Fast = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: {DestVT, MVT::Other},
2911 Ops: {Node->getOperand(Num: 0), InCvt}, Flags);
2912 Flags.setNoFPExcept(true);
2913 Slow = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: {DestVT, MVT::Other},
2914 Ops: {Fast.getValue(R: 1), Fast, Fast}, Flags);
2915 Chain = Slow.getValue(R: 1);
2916 } else {
2917 SDValue SignCvt = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Or);
2918 Slow = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: SignCvt, N2: SignCvt);
2919 Fast = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2920 }
2921
2922 return DAG.getSelect(DL: dl, VT: DestVT, Cond: SignBitTest, LHS: Slow, RHS: Fast);
2923 }
2924
2925 // Don't expand it if there isn't cheap fadd.
2926 if (!TLI.isOperationLegalOrCustom(
2927 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, VT: DestVT))
2928 return SDValue();
2929
2930 // The following optimization is valid only if every value in SrcVT (when
2931 // treated as signed) is representable in DestVT. Check that the mantissa
2932 // size of DestVT is >= than the number of bits in SrcVT -1.
2933 assert(APFloat::semanticsPrecision(DestVT.getFltSemantics()) >=
2934 SrcVT.getSizeInBits() - 1 &&
2935 "Cannot perform lossless SINT_TO_FP!");
2936
2937 SDValue Tmp1;
2938 if (Node->isStrictFPOpcode()) {
2939 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: dl, ResultTys: { DestVT, MVT::Other },
2940 Ops: { Node->getOperand(Num: 0), Op0 });
2941 } else
2942 Tmp1 = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: dl, VT: DestVT, Operand: Op0);
2943
2944 SDValue SignSet = DAG.getSetCC(DL: dl, VT: getSetCCResultType(VT: SrcVT), LHS: Op0,
2945 RHS: DAG.getConstant(Val: 0, DL: dl, VT: SrcVT), Cond: ISD::SETLT);
2946 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL: dl),
2947 Four = DAG.getIntPtrConstant(Val: 4, DL: dl);
2948 SDValue CstOffset = DAG.getSelect(DL: dl, VT: Zero.getValueType(),
2949 Cond: SignSet, LHS: Four, RHS: Zero);
2950
2951 // If the sign bit of the integer is set, the large number will be treated
2952 // as a negative number. To counteract this, the dynamic code adds an
2953 // offset depending on the data type.
2954 uint64_t FF;
2955 switch (SrcVT.getSimpleVT().SimpleTy) {
2956 default:
2957 return SDValue();
2958 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2959 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2960 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2961 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2962 }
2963 if (DAG.getDataLayout().isLittleEndian())
2964 FF <<= 32;
2965 Constant *FudgeFactor = ConstantInt::get(
2966 Ty: Type::getInt64Ty(C&: *DAG.getContext()), V: FF);
2967
2968 SDValue CPIdx =
2969 DAG.getConstantPool(C: FudgeFactor, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
2970 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
2971 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
2972 Alignment = commonAlignment(A: Alignment, Offset: 4);
2973 SDValue FudgeInReg;
2974 if (DestVT == MVT::f32)
2975 FudgeInReg = DAG.getLoad(
2976 VT: MVT::f32, dl, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2977 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()),
2978 Alignment);
2979 else {
2980 SDValue Load = DAG.getExtLoad(
2981 ExtType: ISD::EXTLOAD, dl, VT: DestVT, Chain: DAG.getEntryNode(), Ptr: CPIdx,
2982 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()), MemVT: MVT::f32,
2983 Alignment);
2984 HandleSDNode Handle(Load);
2985 LegalizeOp(Node: Load.getNode());
2986 FudgeInReg = Handle.getValue();
2987 }
2988
2989 if (Node->isStrictFPOpcode()) {
2990 SDValue Result = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, ResultTys: { DestVT, MVT::Other },
2991 Ops: { Tmp1.getValue(R: 1), Tmp1, FudgeInReg });
2992 Chain = Result.getValue(R: 1);
2993 return Result;
2994 }
2995
2996 return DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DestVT, N1: Tmp1, N2: FudgeInReg);
2997}
2998
2999/// This function is responsible for legalizing a
3000/// *INT_TO_FP operation of the specified operand when the target requests that
3001/// we promote it. At this point, we know that the result and operand types are
3002/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
3003/// operation that takes a larger input.
3004void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
3005 SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
3006 bool IsStrict = N->isStrictFPOpcode();
3007 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
3008 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
3009 EVT DestVT = N->getValueType(ResNo: 0);
3010 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3011 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
3012 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
3013
3014 // First step, figure out the appropriate *INT_TO_FP operation to use.
3015 EVT NewInTy = LegalOp.getValueType();
3016
3017 unsigned OpToUse = 0;
3018
3019 // Scan for the appropriate larger type to use.
3020 while (true) {
3021 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
3022 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3023
3024 // If the target supports SINT_TO_FP of this type, use it.
3025 if (TLI.isOperationLegalOrCustom(Op: SIntOp, VT: NewInTy)) {
3026 OpToUse = SIntOp;
3027 break;
3028 }
3029 if (IsSigned)
3030 continue;
3031
3032 // If the target supports UINT_TO_FP of this type, use it.
3033 if (TLI.isOperationLegalOrCustom(Op: UIntOp, VT: NewInTy)) {
3034 OpToUse = UIntOp;
3035 break;
3036 }
3037
3038 // Otherwise, try a larger type.
3039 }
3040
3041 // Okay, we found the operation and type to use. Zero extend our input to the
3042 // desired type then run the operation on it.
3043 if (IsStrict) {
3044 SDValue Res =
3045 DAG.getNode(Opcode: OpToUse, DL: dl, ResultTys: {DestVT, MVT::Other},
3046 Ops: {N->getOperand(Num: 0),
3047 DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3048 DL: dl, VT: NewInTy, Operand: LegalOp)});
3049 Results.push_back(Elt: Res);
3050 Results.push_back(Elt: Res.getValue(R: 1));
3051 return;
3052 }
3053
3054 Results.push_back(
3055 Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VT: DestVT,
3056 Operand: DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
3057 DL: dl, VT: NewInTy, Operand: LegalOp)));
3058}
3059
3060/// This function is responsible for legalizing a
3061/// FP_TO_*INT operation of the specified operand when the target requests that
3062/// we promote it. At this point, we know that the result and operand types are
3063/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
3064/// operation that returns a larger result.
3065void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3066 SmallVectorImpl<SDValue> &Results) {
3067 bool IsStrict = N->isStrictFPOpcode();
3068 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3069 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3070 EVT DestVT = N->getValueType(ResNo: 0);
3071 SDValue LegalOp = N->getOperand(Num: IsStrict ? 1 : 0);
3072 // First step, figure out the appropriate FP_TO*INT operation to use.
3073 EVT NewOutTy = DestVT;
3074
3075 unsigned OpToUse = 0;
3076
3077 // Scan for the appropriate larger type to use.
3078 while (true) {
3079 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
3080 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3081
3082 // A larger signed type can hold all unsigned values of the requested type,
3083 // so using FP_TO_SINT is valid
3084 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
3085 if (TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3086 break;
3087
3088 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
3089 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
3090 if (!IsSigned && TLI.isOperationLegalOrCustom(Op: OpToUse, VT: NewOutTy))
3091 break;
3092
3093 // Otherwise, try a larger type.
3094 }
3095
3096 // Okay, we found the operation and type to use.
3097 SDValue Operation;
3098 if (IsStrict) {
3099 SDVTList VTs = DAG.getVTList(VT1: NewOutTy, VT2: MVT::Other);
3100 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: N->getOperand(Num: 0), N2: LegalOp);
3101 } else
3102 Operation = DAG.getNode(Opcode: OpToUse, DL: dl, VT: NewOutTy, Operand: LegalOp);
3103
3104 // Truncate the result of the extended FP_TO_*INT operation to the desired
3105 // size.
3106 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: DestVT, Operand: Operation);
3107 Results.push_back(Elt: Trunc);
3108 if (IsStrict)
3109 Results.push_back(Elt: Operation.getValue(R: 1));
3110}
3111
3112/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
3113/// the result and operand types are legal and there must be a legal
3114/// FP_TO_*INT_SAT operation for a larger result type.
3115SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3116 const SDLoc &dl) {
3117 unsigned Opcode = Node->getOpcode();
3118
3119 // Scan for the appropriate larger type to use.
3120 EVT NewOutTy = Node->getValueType(ResNo: 0);
3121 while (true) {
3122 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
3123 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3124
3125 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: NewOutTy))
3126 break;
3127 }
3128
3129 // Saturation width is determined by second operand, so we don't have to
3130 // perform any fixup and can directly truncate the result.
3131 SDValue Result = DAG.getNode(Opcode, DL: dl, VT: NewOutTy, N1: Node->getOperand(Num: 0),
3132 N2: Node->getOperand(Num: 1));
3133 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Result);
3134}
3135
3136/// Open code the operations for PARITY of the specified operation.
3137SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3138 EVT VT = Op.getValueType();
3139 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
3140 unsigned Sz = VT.getScalarSizeInBits();
3141
3142 // If CTPOP is legal, use it. Otherwise use shifts and xor.
3143 SDValue Result;
3144 if (TLI.isOperationLegalOrPromote(Op: ISD::CTPOP, VT)) {
3145 Result = DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
3146 } else {
3147 Result = Op;
3148 for (unsigned i = Log2_32_Ceil(Value: Sz); i != 0;) {
3149 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Result,
3150 N2: DAG.getConstant(Val: 1ULL << (--i), DL: dl, VT: ShVT));
3151 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Result, N2: Shift);
3152 }
3153 }
3154
3155 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Result, N2: DAG.getConstant(Val: 1, DL: dl, VT));
3156}
3157
3158SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3159 bool IsVPOpcode = ISD::isVPOpcode(Opcode: Node->getOpcode());
3160 MVT VecVT = IsVPOpcode ? Node->getOperand(Num: 1).getSimpleValueType()
3161 : Node->getOperand(Num: 0).getSimpleValueType();
3162 MVT NewVecVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: VecVT);
3163 MVT ScalarVT = Node->getSimpleValueType(ResNo: 0);
3164 MVT NewScalarVT = NewVecVT.getVectorElementType();
3165
3166 SDLoc DL(Node);
3167 SmallVector<SDValue, 4> Operands(Node->getNumOperands());
3168
3169 // FIXME: Support integer.
3170 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3171 "Only FP promotion is supported");
3172
3173 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3174 if (Node->getOperand(Num: j).getValueType().isVector() &&
3175 !(IsVPOpcode &&
3176 ISD::getVPMaskIdx(Opcode: Node->getOpcode()) == j)) { // Skip mask operand.
3177 // promote the vector operand.
3178 // FIXME: Support integer.
3179 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3180 "Only FP promotion is supported");
3181 Operands[j] =
3182 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewVecVT, Operand: Node->getOperand(Num: j));
3183 } else if (Node->getOperand(Num: j).getValueType().isFloatingPoint()) {
3184 // promote the initial value.
3185 Operands[j] =
3186 DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NewScalarVT, Operand: Node->getOperand(Num: j));
3187 } else {
3188 Operands[j] = Node->getOperand(Num: j); // Skip VL operand.
3189 }
3190
3191 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: NewScalarVT, Ops: Operands,
3192 Flags: Node->getFlags());
3193
3194 assert(ScalarVT.isFloatingPoint() && "Only FP promotion is supported");
3195 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: ScalarVT, N1: Res,
3196 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3197}
3198
3199bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3200 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3201 SmallVector<SDValue, 8> Results;
3202 SDLoc dl(Node);
3203 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3204 bool NeedInvert;
3205 switch (Node->getOpcode()) {
3206 case ISD::ABS:
3207 case ISD::ABS_MIN_POISON:
3208 if ((Tmp1 = TLI.expandABS(N: Node, DAG)))
3209 Results.push_back(Elt: Tmp1);
3210 break;
3211 case ISD::ABDS:
3212 case ISD::ABDU:
3213 if ((Tmp1 = TLI.expandABD(N: Node, DAG)))
3214 Results.push_back(Elt: Tmp1);
3215 break;
3216 case ISD::AVGCEILS:
3217 case ISD::AVGCEILU:
3218 case ISD::AVGFLOORS:
3219 case ISD::AVGFLOORU:
3220 if ((Tmp1 = TLI.expandAVG(N: Node, DAG)))
3221 Results.push_back(Elt: Tmp1);
3222 break;
3223 case ISD::CTPOP:
3224 if ((Tmp1 = TLI.expandCTPOP(N: Node, DAG)))
3225 Results.push_back(Elt: Tmp1);
3226 break;
3227 case ISD::CTLZ:
3228 case ISD::CTLZ_ZERO_POISON:
3229 if ((Tmp1 = TLI.expandCTLZ(N: Node, DAG)))
3230 Results.push_back(Elt: Tmp1);
3231 break;
3232 case ISD::CTLS:
3233 if ((Tmp1 = TLI.expandCTLS(N: Node, DAG)))
3234 Results.push_back(Elt: Tmp1);
3235 break;
3236 case ISD::CTTZ:
3237 case ISD::CTTZ_ZERO_POISON:
3238 if ((Tmp1 = TLI.expandCTTZ(N: Node, DAG)))
3239 Results.push_back(Elt: Tmp1);
3240 break;
3241 case ISD::BITREVERSE:
3242 if ((Tmp1 = TLI.expandBITREVERSE(N: Node, DAG)))
3243 Results.push_back(Elt: Tmp1);
3244 break;
3245 case ISD::BSWAP:
3246 if ((Tmp1 = TLI.expandBSWAP(N: Node, DAG)))
3247 Results.push_back(Elt: Tmp1);
3248 break;
3249 case ISD::PARITY:
3250 Results.push_back(Elt: ExpandPARITY(Op: Node->getOperand(Num: 0), dl));
3251 break;
3252 case ISD::FRAMEADDR:
3253 case ISD::RETURNADDR:
3254 case ISD::FRAME_TO_ARGS_OFFSET:
3255 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3256 break;
3257 case ISD::EH_DWARF_CFA: {
3258 SDValue CfaArg = DAG.getSExtOrTrunc(Op: Node->getOperand(Num: 0), DL: dl,
3259 VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
3260 SDValue Offset = DAG.getNode(Opcode: ISD::ADD, DL: dl,
3261 VT: CfaArg.getValueType(),
3262 N1: DAG.getNode(Opcode: ISD::FRAME_TO_ARGS_OFFSET, DL: dl,
3263 VT: CfaArg.getValueType()),
3264 N2: CfaArg);
3265 SDValue FA = DAG.getNode(
3266 Opcode: ISD::FRAMEADDR, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
3267 Operand: DAG.getConstant(Val: 0, DL: dl, VT: TLI.getPointerTy(DL: DAG.getDataLayout())));
3268 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FA.getValueType(),
3269 N1: FA, N2: Offset));
3270 break;
3271 }
3272 case ISD::GET_ROUNDING:
3273 Results.push_back(Elt: DAG.getConstant(Val: 1, DL: dl, VT: Node->getValueType(ResNo: 0)));
3274 Results.push_back(Elt: Node->getOperand(Num: 0));
3275 break;
3276 case ISD::EH_RETURN:
3277 case ISD::PREFETCH:
3278 case ISD::VAEND:
3279 case ISD::EH_SJLJ_LONGJMP:
3280 // If the target didn't expand these, there's nothing to do, so just
3281 // preserve the chain and be done.
3282 Results.push_back(Elt: Node->getOperand(Num: 0));
3283 break;
3284 case ISD::READCYCLECOUNTER:
3285 case ISD::READSTEADYCOUNTER:
3286 // If the target didn't expand this, just return 'zero' and preserve the
3287 // chain.
3288 Results.append(NumInputs: Node->getNumValues() - 1,
3289 Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3290 Results.push_back(Elt: Node->getOperand(Num: 0));
3291 break;
3292 case ISD::EH_SJLJ_SETJMP:
3293 // If the target didn't expand this, just return 'zero' and preserve the
3294 // chain.
3295 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
3296 Results.push_back(Elt: Node->getOperand(Num: 0));
3297 break;
3298 case ISD::ATOMIC_LOAD: {
3299 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3300 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0));
3301 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3302 SDValue Swap = DAG.getAtomicCmpSwap(
3303 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3304 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Zero, Swp: Zero,
3305 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3306 Results.push_back(Elt: Swap.getValue(R: 0));
3307 Results.push_back(Elt: Swap.getValue(R: 1));
3308 break;
3309 }
3310 case ISD::ATOMIC_STORE: {
3311 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3312 SDValue Swap = DAG.getAtomic(
3313 Opcode: ISD::ATOMIC_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(),
3314 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 2), Val: Node->getOperand(Num: 1),
3315 MMO: cast<AtomicSDNode>(Val: Node)->getMemOperand());
3316 Results.push_back(Elt: Swap.getValue(R: 1));
3317 break;
3318 }
3319 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3320 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3321 // splits out the success value as a comparison. Expanding the resulting
3322 // ATOMIC_CMP_SWAP will produce a libcall.
3323 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
3324 SDValue Res = DAG.getAtomicCmpSwap(
3325 Opcode: ISD::ATOMIC_CMP_SWAP, dl, MemVT: cast<AtomicSDNode>(Val: Node)->getMemoryVT(), VTs,
3326 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1), Cmp: Node->getOperand(Num: 2),
3327 Swp: Node->getOperand(Num: 3), MMO: cast<MemSDNode>(Val: Node)->getMemOperand());
3328
3329 SDValue ExtRes = Res;
3330 SDValue LHS = Res;
3331 SDValue RHS = Node->getOperand(Num: 1);
3332
3333 EVT AtomicType = cast<AtomicSDNode>(Val: Node)->getMemoryVT();
3334 EVT OuterType = Node->getValueType(ResNo: 0);
3335 switch (TLI.getExtendForAtomicOps()) {
3336 case ISD::SIGN_EXTEND:
3337 LHS = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: OuterType, N1: Res,
3338 N2: DAG.getValueType(AtomicType));
3339 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OuterType,
3340 N1: Node->getOperand(Num: 2), N2: DAG.getValueType(AtomicType));
3341 ExtRes = LHS;
3342 break;
3343 case ISD::ZERO_EXTEND:
3344 LHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: OuterType, N1: Res,
3345 N2: DAG.getValueType(AtomicType));
3346 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3347 ExtRes = LHS;
3348 break;
3349 case ISD::ANY_EXTEND:
3350 LHS = DAG.getZeroExtendInReg(Op: Res, DL: dl, VT: AtomicType);
3351 RHS = DAG.getZeroExtendInReg(Op: Node->getOperand(Num: 2), DL: dl, VT: AtomicType);
3352 break;
3353 default:
3354 llvm_unreachable("Invalid atomic op extension");
3355 }
3356
3357 SDValue Success =
3358 DAG.getSetCC(DL: dl, VT: Node->getValueType(ResNo: 1), LHS, RHS, Cond: ISD::SETEQ);
3359
3360 Results.push_back(Elt: ExtRes.getValue(R: 0));
3361 Results.push_back(Elt: Success);
3362 Results.push_back(Elt: Res.getValue(R: 1));
3363 break;
3364 }
3365 case ISD::ATOMIC_LOAD_SUB: {
3366 SDLoc DL(Node);
3367 EVT VT = Node->getValueType(ResNo: 0);
3368 SDValue RHS = Node->getOperand(Num: 2);
3369 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3370 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3371 cast<VTSDNode>(Val: RHS->getOperand(Num: 1))->getVT() == AN->getMemoryVT())
3372 RHS = RHS->getOperand(Num: 0);
3373 SDValue NewRHS =
3374 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: RHS);
3375 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_ADD, dl: DL, MemVT: AN->getMemoryVT(),
3376 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3377 Val: NewRHS, MMO: AN->getMemOperand());
3378 Results.push_back(Elt: Res);
3379 Results.push_back(Elt: Res.getValue(R: 1));
3380 break;
3381 }
3382 case ISD::ATOMIC_LOAD_AND: {
3383 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3384 if (TLI.getOperationAction(Op: ISD::ATOMIC_LOAD_CLR, VT: AN->getMemoryVT()) ==
3385 TargetLowering::Expand)
3386 break;
3387
3388 SDLoc DL(Node);
3389 EVT VT = Node->getValueType(ResNo: 0);
3390 SDValue NewRHS = DAG.getNOT(DL, Val: Node->getOperand(Num: 2), VT);
3391 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_CLR, dl: DL, MemVT: AN->getMemoryVT(),
3392 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3393 Val: NewRHS, MMO: AN->getMemOperand());
3394 Results.push_back(Elt: Res);
3395 Results.push_back(Elt: Res.getValue(R: 1));
3396 break;
3397 }
3398 case ISD::ATOMIC_LOAD_FSUB: {
3399 SDLoc DL(Node);
3400 EVT VT = Node->getValueType(ResNo: 0);
3401 AtomicSDNode *AN = cast<AtomicSDNode>(Val: Node);
3402 SDValue NewRHS = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Node->getOperand(Num: 2));
3403 SDValue Res = DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD_FADD, dl: DL, MemVT: AN->getMemoryVT(),
3404 Chain: Node->getOperand(Num: 0), Ptr: Node->getOperand(Num: 1),
3405 Val: NewRHS, MMO: AN->getMemOperand());
3406 Results.push_back(Elt: Res);
3407 Results.push_back(Elt: Res.getValue(R: 1));
3408 break;
3409 }
3410 case ISD::DYNAMIC_STACKALLOC:
3411 ExpandDYNAMIC_STACKALLOC(Node, Results);
3412 break;
3413 case ISD::MERGE_VALUES:
3414 for (unsigned i = 0; i < Node->getNumValues(); i++)
3415 Results.push_back(Elt: Node->getOperand(Num: i));
3416 break;
3417 case ISD::POISON:
3418 case ISD::UNDEF: {
3419 EVT VT = Node->getValueType(ResNo: 0);
3420 if (VT.isInteger())
3421 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT));
3422 else {
3423 assert(VT.isFloatingPoint() && "Unknown value type!");
3424 Results.push_back(Elt: DAG.getConstantFP(Val: 0, DL: dl, VT));
3425 }
3426 break;
3427 }
3428 case ISD::STRICT_FP_ROUND:
3429 // When strict mode is enforced we can't do expansion because it
3430 // does not honor the "strict" properties. Only libcall is allowed.
3431 if (TLI.isStrictFPEnabled())
3432 break;
3433 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3434 // since this operation is more efficient than stack operation.
3435 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3436 VT: Node->getValueType(ResNo: 0))
3437 == TargetLowering::Legal)
3438 break;
3439 // We fall back to use stack operation when the FP_ROUND operation
3440 // isn't available.
3441 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getValueType(ResNo: 0),
3442 DestVT: Node->getValueType(ResNo: 0), dl,
3443 Chain: Node->getOperand(Num: 0)))) {
3444 ReplaceNode(Old: Node, New: Tmp1.getNode());
3445 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3446 return true;
3447 }
3448 break;
3449 case ISD::FP_ROUND: {
3450 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3451 Results.push_back(Elt: Tmp1);
3452 break;
3453 }
3454
3455 [[fallthrough]];
3456 }
3457 case ISD::BITCAST:
3458 if ((Tmp1 = EmitStackConvert(SrcOp: Node->getOperand(Num: 0), SlotVT: Node->getValueType(ResNo: 0),
3459 DestVT: Node->getValueType(ResNo: 0), dl)))
3460 Results.push_back(Elt: Tmp1);
3461 break;
3462 case ISD::STRICT_FP_EXTEND:
3463 // When strict mode is enforced we can't do expansion because it
3464 // does not honor the "strict" properties. Only libcall is allowed.
3465 if (TLI.isStrictFPEnabled())
3466 break;
3467 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3468 // since this operation is more efficient than stack operation.
3469 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
3470 VT: Node->getValueType(ResNo: 0))
3471 == TargetLowering::Legal)
3472 break;
3473 // We fall back to use stack operation when the FP_EXTEND operation
3474 // isn't available.
3475 if ((Tmp1 = EmitStackConvert(
3476 SrcOp: Node->getOperand(Num: 1), SlotVT: Node->getOperand(Num: 1).getValueType(),
3477 DestVT: Node->getValueType(ResNo: 0), dl, Chain: Node->getOperand(Num: 0)))) {
3478 ReplaceNode(Old: Node, New: Tmp1.getNode());
3479 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3480 return true;
3481 }
3482 break;
3483 case ISD::FP_EXTEND: {
3484 SDValue Op = Node->getOperand(Num: 0);
3485 EVT SrcVT = Op.getValueType();
3486 EVT DstVT = Node->getValueType(ResNo: 0);
3487 if (SrcVT.getScalarType() == MVT::bf16) {
3488 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BF16_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Op));
3489 break;
3490 }
3491
3492 if ((Tmp1 = EmitStackConvert(SrcOp: Op, SlotVT: SrcVT, DestVT: DstVT, dl)))
3493 Results.push_back(Elt: Tmp1);
3494 break;
3495 }
3496 case ISD::BF16_TO_FP: {
3497 // Always expand bf16 to f32 casts, they lower to ext + shift.
3498 //
3499 // Note that the operand of this code can be bf16 or an integer type in case
3500 // bf16 is not supported on the target and was softened.
3501 SDValue Op = Node->getOperand(Num: 0);
3502 if (Op.getValueType() == MVT::bf16) {
3503 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32,
3504 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i16, Operand: Op));
3505 } else {
3506 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: MVT::i32);
3507 }
3508 Op = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i32, N1: Op,
3509 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3510 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f32, Operand: Op);
3511 // Add fp_extend in case the output is bigger than f32.
3512 if (Node->getValueType(ResNo: 0) != MVT::f32)
3513 Op = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op);
3514 Results.push_back(Elt: Op);
3515 break;
3516 }
3517 case ISD::FP_TO_BF16: {
3518 SDValue Op = Node->getOperand(Num: 0);
3519 if (Op.getValueType() != MVT::f32)
3520 Op = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3521 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3522 // Certain SNaNs will turn into infinities if we do a simple shift right.
3523 if (!DAG.isKnownNeverSNaN(Op)) {
3524 Op = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT: MVT::f32, Operand: Op, Flags: Node->getFlags());
3525 }
3526 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
3527 N1: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Op),
3528 N2: DAG.getShiftAmountConstant(Val: 16, VT: MVT::i32, DL: dl));
3529 // The result of this node can be bf16 or an integer type in case bf16 is
3530 // not supported on the target and was softened to i16 for storage.
3531 if (Node->getValueType(ResNo: 0) == MVT::bf16) {
3532 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::bf16,
3533 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: Op));
3534 } else {
3535 Op = DAG.getAnyExtOrTrunc(Op, DL: dl, VT: Node->getValueType(ResNo: 0));
3536 }
3537 Results.push_back(Elt: Op);
3538 break;
3539 }
3540 case ISD::CONVERT_FROM_ARBITRARY_FP: {
3541 // Expand conversion from arbitrary FP format stored in an integer to a
3542 // native IEEE float type using integer bit manipulation.
3543 //
3544 // TODO: currently only conversions from FP4, FP6 and FP8 formats from OCP
3545 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3546 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3547 // Float8E8M0FNU.
3548 EVT DstVT = Node->getValueType(ResNo: 0);
3549 if (SDValue Expanded = TLI.expandCONVERT_FROM_ARBITRARY_FP(Node, DAG))
3550 Results.push_back(Elt: Expanded);
3551 else
3552 Results.push_back(Elt: DAG.getPOISON(VT: DstVT));
3553 break;
3554 }
3555 case ISD::CONVERT_TO_ARBITRARY_FP: {
3556 // Expand conversion from a native IEEE float type to an arbitrary FP
3557 // format, returning the result as an integer using bit manipulation.
3558 //
3559 // TODO: currently only conversions to FP4, FP6 and FP8 formats from OCP
3560 // specification are expanded. Remaining arbitrary FP types: Float8E4M3,
3561 // Float8E3M4, Float8E5M2FNUZ, Float8E4M3FNUZ, Float8E4M3B11FNUZ,
3562 // Float8E8M0FNU.
3563 EVT ResVT = Node->getValueType(ResNo: 0);
3564 if (SDValue Expanded = TLI.expandCONVERT_TO_ARBITRARY_FP(Node, DAG))
3565 Results.push_back(Elt: Expanded);
3566 else
3567 Results.push_back(Elt: DAG.getPOISON(VT: ResVT));
3568 break;
3569 }
3570 case ISD::FCANONICALIZE: {
3571 SDValue Mul = TLI.expandFCANONICALIZE(Node, DAG);
3572 Results.push_back(Elt: Mul);
3573 break;
3574 }
3575 case ISD::SIGN_EXTEND_INREG: {
3576 EVT ExtraVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
3577 EVT VT = Node->getValueType(ResNo: 0);
3578
3579 // An in-register sign-extend of a boolean is a negation:
3580 // 'true' (1) sign-extended is -1.
3581 // 'false' (0) sign-extended is 0.
3582 // However, we must mask the high bits of the source operand because the
3583 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3584
3585 // TODO: Do this for vectors too?
3586 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3587 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
3588 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: One);
3589 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
3590 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: And);
3591 Results.push_back(Elt: Neg);
3592 break;
3593 }
3594
3595 // NOTE: we could fall back on load/store here too for targets without
3596 // SRA. However, it is doubtful that any exist.
3597 unsigned BitsDiff = VT.getScalarSizeInBits() -
3598 ExtraVT.getScalarSizeInBits();
3599 SDValue ShiftCst = DAG.getShiftAmountConstant(Val: BitsDiff, VT, DL: dl);
3600 Tmp1 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: ShiftCst);
3601 Tmp1 = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Tmp1, N2: ShiftCst);
3602 Results.push_back(Elt: Tmp1);
3603 break;
3604 }
3605 case ISD::UINT_TO_FP:
3606 case ISD::STRICT_UINT_TO_FP:
3607 if (TLI.expandUINT_TO_FP(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3608 Results.push_back(Elt: Tmp1);
3609 if (Node->isStrictFPOpcode())
3610 Results.push_back(Elt: Tmp2);
3611 break;
3612 }
3613 [[fallthrough]];
3614 case ISD::SINT_TO_FP:
3615 case ISD::STRICT_SINT_TO_FP:
3616 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Chain&: Tmp2))) {
3617 Results.push_back(Elt: Tmp1);
3618 if (Node->isStrictFPOpcode())
3619 Results.push_back(Elt: Tmp2);
3620 }
3621 break;
3622 case ISD::FP_TO_SINT:
3623 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG))
3624 Results.push_back(Elt: Tmp1);
3625 break;
3626 case ISD::STRICT_FP_TO_SINT:
3627 if (TLI.expandFP_TO_SINT(N: Node, Result&: Tmp1, DAG)) {
3628 ReplaceNode(Old: Node, New: Tmp1.getNode());
3629 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3630 return true;
3631 }
3632 break;
3633 case ISD::FP_TO_UINT:
3634 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG))
3635 Results.push_back(Elt: Tmp1);
3636 break;
3637 case ISD::STRICT_FP_TO_UINT:
3638 if (TLI.expandFP_TO_UINT(N: Node, Result&: Tmp1, Chain&: Tmp2, DAG)) {
3639 // Relink the chain.
3640 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node,1), To: Tmp2);
3641 // Replace the new UINT result.
3642 ReplaceNodeWithValue(Old: SDValue(Node, 0), New: Tmp1);
3643 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3644 return true;
3645 }
3646 break;
3647 case ISD::FP_TO_SINT_SAT:
3648 case ISD::FP_TO_UINT_SAT:
3649 Results.push_back(Elt: TLI.expandFP_TO_INT_SAT(N: Node, DAG));
3650 break;
3651 case ISD::LROUND:
3652 case ISD::LLROUND: {
3653 SDValue Arg = Node->getOperand(Num: 0);
3654 EVT ArgVT = Arg.getValueType();
3655 EVT ResVT = Node->getValueType(ResNo: 0);
3656 SDLoc dl(Node);
3657 SDValue RoundNode = DAG.getNode(Opcode: ISD::FROUND, DL: dl, VT: ArgVT, Operand: Arg);
3658 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: ResVT, Operand: RoundNode));
3659 break;
3660 }
3661 case ISD::VAARG:
3662 Results.push_back(Elt: DAG.expandVAArg(Node));
3663 Results.push_back(Elt: Results[0].getValue(R: 1));
3664 break;
3665 case ISD::VACOPY:
3666 Results.push_back(Elt: DAG.expandVACopy(Node));
3667 break;
3668 case ISD::EXTRACT_VECTOR_ELT:
3669 if (Node->getOperand(Num: 0).getValueType().getVectorElementCount().isScalar())
3670 // This must be an access of the only element. Return it.
3671 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0),
3672 Operand: Node->getOperand(Num: 0));
3673 else
3674 Tmp1 = ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0));
3675 Results.push_back(Elt: Tmp1);
3676 break;
3677 case ISD::EXTRACT_SUBVECTOR:
3678 Results.push_back(Elt: ExpandExtractFromVectorThroughStack(Op: SDValue(Node, 0)));
3679 break;
3680 case ISD::INSERT_SUBVECTOR:
3681 Results.push_back(Elt: ExpandInsertToVectorThroughStack(Op: SDValue(Node, 0)));
3682 break;
3683 case ISD::CONCAT_VECTORS:
3684 if (EVT VectorValueType = Node->getOperand(Num: 0).getValueType();
3685 VectorValueType.isScalableVector() ||
3686 TLI.isOperationExpand(Op: ISD::EXTRACT_VECTOR_ELT, VT: VectorValueType))
3687 Results.push_back(Elt: ExpandVectorBuildThroughStack(Node));
3688 else
3689 Results.push_back(Elt: ExpandConcatVectors(Node));
3690 break;
3691 case ISD::SCALAR_TO_VECTOR:
3692 Results.push_back(Elt: ExpandSCALAR_TO_VECTOR(Node));
3693 break;
3694 case ISD::INSERT_VECTOR_ELT:
3695 Results.push_back(Elt: ExpandINSERT_VECTOR_ELT(Op: SDValue(Node, 0)));
3696 break;
3697 case ISD::VECTOR_SHUFFLE: {
3698 SmallVector<int, 32> NewMask;
3699 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
3700
3701 EVT VT = Node->getValueType(ResNo: 0);
3702 EVT EltVT = VT.getVectorElementType();
3703 SDValue Op0 = Node->getOperand(Num: 0);
3704 SDValue Op1 = Node->getOperand(Num: 1);
3705 if (!TLI.isTypeLegal(VT: EltVT)) {
3706 EVT NewEltVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT);
3707
3708 // BUILD_VECTOR operands are allowed to be wider than the element type.
3709 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3710 // it.
3711 if (NewEltVT.bitsLT(VT: EltVT)) {
3712 // Convert shuffle node.
3713 // If original node was v4i64 and the new EltVT is i32,
3714 // cast operands to v8i32 and re-build the mask.
3715
3716 // Calculate new VT, the size of the new VT should be equal to original.
3717 EVT NewVT =
3718 EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewEltVT,
3719 NumElements: VT.getSizeInBits() / NewEltVT.getSizeInBits());
3720 assert(NewVT.bitsEq(VT));
3721
3722 // cast operands to new VT
3723 Op0 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op0);
3724 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NewVT, Operand: Op1);
3725
3726 // Convert the shuffle mask
3727 unsigned int factor =
3728 NewVT.getVectorNumElements()/VT.getVectorNumElements();
3729
3730 // EltVT gets smaller
3731 assert(factor > 0);
3732
3733 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3734 if (Mask[i] < 0) {
3735 for (unsigned fi = 0; fi < factor; ++fi)
3736 NewMask.push_back(Elt: Mask[i]);
3737 }
3738 else {
3739 for (unsigned fi = 0; fi < factor; ++fi)
3740 NewMask.push_back(Elt: Mask[i]*factor+fi);
3741 }
3742 }
3743 Mask = NewMask;
3744 VT = NewVT;
3745 }
3746 EltVT = NewEltVT;
3747 }
3748 unsigned NumElems = VT.getVectorNumElements();
3749 SmallVector<SDValue, 16> Ops;
3750 for (unsigned i = 0; i != NumElems; ++i) {
3751 if (Mask[i] < 0) {
3752 Ops.push_back(Elt: DAG.getUNDEF(VT: EltVT));
3753 continue;
3754 }
3755 unsigned Idx = Mask[i];
3756 if (Idx < NumElems)
3757 Ops.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op0,
3758 N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl)));
3759 else
3760 Ops.push_back(
3761 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: EltVT, N1: Op1,
3762 N2: DAG.getVectorIdxConstant(Val: Idx - NumElems, DL: dl)));
3763 }
3764
3765 Tmp1 = DAG.getBuildVector(VT, DL: dl, Ops);
3766 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3767 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3768 Results.push_back(Elt: Tmp1);
3769 break;
3770 }
3771 case ISD::VECTOR_SPLICE_LEFT:
3772 case ISD::VECTOR_SPLICE_RIGHT: {
3773 Results.push_back(Elt: TLI.expandVectorSplice(Node, DAG));
3774 break;
3775 }
3776 case ISD::VECTOR_INTERLEAVE:
3777 TLI.expandVectorInterleave(Node, Results, DAG);
3778 break;
3779 case ISD::VECTOR_DEINTERLEAVE:
3780 TLI.expandVectorDeinterleave(Node, Results, DAG);
3781 break;
3782 case ISD::EXTRACT_ELEMENT: {
3783 EVT OpTy = Node->getOperand(Num: 0).getValueType();
3784 if (Node->getConstantOperandVal(Num: 1)) {
3785 // 1 -> Hi
3786 Tmp1 = DAG.getNode(
3787 Opcode: ISD::SRL, DL: dl, VT: OpTy, N1: Node->getOperand(Num: 0),
3788 N2: DAG.getShiftAmountConstant(Val: OpTy.getSizeInBits() / 2, VT: OpTy, DL: dl));
3789 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
3790 } else {
3791 // 0 -> Lo
3792 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Node->getValueType(ResNo: 0),
3793 Operand: Node->getOperand(Num: 0));
3794 }
3795 Results.push_back(Elt: Tmp1);
3796 break;
3797 }
3798 case ISD::STACKADDRESS:
3799 case ISD::STACKSAVE:
3800 // Expand to CopyFromReg if the target set
3801 // StackPointerRegisterToSaveRestore.
3802 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3803 Results.push_back(Elt: DAG.getCopyFromReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3804 VT: Node->getValueType(ResNo: 0)));
3805 Results.push_back(Elt: Results[0].getValue(R: 1));
3806 } else {
3807 Results.push_back(Elt: DAG.getUNDEF(VT: Node->getValueType(ResNo: 0)));
3808 Results.push_back(Elt: Node->getOperand(Num: 0));
3809
3810 StringRef IntrinsicName = Node->getOpcode() == ISD::STACKADDRESS
3811 ? "llvm.stackaddress"
3812 : "llvm.stacksave";
3813 DAG.getContext()->diagnose(DI: DiagnosticInfoLegalizationFailure(
3814 Twine(IntrinsicName) + " is not supported on this target.",
3815 DAG.getMachineFunction().getFunction(), dl.getDebugLoc()));
3816 }
3817 break;
3818 case ISD::STACKRESTORE:
3819 // Expand to CopyToReg if the target set
3820 // StackPointerRegisterToSaveRestore.
3821 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3822 Results.push_back(Elt: DAG.getCopyToReg(Chain: Node->getOperand(Num: 0), dl, Reg: SP,
3823 N: Node->getOperand(Num: 1)));
3824 } else {
3825 Results.push_back(Elt: Node->getOperand(Num: 0));
3826 }
3827 break;
3828 case ISD::GET_DYNAMIC_AREA_OFFSET:
3829 Results.push_back(Elt: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)));
3830 Results.push_back(Elt: Results[0].getValue(R: 0));
3831 break;
3832 case ISD::FCOPYSIGN:
3833 Results.push_back(Elt: ExpandFCOPYSIGN(Node));
3834 break;
3835 case ISD::FNEG:
3836 Results.push_back(Elt: ExpandFNEG(Node));
3837 break;
3838 case ISD::FABS:
3839 Results.push_back(Elt: ExpandFABS(Node));
3840 break;
3841 case ISD::IS_FPCLASS: {
3842 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(Num: 1));
3843 if (SDValue Expanded =
3844 TLI.expandIS_FPCLASS(ResultVT: Node->getValueType(ResNo: 0), Op: Node->getOperand(Num: 0),
3845 Test, Flags: Node->getFlags(), DL: SDLoc(Node), DAG))
3846 Results.push_back(Elt: Expanded);
3847 break;
3848 }
3849 case ISD::SMIN:
3850 case ISD::SMAX:
3851 case ISD::UMIN:
3852 case ISD::UMAX: {
3853 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3854 ISD::CondCode Pred;
3855 switch (Node->getOpcode()) {
3856 default: llvm_unreachable("How did we get here?");
3857 case ISD::SMAX: Pred = ISD::SETGT; break;
3858 case ISD::SMIN: Pred = ISD::SETLT; break;
3859 case ISD::UMAX: Pred = ISD::SETUGT; break;
3860 case ISD::UMIN: Pred = ISD::SETULT; break;
3861 }
3862 Tmp1 = Node->getOperand(Num: 0);
3863 Tmp2 = Node->getOperand(Num: 1);
3864 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp1, False: Tmp2, Cond: Pred);
3865 Results.push_back(Elt: Tmp1);
3866 break;
3867 }
3868 case ISD::FMINNUM:
3869 case ISD::FMAXNUM: {
3870 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(N: Node, DAG))
3871 Results.push_back(Elt: Expanded);
3872 break;
3873 }
3874 case ISD::FMINIMUM:
3875 case ISD::FMAXIMUM: {
3876 if (SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(N: Node, DAG))
3877 Results.push_back(Elt: Expanded);
3878 break;
3879 }
3880 case ISD::FMINIMUMNUM:
3881 case ISD::FMAXIMUMNUM: {
3882 Results.push_back(Elt: TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N: Node, DAG));
3883 break;
3884 }
3885 case ISD::FSIN:
3886 case ISD::FCOS: {
3887 EVT VT = Node->getValueType(ResNo: 0);
3888 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3889 // fcos which share the same operand and both are used.
3890 if ((TLI.isOperationLegal(Op: ISD::FSINCOS, VT) ||
3891 isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls())) &&
3892 useSinCos(Node)) {
3893 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
3894 Tmp1 = DAG.getNode(Opcode: ISD::FSINCOS, DL: dl, VTList: VTs, N: Node->getOperand(Num: 0));
3895 if (Node->getOpcode() == ISD::FCOS)
3896 Tmp1 = Tmp1.getValue(R: 1);
3897 Results.push_back(Elt: Tmp1);
3898 }
3899 break;
3900 }
3901 case ISD::FLDEXP:
3902 case ISD::STRICT_FLDEXP: {
3903 EVT VT = Node->getValueType(ResNo: 0);
3904 RTLIB::Libcall LC = RTLIB::getLDEXP(VT);
3905 // Use the LibCall instead, it is very likely faster
3906 // FIXME: Use separate LibCall action.
3907 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3908 break;
3909
3910 if (SDValue Expanded = expandLdexp(Node)) {
3911 Results.push_back(Elt: Expanded);
3912 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3913 Results.push_back(Elt: Expanded.getValue(R: 1));
3914 }
3915
3916 break;
3917 }
3918 case ISD::FFREXP: {
3919 RTLIB::Libcall LC = RTLIB::getFREXP(VT: Node->getValueType(ResNo: 0));
3920 // Use the LibCall instead, it is very likely faster
3921 // FIXME: Use separate LibCall action.
3922 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3923 break;
3924
3925 if (SDValue Expanded = expandFrexp(Node)) {
3926 Results.push_back(Elt: Expanded);
3927 Results.push_back(Elt: Expanded.getValue(R: 1));
3928 }
3929 break;
3930 }
3931 case ISD::FMODF: {
3932 RTLIB::Libcall LC = RTLIB::getMODF(VT: Node->getValueType(ResNo: 0));
3933 // Use the LibCall instead, it is very likely faster
3934 // FIXME: Use separate LibCall action.
3935 if (DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported)
3936 break;
3937
3938 TLI.expandMODF(N: Node, DAG, Results);
3939 break;
3940 }
3941 case ISD::FSINCOS: {
3942 if (isSinCosLibcallAvailable(Node, Libcalls: DAG.getLibcalls()))
3943 break;
3944 EVT VT = Node->getValueType(ResNo: 0);
3945 SDValue Op = Node->getOperand(Num: 0);
3946 SDNodeFlags Flags = Node->getFlags();
3947 Tmp1 = DAG.getNode(Opcode: ISD::FSIN, DL: dl, VT, Operand: Op, Flags);
3948 Tmp2 = DAG.getNode(Opcode: ISD::FCOS, DL: dl, VT, Operand: Op, Flags);
3949 Results.append(IL: {Tmp1, Tmp2});
3950 break;
3951 }
3952 case ISD::FMAD:
3953 llvm_unreachable("Illegal fmad should never be formed");
3954
3955 case ISD::FP16_TO_FP:
3956 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3957 // We can extend to types bigger than f32 in two steps without changing
3958 // the result. Since "f16 -> f32" is much more commonly available, give
3959 // CodeGen the option of emitting that before resorting to a libcall.
3960 SDValue Res =
3961 DAG.getNode(Opcode: ISD::FP16_TO_FP, DL: dl, VT: MVT::f32, Operand: Node->getOperand(Num: 0));
3962 Results.push_back(
3963 Elt: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res));
3964 }
3965 break;
3966 case ISD::STRICT_BF16_TO_FP:
3967 case ISD::STRICT_FP16_TO_FP:
3968 if (Node->getValueType(ResNo: 0) != MVT::f32) {
3969 // We can extend to types bigger than f32 in two steps without changing
3970 // the result. Since "f16 -> f32" is much more commonly available, give
3971 // CodeGen the option of emitting that before resorting to a libcall.
3972 SDValue Res = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {MVT::f32, MVT::Other},
3973 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
3974 Res = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
3975 ResultTys: {Node->getValueType(ResNo: 0), MVT::Other},
3976 Ops: {Res.getValue(R: 1), Res});
3977 Results.push_back(Elt: Res);
3978 Results.push_back(Elt: Res.getValue(R: 1));
3979 }
3980 break;
3981 case ISD::FP_TO_FP16:
3982 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3983 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
3984 SDValue Op = Node->getOperand(Num: 0);
3985 MVT SVT = Op.getSimpleValueType();
3986 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3987 TLI.isOperationLegalOrCustom(Op: ISD::FP_TO_FP16, VT: MVT::f32)) {
3988 // Under fastmath, we can expand this node into a fround followed by
3989 // a float-half conversion.
3990 SDValue FloatVal =
3991 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: Op,
3992 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
3993 Results.push_back(
3994 Elt: DAG.getNode(Opcode: ISD::FP_TO_FP16, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: FloatVal));
3995 }
3996 }
3997 break;
3998 case ISD::ConstantFP: {
3999 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val: Node);
4000 EVT VT = Node->getValueType(ResNo: 0);
4001 TargetLowering::FPImmAction Action =
4002 TLI.getFPImmAction(CFP->getValueAPF(), VT, ForCodeSize: DAG.shouldOptForSize());
4003 switch (Action) {
4004 case TargetLowering::FPImmAction::Legal:
4005 break;
4006 case TargetLowering::FPImmAction::Bitcast: {
4007 EVT IntVT = VT.changeTypeToInteger();
4008 assert(TLI.isTypeLegal(IntVT) &&
4009 "integer type for FP immediate bits must be legal");
4010 // The constant is opaque so the bitcast is not folded back.
4011 SDValue Bits =
4012 DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt(), DL: dl, VT: IntVT,
4013 /*isTarget=*/false, /*isOpaque=*/true);
4014 Results.push_back(Elt: DAG.getBitcast(VT, V: Bits));
4015 break;
4016 }
4017 case TargetLowering::FPImmAction::ConstantPool:
4018 Results.push_back(Elt: ExpandConstantFP(CFP));
4019 break;
4020 }
4021
4022 break;
4023 }
4024 case ISD::Constant: {
4025 ConstantSDNode *CP = cast<ConstantSDNode>(Val: Node);
4026 Results.push_back(Elt: ExpandConstant(CP));
4027 break;
4028 }
4029 case ISD::FSUB: {
4030 EVT VT = Node->getValueType(ResNo: 0);
4031 if (TLI.isOperationLegalOrCustom(Op: ISD::FADD, VT) &&
4032 TLI.isOperationLegalOrCustom(Op: ISD::FNEG, VT)) {
4033 const SDNodeFlags Flags = Node->getFlags();
4034 Tmp1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 1));
4035 Tmp1 = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1, Flags);
4036 Results.push_back(Elt: Tmp1);
4037 }
4038 break;
4039 }
4040 case ISD::SUB: {
4041 EVT VT = Node->getValueType(ResNo: 0);
4042 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
4043 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
4044 "Don't know how to expand this subtraction!");
4045 Tmp1 = DAG.getNOT(DL: dl, Val: Node->getOperand(Num: 1), VT);
4046 Tmp1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Tmp1, N2: DAG.getConstant(Val: 1, DL: dl, VT));
4047 Results.push_back(Elt: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Node->getOperand(Num: 0), N2: Tmp1));
4048 break;
4049 }
4050 case ISD::UREM:
4051 case ISD::SREM:
4052 if (TLI.expandREM(Node, Result&: Tmp1, DAG))
4053 Results.push_back(Elt: Tmp1);
4054 break;
4055 case ISD::UDIV:
4056 case ISD::SDIV: {
4057 bool isSigned = Node->getOpcode() == ISD::SDIV;
4058 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
4059 EVT VT = Node->getValueType(ResNo: 0);
4060 if (TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
4061 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4062 Tmp1 = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4063 N2: Node->getOperand(Num: 1));
4064 Results.push_back(Elt: Tmp1);
4065 }
4066 break;
4067 }
4068 case ISD::MULHU:
4069 case ISD::MULHS: {
4070 unsigned ExpandOpcode =
4071 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
4072 EVT VT = Node->getValueType(ResNo: 0);
4073 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4074
4075 Tmp1 = DAG.getNode(Opcode: ExpandOpcode, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4076 N2: Node->getOperand(Num: 1));
4077 Results.push_back(Elt: Tmp1.getValue(R: 1));
4078 break;
4079 }
4080 case ISD::UMUL_LOHI:
4081 case ISD::SMUL_LOHI: {
4082 SDValue LHS = Node->getOperand(Num: 0);
4083 SDValue RHS = Node->getOperand(Num: 1);
4084 EVT VT = LHS.getValueType();
4085 bool IsSigned = Node->getOpcode() == ISD::SMUL_LOHI;
4086 unsigned MULHOpcode = IsSigned ? ISD::MULHS : ISD::MULHU;
4087
4088 if (TLI.isOperationLegalOrCustom(Op: MULHOpcode, VT)) {
4089 Results.push_back(Elt: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS));
4090 Results.push_back(Elt: DAG.getNode(Opcode: MULHOpcode, DL: dl, VT, N1: LHS, N2: RHS));
4091 break;
4092 }
4093
4094 SmallVector<SDValue, 4> Halves;
4095 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4096 if (TLI.isTypeLegal(VT: HalfType) &&
4097 TLI.expandMUL_LOHI(Opcode: Node->getOpcode(), VT, dl, LHS, RHS, Result&: Halves,
4098 HiLoVT: HalfType, DAG,
4099 Kind: TargetLowering::MulExpansionKind::Always)) {
4100 for (unsigned i = 0; i < 2; ++i) {
4101 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Halves[2 * i]);
4102 SDValue Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Halves[2 * i + 1]);
4103 SDValue Shift =
4104 DAG.getShiftAmountConstant(Val: HalfType.getScalarSizeInBits(), VT, DL: dl);
4105 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4106 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4107 }
4108 break;
4109 }
4110
4111 SDValue Lo, Hi;
4112 TLI.forceExpandWideMUL(DAG, dl, Signed: IsSigned, LHS, RHS, Lo, Hi);
4113 Results.push_back(Elt: Lo);
4114 Results.push_back(Elt: Hi);
4115 break;
4116 }
4117 case ISD::MUL: {
4118 EVT VT = Node->getValueType(ResNo: 0);
4119 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
4120 // See if multiply or divide can be lowered using two-result operations.
4121 // We just need the low half of the multiply; try both the signed
4122 // and unsigned forms. If the target supports both SMUL_LOHI and
4123 // UMUL_LOHI, form a preference by checking which forms of plain
4124 // MULH it supports.
4125 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT);
4126 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT);
4127 bool HasMULHS = TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT);
4128 bool HasMULHU = TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT);
4129 unsigned OpToUse = 0;
4130 if (HasSMUL_LOHI && !HasMULHS) {
4131 OpToUse = ISD::SMUL_LOHI;
4132 } else if (HasUMUL_LOHI && !HasMULHU) {
4133 OpToUse = ISD::UMUL_LOHI;
4134 } else if (HasSMUL_LOHI) {
4135 OpToUse = ISD::SMUL_LOHI;
4136 } else if (HasUMUL_LOHI) {
4137 OpToUse = ISD::UMUL_LOHI;
4138 }
4139 if (OpToUse) {
4140 Results.push_back(Elt: DAG.getNode(Opcode: OpToUse, DL: dl, VTList: VTs, N1: Node->getOperand(Num: 0),
4141 N2: Node->getOperand(Num: 1)));
4142 break;
4143 }
4144
4145 SDValue Lo, Hi;
4146 EVT HalfType = VT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
4147 if (TLI.isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT) &&
4148 TLI.isOperationLegalOrCustom(Op: ISD::ANY_EXTEND, VT) &&
4149 TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT) &&
4150 TLI.isOperationLegalOrCustom(Op: ISD::OR, VT) &&
4151 TLI.expandMUL(N: Node, Lo, Hi, HiLoVT: HalfType, DAG,
4152 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4153 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
4154 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Hi);
4155 SDValue Shift =
4156 DAG.getShiftAmountConstant(Val: HalfType.getSizeInBits(), VT, DL: dl);
4157 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
4158 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi));
4159 }
4160 break;
4161 }
4162 case ISD::FSHL:
4163 case ISD::FSHR:
4164 if (SDValue Expanded = TLI.expandFunnelShift(N: Node, DAG))
4165 Results.push_back(Elt: Expanded);
4166 break;
4167 case ISD::ROTL:
4168 case ISD::ROTR:
4169 if (SDValue Expanded = TLI.expandROT(N: Node, AllowVectorOps: true /*AllowVectorOps*/, DAG))
4170 Results.push_back(Elt: Expanded);
4171 break;
4172 case ISD::CLMUL:
4173 case ISD::CLMULR:
4174 case ISD::CLMULH:
4175 if (SDValue Expanded = TLI.expandCLMUL(N: Node, DAG))
4176 Results.push_back(Elt: Expanded);
4177 break;
4178 case ISD::PEXT:
4179 Results.push_back(Elt: TLI.expandPEXT(N: Node, DAG));
4180 break;
4181 case ISD::PDEP:
4182 Results.push_back(Elt: TLI.expandPDEP(N: Node, DAG));
4183 break;
4184 case ISD::SADDSAT:
4185 case ISD::UADDSAT:
4186 case ISD::SSUBSAT:
4187 case ISD::USUBSAT:
4188 Results.push_back(Elt: TLI.expandAddSubSat(Node, DAG));
4189 break;
4190 case ISD::SCMP:
4191 case ISD::UCMP:
4192 Results.push_back(Elt: TLI.expandCMP(Node, DAG));
4193 break;
4194 case ISD::SSHLSAT:
4195 case ISD::USHLSAT:
4196 Results.push_back(Elt: TLI.expandShlSat(Node, DAG));
4197 break;
4198 case ISD::SMULFIX:
4199 case ISD::SMULFIXSAT:
4200 case ISD::UMULFIX:
4201 case ISD::UMULFIXSAT:
4202 Results.push_back(Elt: TLI.expandFixedPointMul(Node, DAG));
4203 break;
4204 case ISD::SDIVFIX:
4205 case ISD::SDIVFIXSAT:
4206 case ISD::UDIVFIX:
4207 case ISD::UDIVFIXSAT:
4208 if (SDValue V = TLI.expandFixedPointDiv(Opcode: Node->getOpcode(), dl: SDLoc(Node),
4209 LHS: Node->getOperand(Num: 0),
4210 RHS: Node->getOperand(Num: 1),
4211 Scale: Node->getConstantOperandVal(Num: 2),
4212 DAG)) {
4213 Results.push_back(Elt: V);
4214 break;
4215 }
4216 // FIXME: We might want to retry here with a wider type if we fail, if that
4217 // type is legal.
4218 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
4219 // <= 128 (which is the case for all of the default Embedded-C types),
4220 // we will only get here with types and scales that we could always expand
4221 // if we were allowed to generate libcalls to division functions of illegal
4222 // type. But we cannot do that.
4223 llvm_unreachable("Cannot expand DIVFIX!");
4224 case ISD::UADDO_CARRY:
4225 case ISD::USUBO_CARRY: {
4226 SDValue LHS = Node->getOperand(Num: 0);
4227 SDValue RHS = Node->getOperand(Num: 1);
4228 SDValue Carry = Node->getOperand(Num: 2);
4229
4230 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
4231
4232 // Initial add of the 2 operands.
4233 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
4234 EVT VT = LHS.getValueType();
4235 SDValue Sum = DAG.getNode(Opcode: Op, DL: dl, VT, N1: LHS, N2: RHS);
4236
4237 // Initial check for overflow.
4238 EVT CarryType = Node->getValueType(ResNo: 1);
4239 EVT SetCCType = getSetCCResultType(VT: Node->getValueType(ResNo: 0));
4240 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
4241 SDValue Overflow = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LHS, Cond: CC);
4242
4243 // Add of the sum and the carry.
4244 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
4245 SDValue CarryExt =
4246 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT), N2: One);
4247 SDValue Sum2 = DAG.getNode(Opcode: Op, DL: dl, VT, N1: Sum, N2: CarryExt);
4248
4249 // Second check for overflow. If we are adding, we can only overflow if the
4250 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
4251 // If we are subtracting, we can only overflow if the initial sum is 0 and
4252 // the carry is set, resulting in a new sum of all 1s.
4253 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
4254 SDValue Overflow2 =
4255 IsAdd ? DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum2, RHS: Zero, Cond: ISD::SETEQ)
4256 : DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: Zero, Cond: ISD::SETEQ);
4257 Overflow2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCType, N1: Overflow2,
4258 N2: DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: SetCCType));
4259
4260 SDValue ResultCarry =
4261 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCType, N1: Overflow, N2: Overflow2);
4262
4263 Results.push_back(Elt: Sum2);
4264 Results.push_back(Elt: DAG.getBoolExtOrTrunc(Op: ResultCarry, SL: dl, VT: CarryType, OpVT: VT));
4265 break;
4266 }
4267 case ISD::SADDO:
4268 case ISD::SSUBO: {
4269 SDValue Result, Overflow;
4270 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
4271 Results.push_back(Elt: Result);
4272 Results.push_back(Elt: Overflow);
4273 break;
4274 }
4275 case ISD::UADDO:
4276 case ISD::USUBO: {
4277 SDValue Result, Overflow;
4278 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
4279 Results.push_back(Elt: Result);
4280 Results.push_back(Elt: Overflow);
4281 break;
4282 }
4283 case ISD::UMULO:
4284 case ISD::SMULO: {
4285 SDValue Result, Overflow;
4286 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4287 Results.push_back(Elt: Result);
4288 Results.push_back(Elt: Overflow);
4289 }
4290 break;
4291 }
4292 case ISD::BUILD_PAIR: {
4293 EVT PairTy = Node->getValueType(ResNo: 0);
4294 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 0));
4295 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PairTy, Operand: Node->getOperand(Num: 1));
4296 Tmp2 = DAG.getNode(
4297 Opcode: ISD::SHL, DL: dl, VT: PairTy, N1: Tmp2,
4298 N2: DAG.getShiftAmountConstant(Val: PairTy.getSizeInBits() / 2, VT: PairTy, DL: dl));
4299 Results.push_back(Elt: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PairTy, N1: Tmp1, N2: Tmp2));
4300 break;
4301 }
4302 case ISD::SELECT:
4303 Tmp1 = Node->getOperand(Num: 0);
4304 Tmp2 = Node->getOperand(Num: 1);
4305 Tmp3 = Node->getOperand(Num: 2);
4306 if (Tmp1.getOpcode() == ISD::SETCC) {
4307 Tmp1 =
4308 DAG.getSelectCC(DL: dl, LHS: Tmp1.getOperand(i: 0), RHS: Tmp1.getOperand(i: 1), True: Tmp2,
4309 False: Tmp3, Cond: Tmp1.getOperandCondCode(i: 2), Flags: Node->getFlags());
4310 } else {
4311 Tmp1 =
4312 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType()),
4313 True: Tmp2, False: Tmp3, Cond: ISD::SETNE, Flags: Node->getFlags());
4314 }
4315 Results.push_back(Elt: Tmp1);
4316 break;
4317 case ISD::BR_JT: {
4318 SDValue Chain = Node->getOperand(Num: 0);
4319 SDValue Table = Node->getOperand(Num: 1);
4320 SDValue Index = Node->getOperand(Num: 2);
4321 int JTI = cast<JumpTableSDNode>(Val: Table.getNode())->getIndex();
4322
4323 const DataLayout &TD = DAG.getDataLayout();
4324 EVT PTy = TLI.getPointerTy(DL: TD);
4325
4326 unsigned EntrySize =
4327 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4328
4329 // For power-of-two jumptable entry sizes convert multiplication to a shift.
4330 // This transformation needs to be done here since otherwise the MIPS
4331 // backend will end up emitting a three instruction multiply sequence
4332 // instead of a single shift and MSP430 will call a runtime function.
4333 if (llvm::isPowerOf2_32(Value: EntrySize))
4334 Index = DAG.getNode(
4335 Opcode: ISD::SHL, DL: dl, VT: Index.getValueType(), N1: Index,
4336 N2: DAG.getConstant(Val: llvm::Log2_32(Value: EntrySize), DL: dl, VT: Index.getValueType()));
4337 else
4338 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: Index.getValueType(), N1: Index,
4339 N2: DAG.getConstant(Val: EntrySize, DL: dl, VT: Index.getValueType()));
4340 SDValue Addr = DAG.getMemBasePlusOffset(Base: Table, Offset: Index, DL: dl);
4341
4342 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EntrySize * 8);
4343 SDValue LD = DAG.getExtLoad(
4344 ExtType: ISD::SEXTLOAD, dl, VT: PTy, Chain, Ptr: Addr,
4345 PtrInfo: MachinePointerInfo::getJumpTable(MF&: DAG.getMachineFunction()), MemVT);
4346 Addr = LD;
4347 if (TLI.isJumpTableRelative()) {
4348 // For PIC, the sequence is:
4349 // BRIND(RelocBase + load(Jumptable + index))
4350 // RelocBase can be JumpTable, GOT or some sort of global base.
4351 Addr = DAG.getMemBasePlusOffset(Base: TLI.getPICJumpTableRelocBase(Table, DAG),
4352 Offset: Addr, DL: dl);
4353 }
4354
4355 Tmp1 = TLI.expandIndirectJTBranch(dl, Value: LD.getValue(R: 1), Addr, JTI, DAG);
4356 Results.push_back(Elt: Tmp1);
4357 break;
4358 }
4359 case ISD::BRCOND:
4360 // Expand brcond's setcc into its constituent parts and create a BR_CC
4361 // Node.
4362 Tmp1 = Node->getOperand(Num: 0);
4363 Tmp2 = Node->getOperand(Num: 1);
4364 if (Tmp2.getOpcode() == ISD::SETCC &&
4365 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
4366 VT: Tmp2.getOperand(i: 0).getValueType())) {
4367 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1, N2: Tmp2.getOperand(i: 2),
4368 N3: Tmp2.getOperand(i: 0), N4: Tmp2.getOperand(i: 1),
4369 N5: Node->getOperand(Num: 2));
4370 } else {
4371 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4372 if (Tmp2.isUndef() ||
4373 (Tmp2.getOpcode() == ISD::AND && isOneConstant(V: Tmp2.getOperand(i: 1))))
4374 Tmp3 = Tmp2;
4375 else
4376 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: Tmp2.getValueType(), N1: Tmp2,
4377 N2: DAG.getConstant(Val: 1, DL: dl, VT: Tmp2.getValueType()));
4378 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: MVT::Other, N1: Tmp1,
4379 N2: DAG.getCondCode(Cond: ISD::SETNE), N3: Tmp3,
4380 N4: DAG.getConstant(Val: 0, DL: dl, VT: Tmp3.getValueType()),
4381 N5: Node->getOperand(Num: 2));
4382 }
4383 Results.push_back(Elt: Tmp1);
4384 break;
4385 case ISD::SETCC:
4386 case ISD::STRICT_FSETCC:
4387 case ISD::STRICT_FSETCCS: {
4388 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4389 Node->getOpcode() == ISD::STRICT_FSETCCS;
4390 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4391 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
4392 unsigned Offset = IsStrict ? 1 : 0;
4393 Tmp1 = Node->getOperand(Num: 0 + Offset);
4394 Tmp2 = Node->getOperand(Num: 1 + Offset);
4395 Tmp3 = Node->getOperand(Num: 2 + Offset);
4396 bool Legalized =
4397 TLI.LegalizeSetCCCondCode(DAG, VT: Node->getValueType(ResNo: 0), LHS&: Tmp1, RHS&: Tmp2, CC&: Tmp3,
4398 NeedInvert, dl, Chain, IsSignaling);
4399
4400 if (Legalized) {
4401 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4402 // condition code, create a new SETCC node.
4403 if (Tmp3.getNode()) {
4404 if (IsStrict) {
4405 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: Node->getVTList(),
4406 Ops: {Chain, Tmp1, Tmp2, Tmp3}, Flags: Node->getFlags());
4407 Chain = Tmp1.getValue(R: 1);
4408 } else {
4409 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4410 N2: Tmp2, N3: Tmp3, Flags: Node->getFlags());
4411 }
4412 }
4413
4414 // If we expanded the SETCC by inverting the condition code, then wrap
4415 // the existing SETCC in a NOT to restore the intended condition.
4416 if (NeedInvert) {
4417 Tmp1 = DAG.getLogicalNOT(DL: dl, Val: Tmp1, VT: Tmp1->getValueType(ResNo: 0));
4418 }
4419
4420 Results.push_back(Elt: Tmp1);
4421 if (IsStrict)
4422 Results.push_back(Elt: Chain);
4423
4424 break;
4425 }
4426
4427 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4428 // understand if this code is useful for strict nodes.
4429 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4430
4431 // Otherwise, SETCC for the given comparison type must be completely
4432 // illegal; expand it into a SELECT_CC.
4433 EVT VT = Node->getValueType(ResNo: 0);
4434 EVT Tmp1VT = Tmp1.getValueType();
4435 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT, N1: Tmp1, N2: Tmp2,
4436 N3: DAG.getBoolConstant(V: true, DL: dl, VT, OpVT: Tmp1VT),
4437 N4: DAG.getBoolConstant(V: false, DL: dl, VT, OpVT: Tmp1VT), N5: Tmp3,
4438 Flags: Node->getFlags());
4439 Results.push_back(Elt: Tmp1);
4440 break;
4441 }
4442 case ISD::SELECT_CC: {
4443 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4444 Tmp1 = Node->getOperand(Num: 0); // LHS
4445 Tmp2 = Node->getOperand(Num: 1); // RHS
4446 Tmp3 = Node->getOperand(Num: 2); // True
4447 Tmp4 = Node->getOperand(Num: 3); // False
4448 EVT VT = Node->getValueType(ResNo: 0);
4449 SDValue Chain;
4450 SDValue CC = Node->getOperand(Num: 4);
4451 ISD::CondCode CCOp = cast<CondCodeSDNode>(Val&: CC)->get();
4452
4453 if (TLI.isCondCodeLegalOrCustom(CC: CCOp, VT: Tmp1.getSimpleValueType())) {
4454 // If the condition code is legal, then we need to expand this
4455 // node using SETCC and SELECT.
4456 EVT CmpVT = Tmp1.getValueType();
4457 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4458 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4459 "expanded.");
4460 EVT CCVT = getSetCCResultType(VT: CmpVT);
4461 SDValue Cond = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Node->getFlags());
4462 Results.push_back(
4463 Elt: DAG.getSelect(DL: dl, VT, Cond, LHS: Tmp3, RHS: Tmp4, Flags: Node->getFlags()));
4464 break;
4465 }
4466
4467 // SELECT_CC is legal, so the condition code must not be.
4468 bool Legalized = false;
4469 // Try to legalize by inverting the condition. This is for targets that
4470 // might support an ordered version of a condition, but not the unordered
4471 // version (or vice versa).
4472 ISD::CondCode InvCC = ISD::getSetCCInverse(Operation: CCOp, Type: Tmp1.getValueType());
4473 if (TLI.isCondCodeLegalOrCustom(CC: InvCC, VT: Tmp1.getSimpleValueType())) {
4474 // Use the new condition code and swap true and false
4475 Legalized = true;
4476 Tmp1 =
4477 DAG.getSelectCC(DL: dl, LHS: Tmp1, RHS: Tmp2, True: Tmp4, False: Tmp3, Cond: InvCC, Flags: Node->getFlags());
4478 } else {
4479 // If The inverse is not legal, then try to swap the arguments using
4480 // the inverse condition code.
4481 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
4482 if (TLI.isCondCodeLegalOrCustom(CC: SwapInvCC, VT: Tmp1.getSimpleValueType())) {
4483 // The swapped inverse condition is legal, so swap true and false,
4484 // lhs and rhs.
4485 Legalized = true;
4486 Tmp1 = DAG.getSelectCC(DL: dl, LHS: Tmp2, RHS: Tmp1, True: Tmp4, False: Tmp3, Cond: SwapInvCC,
4487 Flags: Node->getFlags());
4488 }
4489 }
4490
4491 if (!Legalized) {
4492 Legalized = TLI.LegalizeSetCCCondCode(
4493 DAG, VT: getSetCCResultType(VT: Tmp1.getValueType()), LHS&: Tmp1, RHS&: Tmp2, CC,
4494 NeedInvert, dl, Chain);
4495
4496 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4497
4498 // If we expanded the SETCC by inverting the condition code, then swap
4499 // the True/False operands to match.
4500 if (NeedInvert)
4501 std::swap(a&: Tmp3, b&: Tmp4);
4502
4503 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4504 // condition code, create a new SELECT_CC node.
4505 if (CC.getNode()) {
4506 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4507 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4508 } else {
4509 Tmp2 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp1.getValueType());
4510 CC = DAG.getCondCode(Cond: ISD::SETNE);
4511 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4512 N2: Tmp2, N3: Tmp3, N4: Tmp4, N5: CC, Flags: Node->getFlags());
4513 }
4514 }
4515 Results.push_back(Elt: Tmp1);
4516 break;
4517 }
4518 case ISD::BR_CC: {
4519 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4520 SDValue Chain;
4521 Tmp1 = Node->getOperand(Num: 0); // Chain
4522 Tmp2 = Node->getOperand(Num: 2); // LHS
4523 Tmp3 = Node->getOperand(Num: 3); // RHS
4524 Tmp4 = Node->getOperand(Num: 1); // CC
4525
4526 bool Legalized =
4527 TLI.LegalizeSetCCCondCode(DAG, VT: getSetCCResultType(VT: Tmp2.getValueType()),
4528 LHS&: Tmp2, RHS&: Tmp3, CC&: Tmp4, NeedInvert, dl, Chain);
4529 (void)Legalized;
4530 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4531
4532 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4533 // node.
4534 if (Tmp4.getNode()) {
4535 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4536
4537 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
4538 N2: Tmp4, N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4539 } else {
4540 Tmp3 = DAG.getConstant(Val: 0, DL: dl, VT: Tmp2.getValueType());
4541 Tmp4 = DAG.getCondCode(Cond: NeedInvert ? ISD::SETEQ : ISD::SETNE);
4542 Tmp1 = DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1, N2: Tmp4,
4543 N3: Tmp2, N4: Tmp3, N5: Node->getOperand(Num: 4));
4544 }
4545 Results.push_back(Elt: Tmp1);
4546 break;
4547 }
4548 case ISD::BUILD_VECTOR:
4549 Results.push_back(Elt: ExpandBUILD_VECTOR(Node));
4550 break;
4551 case ISD::SPLAT_VECTOR:
4552 Results.push_back(Elt: ExpandSPLAT_VECTOR(Node));
4553 break;
4554 case ISD::SRA:
4555 case ISD::SRL:
4556 case ISD::SHL: {
4557 // Scalarize vector SRA/SRL/SHL.
4558 EVT VT = Node->getValueType(ResNo: 0);
4559 assert(VT.isVector() && "Unable to legalize non-vector shift");
4560 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4561 unsigned NumElem = VT.getVectorNumElements();
4562
4563 SmallVector<SDValue, 8> Scalars;
4564 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4565 SDValue Ex =
4566 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4567 N1: Node->getOperand(Num: 0), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4568 SDValue Sh =
4569 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getScalarType(),
4570 N1: Node->getOperand(Num: 1), N2: DAG.getVectorIdxConstant(Val: Idx, DL: dl));
4571 Scalars.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl,
4572 VT: VT.getScalarType(), N1: Ex, N2: Sh));
4573 }
4574
4575 SDValue Result = DAG.getBuildVector(VT: Node->getValueType(ResNo: 0), DL: dl, Ops: Scalars);
4576 Results.push_back(Elt: Result);
4577 break;
4578 }
4579 case ISD::VECREDUCE_FADD:
4580 case ISD::VECREDUCE_FMUL:
4581 case ISD::VECREDUCE_ADD:
4582 case ISD::VECREDUCE_MUL:
4583 case ISD::VECREDUCE_AND:
4584 case ISD::VECREDUCE_OR:
4585 case ISD::VECREDUCE_XOR:
4586 case ISD::VECREDUCE_SMAX:
4587 case ISD::VECREDUCE_SMIN:
4588 case ISD::VECREDUCE_UMAX:
4589 case ISD::VECREDUCE_UMIN:
4590 case ISD::VECREDUCE_FMAX:
4591 case ISD::VECREDUCE_FMIN:
4592 case ISD::VECREDUCE_FMAXIMUM:
4593 case ISD::VECREDUCE_FMINIMUM:
4594 case ISD::VECREDUCE_FMAXIMUMNUM:
4595 case ISD::VECREDUCE_FMINIMUMNUM:
4596 Results.push_back(Elt: TLI.expandVecReduce(Node, DAG));
4597 break;
4598 case ISD::VP_CTTZ_ELTS:
4599 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
4600 Results.push_back(Elt: TLI.expandVPCTTZElements(N: Node, DAG));
4601 break;
4602 case ISD::CLEAR_CACHE:
4603 // The default expansion of llvm.clear_cache is simply a no-op for those
4604 // targets where it is not needed.
4605 Results.push_back(Elt: Node->getOperand(Num: 0));
4606 break;
4607 case ISD::LRINT:
4608 case ISD::LLRINT: {
4609 SDValue Arg = Node->getOperand(Num: 0);
4610 EVT ArgVT = Arg.getValueType();
4611 EVT ResVT = Node->getValueType(ResNo: 0);
4612 SDLoc DL(Node);
4613 SDValue RoundNode = DAG.getNode(Opcode: ISD::FRINT, DL, VT: ArgVT, Operand: Arg);
4614 SDValue ConvertNode = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL, VT: ResVT, Operand: RoundNode);
4615 // Non-deterministic results are equivalent to freeze poison.
4616 Results.push_back(Elt: DAG.getFreeze(V: ConvertNode));
4617 break;
4618 }
4619 case ISD::ADDRSPACECAST:
4620 Results.push_back(Elt: DAG.UnrollVectorOp(N: Node));
4621 break;
4622 case ISD::GLOBAL_OFFSET_TABLE:
4623 case ISD::GlobalAddress:
4624 case ISD::GlobalTLSAddress:
4625 case ISD::ExternalSymbol:
4626 case ISD::ConstantPool:
4627 case ISD::JumpTable:
4628 case ISD::INTRINSIC_W_CHAIN:
4629 case ISD::INTRINSIC_WO_CHAIN:
4630 case ISD::INTRINSIC_VOID:
4631 // FIXME: Custom lowering for these operations shouldn't return null!
4632 // Return true so that we don't call ConvertNodeToLibcall which also won't
4633 // do anything.
4634 return true;
4635 }
4636
4637 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4638 // FIXME: We were asked to expand a strict floating-point operation,
4639 // but there is currently no expansion implemented that would preserve
4640 // the "strict" properties. For now, we just fall back to the non-strict
4641 // version if that is legal on the target. The actual mutation of the
4642 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4643 switch (Node->getOpcode()) {
4644 default:
4645 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4646 VT: Node->getValueType(ResNo: 0))
4647 == TargetLowering::Legal)
4648 return true;
4649 break;
4650 case ISD::STRICT_FSUB: {
4651 if (TLI.getStrictFPOperationAction(
4652 Op: ISD::STRICT_FSUB, VT: Node->getValueType(ResNo: 0)) == TargetLowering::Legal)
4653 return true;
4654 if (TLI.getStrictFPOperationAction(
4655 Op: ISD::STRICT_FADD, VT: Node->getValueType(ResNo: 0)) != TargetLowering::Legal)
4656 break;
4657
4658 EVT VT = Node->getValueType(ResNo: 0);
4659 const SDNodeFlags Flags = Node->getFlags();
4660 SDValue Neg = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Node->getOperand(Num: 2), Flags);
4661 SDValue Fadd = DAG.getNode(Opcode: ISD::STRICT_FADD, DL: dl, VTList: Node->getVTList(),
4662 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1), Neg},
4663 Flags);
4664
4665 Results.push_back(Elt: Fadd);
4666 Results.push_back(Elt: Fadd.getValue(R: 1));
4667 break;
4668 }
4669 case ISD::STRICT_SINT_TO_FP:
4670 case ISD::STRICT_UINT_TO_FP:
4671 case ISD::STRICT_LRINT:
4672 case ISD::STRICT_LLRINT:
4673 case ISD::STRICT_LROUND:
4674 case ISD::STRICT_LLROUND:
4675 // These are registered by the operand type instead of the value
4676 // type. Reflect that here.
4677 if (TLI.getStrictFPOperationAction(Op: Node->getOpcode(),
4678 VT: Node->getOperand(Num: 1).getValueType())
4679 == TargetLowering::Legal)
4680 return true;
4681 break;
4682 }
4683 }
4684
4685 // Replace the original node with the legalized result.
4686 if (Results.empty()) {
4687 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4688 return false;
4689 }
4690
4691 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4692 ReplaceNode(Old: Node, New: Results.data());
4693 return true;
4694}
4695
4696/// Return if we can use the FAST_* variant of a math libcall for the node.
4697/// FIXME: This is just guessing, we probably should have unique specific sets
4698/// flags required per libcall.
4699static bool canUseFastMathLibcall(const SDNode *Node) {
4700 // FIXME: Probably should define fast to respect nan/inf and only be
4701 // approximate functions.
4702
4703 SDNodeFlags Flags = Node->getFlags();
4704 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4705 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4706}
4707
4708void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4709 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4710 SmallVector<SDValue, 8> Results;
4711 SDLoc dl(Node);
4712 TargetLowering::MakeLibCallOptions CallOptions;
4713 CallOptions.IsPostTypeLegalization = true;
4714 // FIXME: Check flags on the node to see if we can use a finite call.
4715 unsigned Opc = Node->getOpcode();
4716 switch (Opc) {
4717 case ISD::ATOMIC_FENCE: {
4718 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4719 // FIXME: handle "fence singlethread" more efficiently.
4720 TargetLowering::ArgListTy Args;
4721
4722 TargetLowering::CallLoweringInfo CLI(DAG);
4723 CLI.setDebugLoc(dl)
4724 .setChain(Node->getOperand(Num: 0))
4725 .setLibCallee(
4726 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4727 Target: DAG.getExternalSymbol(Sym: "__sync_synchronize",
4728 VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4729 ArgsList: std::move(Args));
4730
4731 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4732
4733 Results.push_back(Elt: CallResult.second);
4734 break;
4735 }
4736 // By default, atomic intrinsics are marked Legal and lowered. Targets
4737 // which don't support them directly, however, may want libcalls, in which
4738 // case they mark them Expand, and we get here.
4739 case ISD::ATOMIC_SWAP:
4740 case ISD::ATOMIC_LOAD_ADD:
4741 case ISD::ATOMIC_LOAD_SUB:
4742 case ISD::ATOMIC_LOAD_AND:
4743 case ISD::ATOMIC_LOAD_CLR:
4744 case ISD::ATOMIC_LOAD_OR:
4745 case ISD::ATOMIC_LOAD_XOR:
4746 case ISD::ATOMIC_LOAD_NAND:
4747 case ISD::ATOMIC_LOAD_MIN:
4748 case ISD::ATOMIC_LOAD_MAX:
4749 case ISD::ATOMIC_LOAD_UMIN:
4750 case ISD::ATOMIC_LOAD_UMAX:
4751 case ISD::ATOMIC_CMP_SWAP: {
4752 MVT VT = cast<AtomicSDNode>(Val: Node)->getMemoryVT().getSimpleVT();
4753 AtomicOrdering Order = cast<AtomicSDNode>(Val: Node)->getMergedOrdering();
4754 RTLIB::AtomicLibcall LC =
4755 RTLIB::selectAtomicLibcall(Libcalls: DAG.getLibcalls(), Opc, Order, VT);
4756 EVT RetVT = Node->getValueType(ResNo: 0);
4757 SmallVector<SDValue, 4> Ops;
4758 if (LC.IsOutline) {
4759 // If outline atomic available, prepare its arguments and expand.
4760 Ops.append(in_start: Node->op_begin() + 2, in_end: Node->op_end());
4761 Ops.push_back(Elt: Node->getOperand(Num: 1));
4762 } else {
4763 // Arguments for expansion to sync libcall
4764 Ops.append(in_start: Node->op_begin() + 1, in_end: Node->op_end());
4765 }
4766 std::pair<SDValue, SDValue> Tmp =
4767 TLI.makeLibCall(DAG, LibcallImpl: LC.Impl, RetVT, Ops, CallOptions, dl: SDLoc(Node),
4768 Chain: Node->getOperand(Num: 0));
4769 Results.push_back(Elt: Tmp.first);
4770 Results.push_back(Elt: Tmp.second);
4771 break;
4772 }
4773 case ISD::TRAP: {
4774 // If this operation is not supported, lower it to 'abort()' call
4775 TargetLowering::ArgListTy Args;
4776 TargetLowering::CallLoweringInfo CLI(DAG);
4777 CLI.setDebugLoc(dl)
4778 .setChain(Node->getOperand(Num: 0))
4779 .setLibCallee(CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
4780 Target: DAG.getExternalSymbol(
4781 Sym: "abort", VT: TLI.getPointerTy(DL: DAG.getDataLayout())),
4782 ArgsList: std::move(Args));
4783 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4784
4785 Results.push_back(Elt: CallResult.second);
4786 break;
4787 }
4788 case ISD::CLEAR_CACHE: {
4789 SDValue InputChain = Node->getOperand(Num: 0);
4790 SDValue StartVal = Node->getOperand(Num: 1);
4791 SDValue EndVal = Node->getOperand(Num: 2);
4792 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4793 DAG, LC: RTLIB::CLEAR_CACHE, RetVT: MVT::isVoid, Ops: {StartVal, EndVal}, CallOptions,
4794 dl: SDLoc(Node), Chain: InputChain);
4795 Results.push_back(Elt: Tmp.second);
4796 break;
4797 }
4798 case ISD::FMINNUM:
4799 case ISD::STRICT_FMINNUM:
4800 ExpandFPLibCall(Node, LC: RTLIB::getFMIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4801 break;
4802 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4803 // libcall legalization for these nodes, but there is no default expasion for
4804 // these nodes either (see PR63267 for example).
4805 case ISD::FMAXNUM:
4806 case ISD::STRICT_FMAXNUM:
4807 ExpandFPLibCall(Node, LC: RTLIB::getFMAX(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4808 break;
4809 case ISD::FMINIMUMNUM:
4810 ExpandFPLibCall(Node, LC: RTLIB::getFMINIMUM_NUM(VT: Node->getSimpleValueType(ResNo: 0)),
4811 Results);
4812 break;
4813 case ISD::FMAXIMUMNUM:
4814 ExpandFPLibCall(Node, LC: RTLIB::getFMAXIMUM_NUM(VT: Node->getSimpleValueType(ResNo: 0)),
4815 Results);
4816 break;
4817 case ISD::FSQRT:
4818 case ISD::STRICT_FSQRT: {
4819 // FIXME: Probably should define fast to respect nan/inf and only be
4820 // approximate functions.
4821 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
4822 Call_F32: {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
4823 Call_F64: {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
4824 Call_F80: {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
4825 Call_F128: {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
4826 Call_PPCF128: {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
4827 Results);
4828 break;
4829 }
4830 case ISD::FCBRT:
4831 ExpandFPLibCall(Node, LC: RTLIB::getCBRT(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4832 break;
4833 case ISD::FSIN:
4834 case ISD::STRICT_FSIN:
4835 ExpandFPLibCall(Node, LC: RTLIB::getSIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4836 break;
4837 case ISD::FCOS:
4838 case ISD::STRICT_FCOS:
4839 ExpandFPLibCall(Node, LC: RTLIB::getCOS(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4840 break;
4841 case ISD::FTAN:
4842 case ISD::STRICT_FTAN:
4843 ExpandFPLibCall(Node, LC: RTLIB::getTAN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4844 break;
4845 case ISD::FASIN:
4846 case ISD::STRICT_FASIN:
4847 ExpandFPLibCall(Node, LC: RTLIB::getASIN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4848 break;
4849 case ISD::FACOS:
4850 case ISD::STRICT_FACOS:
4851 ExpandFPLibCall(Node, LC: RTLIB::getACOS(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4852 break;
4853 case ISD::FATAN:
4854 case ISD::STRICT_FATAN:
4855 ExpandFPLibCall(Node, LC: RTLIB::getATAN(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4856 break;
4857 case ISD::FATAN2:
4858 case ISD::STRICT_FATAN2:
4859 ExpandFPLibCall(Node, LC: RTLIB::getATAN2(VT: Node->getSimpleValueType(ResNo: 0)),
4860 Results);
4861 break;
4862 case ISD::FSINH:
4863 case ISD::STRICT_FSINH:
4864 ExpandFPLibCall(Node, LC: RTLIB::getSINH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4865 break;
4866 case ISD::FCOSH:
4867 case ISD::STRICT_FCOSH:
4868 ExpandFPLibCall(Node, LC: RTLIB::getCOSH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4869 break;
4870 case ISD::FTANH:
4871 case ISD::STRICT_FTANH:
4872 ExpandFPLibCall(Node, LC: RTLIB::getTANH(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4873 break;
4874 case ISD::FSINCOS:
4875 case ISD::FSINCOSPI: {
4876 EVT VT = Node->getValueType(ResNo: 0);
4877
4878 if (Node->getOpcode() == ISD::FSINCOS) {
4879 RTLIB::Libcall SincosStret = RTLIB::getSINCOS_STRET(VT);
4880 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
4881 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
4882 Results.push_back(Elt: Expanded);
4883 Results.push_back(Elt: Expanded.getValue(R: 1));
4884 break;
4885 }
4886 }
4887 }
4888
4889 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4890 ? RTLIB::getSINCOS(VT)
4891 : RTLIB::getSINCOSPI(VT);
4892 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results);
4893 if (!Expanded) {
4894 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
4895 Node->getOperationName(G: &DAG));
4896 SDValue Poison = DAG.getPOISON(VT);
4897 Results.push_back(Elt: Poison);
4898 Results.push_back(Elt: Poison);
4899 }
4900
4901 break;
4902 }
4903 case ISD::FLOG:
4904 case ISD::STRICT_FLOG:
4905 ExpandFPLibCall(Node, LC: RTLIB::getLOG(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4906 break;
4907 case ISD::FLOG2:
4908 case ISD::STRICT_FLOG2:
4909 ExpandFPLibCall(Node, LC: RTLIB::getLOG2(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4910 break;
4911 case ISD::FLOG10:
4912 case ISD::STRICT_FLOG10:
4913 ExpandFPLibCall(Node, LC: RTLIB::getLOG10(VT: Node->getSimpleValueType(ResNo: 0)),
4914 Results);
4915 break;
4916 case ISD::FEXP:
4917 case ISD::STRICT_FEXP:
4918 ExpandFPLibCall(Node, LC: RTLIB::getEXP(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4919 break;
4920 case ISD::FEXP2:
4921 case ISD::STRICT_FEXP2:
4922 ExpandFPLibCall(Node, LC: RTLIB::getEXP2(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4923 break;
4924 case ISD::FEXP10:
4925 ExpandFPLibCall(Node, LC: RTLIB::getEXP10(VT: Node->getSimpleValueType(ResNo: 0)),
4926 Results);
4927 break;
4928 case ISD::FTRUNC:
4929 case ISD::STRICT_FTRUNC:
4930 ExpandFPLibCall(Node, LC: RTLIB::getTRUNC(VT: Node->getSimpleValueType(ResNo: 0)),
4931 Results);
4932 break;
4933 case ISD::FFLOOR:
4934 case ISD::STRICT_FFLOOR:
4935 ExpandFPLibCall(Node, LC: RTLIB::getFLOOR(VT: Node->getSimpleValueType(ResNo: 0)),
4936 Results);
4937 break;
4938 case ISD::FCEIL:
4939 case ISD::STRICT_FCEIL:
4940 ExpandFPLibCall(Node, LC: RTLIB::getCEIL(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4941 break;
4942 case ISD::FRINT:
4943 case ISD::STRICT_FRINT:
4944 ExpandFPLibCall(Node, LC: RTLIB::getRINT(VT: Node->getSimpleValueType(ResNo: 0)), Results);
4945 break;
4946 case ISD::FNEARBYINT:
4947 case ISD::STRICT_FNEARBYINT:
4948 ExpandFPLibCall(Node, LC: RTLIB::getNEARBYINT(VT: Node->getSimpleValueType(ResNo: 0)),
4949 Results);
4950 break;
4951 case ISD::FROUND:
4952 case ISD::STRICT_FROUND:
4953 ExpandFPLibCall(Node, LC: RTLIB::getROUND(VT: Node->getSimpleValueType(ResNo: 0)),
4954 Results);
4955 break;
4956 case ISD::FROUNDEVEN:
4957 case ISD::STRICT_FROUNDEVEN:
4958 ExpandFPLibCall(Node, LC: RTLIB::getROUNDEVEN(VT: Node->getSimpleValueType(ResNo: 0)),
4959 Results);
4960 break;
4961 case ISD::FLDEXP:
4962 case ISD::STRICT_FLDEXP:
4963 ExpandFPLibCall(Node, LC: RTLIB::getLDEXP(VT: Node->getSimpleValueType(ResNo: 0)),
4964 Results);
4965 break;
4966 case ISD::FMODF:
4967 case ISD::FFREXP: {
4968 EVT VT = Node->getValueType(ResNo: 0);
4969 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
4970 : RTLIB::getFREXP(VT);
4971 bool Expanded = TLI.expandMultipleResultFPLibCall(DAG, LC, Node, Results,
4972 /*CallRetResNo=*/0);
4973 if (!Expanded) {
4974 DAG.getContext()->emitError(ErrorStr: Twine("no libcall available for ") +
4975 Node->getOperationName(G: &DAG));
4976 for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
4977 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: I)));
4978 }
4979 break;
4980 }
4981 case ISD::FPOWI:
4982 case ISD::STRICT_FPOWI: {
4983 RTLIB::Libcall LC = RTLIB::getPOWI(VT: Node->getSimpleValueType(ResNo: 0));
4984 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4985 if (DAG.getLibcalls().getLibcallImpl(Call: LC) == RTLIB::Unsupported) {
4986 // Some targets don't have a powi libcall; use pow instead.
4987 if (Node->isStrictFPOpcode()) {
4988 SDValue Exponent =
4989 DAG.getNode(Opcode: ISD::STRICT_SINT_TO_FP, DL: SDLoc(Node),
4990 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4991 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
4992 SDValue FPOW =
4993 DAG.getNode(Opcode: ISD::STRICT_FPOW, DL: SDLoc(Node),
4994 ResultTys: {Node->getValueType(ResNo: 0), Node->getValueType(ResNo: 1)},
4995 Ops: {Exponent.getValue(R: 1), Node->getOperand(Num: 1), Exponent});
4996 Results.push_back(Elt: FPOW);
4997 Results.push_back(Elt: FPOW.getValue(R: 1));
4998 } else {
4999 SDValue Exponent =
5000 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: Node->getValueType(ResNo: 0),
5001 Operand: Node->getOperand(Num: 1));
5002 Results.push_back(Elt: DAG.getNode(Opcode: ISD::FPOW, DL: SDLoc(Node),
5003 VT: Node->getValueType(ResNo: 0),
5004 N1: Node->getOperand(Num: 0), N2: Exponent));
5005 }
5006 break;
5007 }
5008 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
5009 bool ExponentHasSizeOfInt =
5010 DAG.getLibInfo().getIntSize() ==
5011 Node->getOperand(Num: 1 + Offset).getValueType().getSizeInBits();
5012 if (!ExponentHasSizeOfInt) {
5013 // If the exponent does not match with sizeof(int) a libcall to
5014 // RTLIB::POWI would use the wrong type for the argument.
5015 DAG.getContext()->emitError(ErrorStr: "POWI exponent does not match sizeof(int)");
5016 Results.push_back(Elt: DAG.getPOISON(VT: Node->getValueType(ResNo: 0)));
5017 break;
5018 }
5019 ExpandFPLibCall(Node, LC, Results);
5020 break;
5021 }
5022 case ISD::FPOW:
5023 case ISD::STRICT_FPOW:
5024 ExpandFPLibCall(Node, LC: RTLIB::getPOW(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5025 break;
5026 case ISD::LROUND:
5027 case ISD::STRICT_LROUND:
5028 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LROUND_F32,
5029 Call_F64: RTLIB::LROUND_F64, Call_F80: RTLIB::LROUND_F80,
5030 Call_F128: RTLIB::LROUND_F128,
5031 Call_PPCF128: RTLIB::LROUND_PPCF128, Results);
5032 break;
5033 case ISD::LLROUND:
5034 case ISD::STRICT_LLROUND:
5035 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLROUND_F32,
5036 Call_F64: RTLIB::LLROUND_F64, Call_F80: RTLIB::LLROUND_F80,
5037 Call_F128: RTLIB::LLROUND_F128,
5038 Call_PPCF128: RTLIB::LLROUND_PPCF128, Results);
5039 break;
5040 case ISD::LRINT:
5041 case ISD::STRICT_LRINT:
5042 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LRINT_F32,
5043 Call_F64: RTLIB::LRINT_F64, Call_F80: RTLIB::LRINT_F80,
5044 Call_F128: RTLIB::LRINT_F128,
5045 Call_PPCF128: RTLIB::LRINT_PPCF128, Results);
5046 break;
5047 case ISD::LLRINT:
5048 case ISD::STRICT_LLRINT:
5049 ExpandArgFPLibCall(Node, Call_F32: RTLIB::LLRINT_F32,
5050 Call_F64: RTLIB::LLRINT_F64, Call_F80: RTLIB::LLRINT_F80,
5051 Call_F128: RTLIB::LLRINT_F128,
5052 Call_PPCF128: RTLIB::LLRINT_PPCF128, Results);
5053 break;
5054 case ISD::FDIV:
5055 case ISD::STRICT_FDIV: {
5056 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5057 Call_F32: {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5058 Call_F64: {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5059 Call_F80: {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5060 Call_F128: {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5061 Call_PPCF128: {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5062 break;
5063 }
5064 case ISD::FREM:
5065 case ISD::STRICT_FREM:
5066 ExpandFPLibCall(Node, LC: RTLIB::getREM(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5067 break;
5068 case ISD::FMA:
5069 case ISD::STRICT_FMA:
5070 ExpandFPLibCall(Node, LC: RTLIB::getFMA(VT: Node->getSimpleValueType(ResNo: 0)), Results);
5071 break;
5072 case ISD::FADD:
5073 case ISD::STRICT_FADD: {
5074 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5075 Call_F32: {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5076 Call_F64: {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5077 Call_F80: {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5078 Call_F128: {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5079 Call_PPCF128: {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5080 break;
5081 }
5082 case ISD::FMUL:
5083 case ISD::STRICT_FMUL: {
5084 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5085 Call_F32: {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5086 Call_F64: {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5087 Call_F80: {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5088 Call_F128: {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5089 Call_PPCF128: {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5090 break;
5091 }
5092 case ISD::FP16_TO_FP:
5093 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5094 Results.push_back(Elt: ExpandLibCall(LC: RTLIB::FPEXT_F16_F32, Node, isSigned: false).first);
5095 }
5096 break;
5097 case ISD::STRICT_BF16_TO_FP:
5098 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5099 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5100 DAG, LC: RTLIB::FPEXT_BF16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1),
5101 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5102 Results.push_back(Elt: Tmp.first);
5103 Results.push_back(Elt: Tmp.second);
5104 }
5105 break;
5106 case ISD::STRICT_FP16_TO_FP: {
5107 if (Node->getValueType(ResNo: 0) == MVT::f32) {
5108 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5109 DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Node->getOperand(Num: 1), CallOptions,
5110 dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5111 Results.push_back(Elt: Tmp.first);
5112 Results.push_back(Elt: Tmp.second);
5113 }
5114 break;
5115 }
5116 case ISD::FP_TO_FP16: {
5117 RTLIB::Libcall LC =
5118 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::f16);
5119 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5120 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5121 break;
5122 }
5123 case ISD::FP_TO_BF16: {
5124 RTLIB::Libcall LC =
5125 RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 0).getValueType(), RetVT: MVT::bf16);
5126 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5127 Results.push_back(Elt: ExpandLibCall(LC, Node, isSigned: false).first);
5128 break;
5129 }
5130 case ISD::STRICT_SINT_TO_FP:
5131 case ISD::STRICT_UINT_TO_FP:
5132 case ISD::SINT_TO_FP:
5133 case ISD::UINT_TO_FP: {
5134 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
5135 bool IsStrict = Node->isStrictFPOpcode();
5136 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
5137 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
5138 EVT SVT = Node->getOperand(Num: IsStrict ? 1 : 0).getValueType();
5139 EVT RVT = Node->getValueType(ResNo: 0);
5140 EVT NVT = EVT();
5141 SDLoc dl(Node);
5142
5143 // Even if the input is legal, no libcall may exactly match, eg. we don't
5144 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
5145 // eg: i13 -> fp. Then, look for an appropriate libcall.
5146 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5147 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5148 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5149 ++t) {
5150 NVT = (MVT::SimpleValueType)t;
5151 // The source needs to big enough to hold the operand.
5152 if (NVT.bitsGE(VT: SVT))
5153 LC = Signed ? RTLIB::getSINTTOFP(OpVT: NVT, RetVT: RVT)
5154 : RTLIB::getUINTTOFP(OpVT: NVT, RetVT: RVT);
5155 }
5156 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5157
5158 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5159 // Sign/zero extend the argument if the libcall takes a larger type.
5160 SDValue Op = DAG.getNode(Opcode: Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL: dl,
5161 VT: NVT, Operand: Node->getOperand(Num: IsStrict ? 1 : 0));
5162 CallOptions.setIsSigned(Signed);
5163 std::pair<SDValue, SDValue> Tmp =
5164 TLI.makeLibCall(DAG, LC, RetVT: RVT, Ops: Op, CallOptions, dl, Chain);
5165 Results.push_back(Elt: Tmp.first);
5166 if (IsStrict)
5167 Results.push_back(Elt: Tmp.second);
5168 break;
5169 }
5170 case ISD::FP_TO_SINT:
5171 case ISD::FP_TO_UINT:
5172 case ISD::STRICT_FP_TO_SINT:
5173 case ISD::STRICT_FP_TO_UINT: {
5174 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
5175 bool IsStrict = Node->isStrictFPOpcode();
5176 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
5177 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
5178
5179 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5180 EVT SVT = Op.getValueType();
5181 EVT RVT = Node->getValueType(ResNo: 0);
5182 EVT NVT = EVT();
5183 SDLoc dl(Node);
5184
5185 // Even if the result is legal, no libcall may exactly match, eg. we don't
5186 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
5187 // eg: fp -> i32. Then, look for an appropriate libcall.
5188 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5189 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5190 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5191 ++IntVT) {
5192 NVT = (MVT::SimpleValueType)IntVT;
5193 // The type needs to big enough to hold the result.
5194 if (NVT.bitsGE(VT: RVT))
5195 LC = Signed ? RTLIB::getFPTOSINT(OpVT: SVT, RetVT: NVT)
5196 : RTLIB::getFPTOUINT(OpVT: SVT, RetVT: NVT);
5197 }
5198 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5199
5200 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5201 std::pair<SDValue, SDValue> Tmp =
5202 TLI.makeLibCall(DAG, LC, RetVT: NVT, Ops: Op, CallOptions, dl, Chain);
5203
5204 // Truncate the result if the libcall returns a larger type.
5205 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RVT, Operand: Tmp.first));
5206 if (IsStrict)
5207 Results.push_back(Elt: Tmp.second);
5208 break;
5209 }
5210
5211 case ISD::FP_ROUND:
5212 case ISD::STRICT_FP_ROUND: {
5213 // X = FP_ROUND(Y, TRUNC)
5214 // TRUNC is a flag, which is always an integer that is zero or one.
5215 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
5216 // is known to not change the value of Y.
5217 // We can only expand it into libcall if the TRUNC is 0.
5218 bool IsStrict = Node->isStrictFPOpcode();
5219 SDValue Op = Node->getOperand(Num: IsStrict ? 1 : 0);
5220 SDValue Chain = IsStrict ? Node->getOperand(Num: 0) : SDValue();
5221 EVT VT = Node->getValueType(ResNo: 0);
5222 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
5223 "Unable to expand as libcall if it is not normal rounding");
5224
5225 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op.getValueType(), RetVT: VT);
5226 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5227
5228 std::pair<SDValue, SDValue> Tmp =
5229 TLI.makeLibCall(DAG, LC, RetVT: VT, Ops: Op, CallOptions, dl: SDLoc(Node), Chain);
5230 Results.push_back(Elt: Tmp.first);
5231 if (IsStrict)
5232 Results.push_back(Elt: Tmp.second);
5233 break;
5234 }
5235 case ISD::FP_EXTEND: {
5236 Results.push_back(
5237 Elt: ExpandLibCall(LC: RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 0).getValueType(),
5238 RetVT: Node->getValueType(ResNo: 0)),
5239 Node, isSigned: false).first);
5240 break;
5241 }
5242 case ISD::STRICT_FP_EXTEND:
5243 case ISD::STRICT_FP_TO_FP16:
5244 case ISD::STRICT_FP_TO_BF16: {
5245 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5246 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5247 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::f16);
5248 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5249 LC = RTLIB::getFPROUND(OpVT: Node->getOperand(Num: 1).getValueType(), RetVT: MVT::bf16);
5250 else
5251 LC = RTLIB::getFPEXT(OpVT: Node->getOperand(Num: 1).getValueType(),
5252 RetVT: Node->getValueType(ResNo: 0));
5253
5254 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5255
5256 std::pair<SDValue, SDValue> Tmp =
5257 TLI.makeLibCall(DAG, LC, RetVT: Node->getValueType(ResNo: 0), Ops: Node->getOperand(Num: 1),
5258 CallOptions, dl: SDLoc(Node), Chain: Node->getOperand(Num: 0));
5259 Results.push_back(Elt: Tmp.first);
5260 Results.push_back(Elt: Tmp.second);
5261 break;
5262 }
5263 case ISD::FSUB:
5264 case ISD::STRICT_FSUB: {
5265 ExpandFastFPLibCall(Node, IsFast: canUseFastMathLibcall(Node),
5266 Call_F32: {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5267 Call_F64: {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5268 Call_F80: {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5269 Call_F128: {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5270 Call_PPCF128: {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5271 break;
5272 }
5273 case ISD::SREM:
5274 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5275 Call_I8: RTLIB::SREM_I8,
5276 Call_I16: RTLIB::SREM_I16, Call_I32: RTLIB::SREM_I32,
5277 Call_I64: RTLIB::SREM_I64, Call_I128: RTLIB::SREM_I128));
5278 break;
5279 case ISD::UREM:
5280 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5281 Call_I8: RTLIB::UREM_I8,
5282 Call_I16: RTLIB::UREM_I16, Call_I32: RTLIB::UREM_I32,
5283 Call_I64: RTLIB::UREM_I64, Call_I128: RTLIB::UREM_I128));
5284 break;
5285 case ISD::SDIV:
5286 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: true,
5287 Call_I8: RTLIB::SDIV_I8,
5288 Call_I16: RTLIB::SDIV_I16, Call_I32: RTLIB::SDIV_I32,
5289 Call_I64: RTLIB::SDIV_I64, Call_I128: RTLIB::SDIV_I128));
5290 break;
5291 case ISD::UDIV:
5292 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5293 Call_I8: RTLIB::UDIV_I8,
5294 Call_I16: RTLIB::UDIV_I16, Call_I32: RTLIB::UDIV_I32,
5295 Call_I64: RTLIB::UDIV_I64, Call_I128: RTLIB::UDIV_I128));
5296 break;
5297 case ISD::SDIVREM:
5298 case ISD::UDIVREM:
5299 // Expand into divrem libcall
5300 ExpandDivRemLibCall(Node, Results);
5301 break;
5302 case ISD::MUL:
5303 Results.push_back(Elt: ExpandIntLibCall(Node, isSigned: false,
5304 Call_I8: RTLIB::MUL_I8,
5305 Call_I16: RTLIB::MUL_I16, Call_I32: RTLIB::MUL_I32,
5306 Call_I64: RTLIB::MUL_I64, Call_I128: RTLIB::MUL_I128));
5307 break;
5308 case ISD::CTLZ_ZERO_POISON:
5309 Results.push_back(Elt: ExpandBitCountingLibCall(
5310 Node, CallI32: RTLIB::CTLZ_I32, CallI64: RTLIB::CTLZ_I64, CallI128: RTLIB::CTLZ_I128));
5311 break;
5312 case ISD::CTPOP:
5313 Results.push_back(Elt: ExpandBitCountingLibCall(
5314 Node, CallI32: RTLIB::CTPOP_I32, CallI64: RTLIB::CTPOP_I64, CallI128: RTLIB::CTPOP_I128));
5315 break;
5316 case ISD::RESET_FPENV: {
5317 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
5318 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
5319 EVT PtrTy = TLI.getPointerTy(DL: DAG.getDataLayout());
5320 SDValue Ptr = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5321 SDValue Chain = Node->getOperand(Num: 0);
5322 Results.push_back(
5323 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr, InChain: Chain, Node));
5324 break;
5325 }
5326 case ISD::GET_FPENV_MEM: {
5327 SDValue Chain = Node->getOperand(Num: 0);
5328 SDValue EnvPtr = Node->getOperand(Num: 1);
5329 Results.push_back(
5330 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETENV, Ptr: EnvPtr, InChain: Chain, Node));
5331 break;
5332 }
5333 case ISD::SET_FPENV_MEM: {
5334 SDValue Chain = Node->getOperand(Num: 0);
5335 SDValue EnvPtr = Node->getOperand(Num: 1);
5336 Results.push_back(
5337 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETENV, Ptr: EnvPtr, InChain: Chain, Node));
5338 break;
5339 }
5340 case ISD::GET_FPMODE: {
5341 // Call fegetmode, which saves control modes into a stack slot. Then load
5342 // the value to return from the stack.
5343 EVT ModeVT = Node->getValueType(ResNo: 0);
5344 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5345 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5346 SDValue Chain = DAG.makeStateFunctionCall(LibFunc: RTLIB::FEGETMODE, Ptr: StackPtr,
5347 InChain: Node->getOperand(Num: 0), Node);
5348 SDValue LdInst = DAG.getLoad(
5349 VT: ModeVT, dl, Chain, Ptr: StackPtr,
5350 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5351 Results.push_back(Elt: LdInst);
5352 Results.push_back(Elt: LdInst.getValue(R: 1));
5353 break;
5354 }
5355 case ISD::SET_FPMODE: {
5356 // Move control modes to stack slot and then call fesetmode with the pointer
5357 // to the slot as argument.
5358 SDValue Mode = Node->getOperand(Num: 1);
5359 EVT ModeVT = Mode.getValueType();
5360 SDValue StackPtr = DAG.CreateStackTemporary(VT: ModeVT);
5361 int SPFI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
5362 SDValue StInst = DAG.getStore(
5363 Chain: Node->getOperand(Num: 0), dl, Val: Mode, Ptr: StackPtr,
5364 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: SPFI));
5365 Results.push_back(
5366 Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: StackPtr, InChain: StInst, Node));
5367 break;
5368 }
5369 case ISD::RESET_FPMODE: {
5370 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
5371 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
5372 // target must provide custom lowering.
5373 const DataLayout &DL = DAG.getDataLayout();
5374 EVT PtrTy = TLI.getPointerTy(DL);
5375 SDValue Mode = DAG.getAllOnesConstant(DL: dl, VT: PtrTy);
5376 Results.push_back(Elt: DAG.makeStateFunctionCall(LibFunc: RTLIB::FESETMODE, Ptr: Mode,
5377 InChain: Node->getOperand(Num: 0), Node));
5378 break;
5379 }
5380 }
5381
5382 // Replace the original node with the legalized result.
5383 if (!Results.empty()) {
5384 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5385 ReplaceNode(Old: Node, New: Results.data());
5386 } else
5387 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5388}
5389
5390// Determine the vector type to use in place of an original scalar element when
5391// promoting equally sized vectors.
5392static MVT getPromotedVectorElementType(const TargetLowering &TLI,
5393 MVT EltVT, MVT NewEltVT) {
5394 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
5395 MVT MidVT = OldEltsPerNewElt == 1
5396 ? NewEltVT
5397 : MVT::getVectorVT(VT: NewEltVT, NumElements: OldEltsPerNewElt);
5398 assert(TLI.isTypeLegal(MidVT) && "unexpected");
5399 return MidVT;
5400}
5401
5402void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5403 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
5404 SmallVector<SDValue, 8> Results;
5405 MVT OVT = Node->getSimpleValueType(ResNo: 0);
5406 if (Node->getOpcode() == ISD::UINT_TO_FP ||
5407 Node->getOpcode() == ISD::SINT_TO_FP || Node->getOpcode() == ISD::SETCC ||
5408 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
5409 Node->getOpcode() == ISD::INSERT_VECTOR_ELT ||
5410 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5411 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5412 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5413 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM ||
5414 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUMNUM ||
5415 Node->getOpcode() == ISD::VECREDUCE_FMINIMUMNUM) {
5416 OVT = Node->getOperand(Num: 0).getSimpleValueType();
5417 }
5418 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5419 Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
5420 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
5421 Node->getOpcode() == ISD::STRICT_FSETCC ||
5422 Node->getOpcode() == ISD::STRICT_FSETCCS ||
5423 Node->getOpcode() == ISD::STRICT_LRINT ||
5424 Node->getOpcode() == ISD::STRICT_LLRINT ||
5425 Node->getOpcode() == ISD::STRICT_LROUND ||
5426 Node->getOpcode() == ISD::STRICT_LLROUND ||
5427 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5428 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5429 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5430 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5431 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5432 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5433 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5434 OVT = Node->getOperand(Num: 1).getSimpleValueType();
5435 if (Node->getOpcode() == ISD::BR_CC ||
5436 Node->getOpcode() == ISD::SELECT_CC)
5437 OVT = Node->getOperand(Num: 2).getSimpleValueType();
5438 // Preserve fast math flags
5439 SDNodeFlags FastMathFlags = Node->getFlags() & SDNodeFlags::FastMathFlags;
5440 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5441 MVT NVT = TLI.getTypeToPromoteTo(Op: Node->getOpcode(), VT: OVT);
5442 SDLoc dl(Node);
5443 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5444 switch (Node->getOpcode()) {
5445 case ISD::CTTZ:
5446 case ISD::CTTZ_ZERO_POISON:
5447 case ISD::CTLZ:
5448 case ISD::CTPOP: {
5449 // Zero extend the argument unless its cttz, then use any_extend.
5450 if (Node->getOpcode() == ISD::CTTZ ||
5451 Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
5452 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5453 else
5454 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5455
5456 unsigned NewOpc = Node->getOpcode();
5457 if (NewOpc == ISD::CTTZ) {
5458 // The count is the same in the promoted type except if the original
5459 // value was zero. This can be handled by setting the bit just off
5460 // the top of the original type.
5461 auto TopBit = APInt::getOneBitSet(numBits: NVT.getSizeInBits(),
5462 BitNo: OVT.getSizeInBits());
5463 Tmp1 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: NVT, N1: Tmp1,
5464 N2: DAG.getConstant(Val: TopBit, DL: dl, VT: NVT));
5465 NewOpc = ISD::CTTZ_ZERO_POISON;
5466 }
5467 // Perform the larger operation. For CTPOP and CTTZ_ZERO_POISON, this is
5468 // already the correct result.
5469 Tmp1 = DAG.getNode(Opcode: NewOpc, DL: dl, VT: NVT, Operand: Tmp1);
5470 if (NewOpc == ISD::CTLZ) {
5471 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5472 Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: NVT, N1: Tmp1,
5473 N2: DAG.getConstant(Val: NVT.getSizeInBits() -
5474 OVT.getSizeInBits(), DL: dl, VT: NVT));
5475 }
5476 Results.push_back(
5477 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1, Flags: SDNodeFlags::NoWrap));
5478 break;
5479 }
5480 case ISD::CTLZ_ZERO_POISON: {
5481 // We know that the argument is unlikely to be zero, hence we can take a
5482 // different approach as compared to ISD::CTLZ
5483
5484 // Any Extend the argument
5485 auto AnyExtendedNode =
5486 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5487
5488 // Tmp1 = Tmp1 << (sizeinbits(NVT) - sizeinbits(Old VT))
5489 auto ShiftConstant = DAG.getShiftAmountConstant(
5490 Val: NVT.getSizeInBits() - OVT.getSizeInBits(), VT: NVT, DL: dl);
5491 auto LeftShiftResult =
5492 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: NVT, N1: AnyExtendedNode, N2: ShiftConstant);
5493
5494 // Perform the larger operation
5495 auto CTLZResult = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: LeftShiftResult);
5496 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: CTLZResult));
5497 break;
5498 }
5499 case ISD::PEXT: {
5500 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5501 Tmp2 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5502 Tmp1 = DAG.getNode(Opcode: ISD::PEXT, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5503 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5504 break;
5505 }
5506 case ISD::PDEP: {
5507 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5508 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5509 Tmp1 = DAG.getNode(Opcode: ISD::PDEP, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5510 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5511 break;
5512 }
5513 case ISD::BITREVERSE:
5514 case ISD::BSWAP: {
5515 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5516 Tmp1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5517 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5518 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5519 N2: DAG.getShiftAmountConstant(Val: DiffBits, VT: NVT, DL: dl));
5520
5521 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5522 break;
5523 }
5524 case ISD::FP_TO_UINT:
5525 case ISD::STRICT_FP_TO_UINT:
5526 case ISD::FP_TO_SINT:
5527 case ISD::STRICT_FP_TO_SINT:
5528 PromoteLegalFP_TO_INT(N: Node, dl, Results);
5529 break;
5530 case ISD::FP_TO_UINT_SAT:
5531 case ISD::FP_TO_SINT_SAT:
5532 Results.push_back(Elt: PromoteLegalFP_TO_INT_SAT(Node, dl));
5533 break;
5534 case ISD::UINT_TO_FP:
5535 case ISD::STRICT_UINT_TO_FP:
5536 case ISD::SINT_TO_FP:
5537 case ISD::STRICT_SINT_TO_FP:
5538 PromoteLegalINT_TO_FP(N: Node, dl, Results);
5539 break;
5540 case ISD::VAARG: {
5541 SDValue Chain = Node->getOperand(Num: 0); // Get the chain.
5542 SDValue Ptr = Node->getOperand(Num: 1); // Get the pointer.
5543
5544 unsigned TruncOp;
5545 if (OVT.isVector()) {
5546 TruncOp = ISD::BITCAST;
5547 } else {
5548 assert(OVT.isInteger()
5549 && "VAARG promotion is supported only for vectors or integer types");
5550 TruncOp = ISD::TRUNCATE;
5551 }
5552
5553 // Perform the larger operation, then convert back
5554 Tmp1 = DAG.getVAArg(VT: NVT, dl, Chain, Ptr, SV: Node->getOperand(Num: 2),
5555 Align: Node->getConstantOperandVal(Num: 3));
5556 Chain = Tmp1.getValue(R: 1);
5557
5558 Tmp2 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1);
5559
5560 // Modified the chain result - switch anything that used the old chain to
5561 // use the new one.
5562 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 0), To: Tmp2);
5563 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Node, 1), To: Chain);
5564 if (UpdatedNodes) {
5565 UpdatedNodes->insert(X: Tmp2.getNode());
5566 UpdatedNodes->insert(X: Chain.getNode());
5567 }
5568 ReplacedNode(N: Node);
5569 break;
5570 }
5571 case ISD::MUL:
5572 case ISD::SDIV:
5573 case ISD::SREM:
5574 case ISD::UDIV:
5575 case ISD::UREM:
5576 case ISD::SMIN:
5577 case ISD::SMAX:
5578 case ISD::UMIN:
5579 case ISD::UMAX:
5580 case ISD::AND:
5581 case ISD::OR:
5582 case ISD::XOR: {
5583 unsigned ExtOp, TruncOp;
5584 if (OVT.isVector()) {
5585 ExtOp = ISD::BITCAST;
5586 TruncOp = ISD::BITCAST;
5587 } else {
5588 assert(OVT.isInteger() && "Cannot promote logic operation");
5589
5590 switch (Node->getOpcode()) {
5591 default:
5592 ExtOp = ISD::ANY_EXTEND;
5593 break;
5594 case ISD::SDIV:
5595 case ISD::SREM:
5596 case ISD::SMIN:
5597 case ISD::SMAX:
5598 ExtOp = ISD::SIGN_EXTEND;
5599 break;
5600 case ISD::UDIV:
5601 case ISD::UREM:
5602 ExtOp = ISD::ZERO_EXTEND;
5603 break;
5604 case ISD::UMIN:
5605 case ISD::UMAX:
5606 if (TLI.isSExtCheaperThanZExt(FromTy: OVT, ToTy: NVT))
5607 ExtOp = ISD::SIGN_EXTEND;
5608 else
5609 ExtOp = ISD::ZERO_EXTEND;
5610 break;
5611 }
5612 TruncOp = ISD::TRUNCATE;
5613 }
5614 // Promote each of the values to the new type.
5615 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5616 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5617 // Perform the larger operation, then convert back
5618 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5619 Results.push_back(Elt: DAG.getNode(Opcode: TruncOp, DL: dl, VT: OVT, Operand: Tmp1));
5620 break;
5621 }
5622 case ISD::UMUL_LOHI:
5623 case ISD::SMUL_LOHI: {
5624 // Promote to a multiply in a wider integer type.
5625 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5626 : ISD::SIGN_EXTEND;
5627 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5628 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5629 Tmp1 = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5630
5631 unsigned OriginalSize = OVT.getScalarSizeInBits();
5632 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: NVT, N1: Tmp1,
5633 N2: DAG.getShiftAmountConstant(Val: OriginalSize, VT: NVT, DL: dl));
5634 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1));
5635 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
5636 break;
5637 }
5638 case ISD::SELECT: {
5639 unsigned ExtOp, TruncOp;
5640 if (Node->getValueType(ResNo: 0).isVector() ||
5641 Node->getValueType(ResNo: 0).getSizeInBits() == NVT.getSizeInBits()) {
5642 ExtOp = ISD::BITCAST;
5643 TruncOp = ISD::BITCAST;
5644 } else if (Node->getValueType(ResNo: 0).isInteger()) {
5645 ExtOp = ISD::ANY_EXTEND;
5646 TruncOp = ISD::TRUNCATE;
5647 } else {
5648 ExtOp = ISD::FP_EXTEND;
5649 TruncOp = ISD::FP_ROUND;
5650 }
5651 Tmp1 = Node->getOperand(Num: 0);
5652 // Promote each of the values to the new type.
5653 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5654 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5655 // Perform the larger operation, then round down.
5656 Tmp1 = DAG.getSelect(DL: dl, VT: NVT, Cond: Tmp1, LHS: Tmp2, RHS: Tmp3);
5657 if (TruncOp != ISD::FP_ROUND)
5658 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
5659 else
5660 Tmp1 = DAG.getNode(Opcode: TruncOp, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5661 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5662 Results.push_back(Elt: Tmp1);
5663 break;
5664 }
5665 case ISD::VECTOR_SHUFFLE: {
5666 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Node)->getMask();
5667
5668 // Cast the two input vectors.
5669 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5670 Tmp2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5671
5672 // Convert the shuffle mask to the right # elements.
5673 Tmp1 = ShuffleWithNarrowerEltType(NVT, VT: OVT, dl, N1: Tmp1, N2: Tmp2, Mask);
5674 Tmp1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: OVT, Operand: Tmp1);
5675 Results.push_back(Elt: Tmp1);
5676 break;
5677 }
5678 case ISD::VECTOR_SPLICE_LEFT:
5679 case ISD::VECTOR_SPLICE_RIGHT: {
5680 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5681 Tmp2 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5682 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2,
5683 N3: Node->getOperand(Num: 2));
5684 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp3));
5685 break;
5686 }
5687 case ISD::SELECT_CC: {
5688 SDValue Cond = Node->getOperand(Num: 4);
5689 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: Cond)->get();
5690 // Type of the comparison operands.
5691 MVT CVT = Node->getSimpleValueType(ResNo: 0);
5692 assert(CVT == OVT && "not handled");
5693
5694 unsigned ExtOp = ISD::FP_EXTEND;
5695 if (NVT.isInteger()) {
5696 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5697 }
5698
5699 // Promote the comparison operands, if needed.
5700 if (TLI.isCondCodeLegal(CC: CCCode, VT: CVT)) {
5701 Tmp1 = Node->getOperand(Num: 0);
5702 Tmp2 = Node->getOperand(Num: 1);
5703 } else {
5704 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5705 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5706 }
5707 // Cast the true/false operands.
5708 Tmp3 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5709 Tmp4 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5710
5711 Tmp1 = DAG.getNode(Opcode: ISD::SELECT_CC, DL: dl, VT: NVT, Ops: {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5712 Flags: Node->getFlags());
5713
5714 // Cast the result back to the original type.
5715 if (ExtOp != ISD::FP_EXTEND)
5716 Tmp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp1);
5717 else
5718 Tmp1 = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp1,
5719 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5720
5721 Results.push_back(Elt: Tmp1);
5722 break;
5723 }
5724 case ISD::SETCC:
5725 case ISD::STRICT_FSETCC:
5726 case ISD::STRICT_FSETCCS: {
5727 unsigned ExtOp = ISD::FP_EXTEND;
5728 if (NVT.isInteger()) {
5729 ISD::CondCode CCCode = Node->getOperandCondCode(Num: 2);
5730 if (isSignedIntSetCC(Code: CCCode) ||
5731 TLI.isSExtCheaperThanZExt(FromTy: Node->getOperand(Num: 0).getValueType(), ToTy: NVT))
5732 ExtOp = ISD::SIGN_EXTEND;
5733 else
5734 ExtOp = ISD::ZERO_EXTEND;
5735 }
5736 if (Node->isStrictFPOpcode()) {
5737 SDValue InChain = Node->getOperand(Num: 0);
5738 std::tie(args&: Tmp1, args: std::ignore) =
5739 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 1), Chain: InChain, DL: dl, VT: NVT);
5740 std::tie(args&: Tmp2, args: std::ignore) =
5741 DAG.getStrictFPExtendOrRound(Op: Node->getOperand(Num: 2), Chain: InChain, DL: dl, VT: NVT);
5742 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(R: 1), Tmp2.getValue(R: 1)};
5743 SDValue OutChain = DAG.getTokenFactor(DL: dl, Vals&: TmpChains);
5744 SDVTList VTs = DAG.getVTList(VT1: Node->getValueType(ResNo: 0), VT2: MVT::Other);
5745 Results.push_back(Elt: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs,
5746 Ops: {OutChain, Tmp1, Tmp2, Node->getOperand(Num: 3)},
5747 Flags: Node->getFlags()));
5748 Results.push_back(Elt: Results.back().getValue(R: 1));
5749 break;
5750 }
5751 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5752 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5753 Results.push_back(Elt: DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Node->getValueType(ResNo: 0), N1: Tmp1,
5754 N2: Tmp2, N3: Node->getOperand(Num: 2), Flags: Node->getFlags()));
5755 break;
5756 }
5757 case ISD::BR_CC: {
5758 unsigned ExtOp = ISD::FP_EXTEND;
5759 if (NVT.isInteger()) {
5760 ISD::CondCode CCCode = Node->getOperandCondCode(Num: 1);
5761 ExtOp = isSignedIntSetCC(Code: CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
5762 }
5763 Tmp1 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5764 Tmp2 = DAG.getNode(Opcode: ExtOp, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 3));
5765 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Node->getValueType(ResNo: 0),
5766 N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
5767 N3: Tmp1, N4: Tmp2, N5: Node->getOperand(Num: 4)));
5768 break;
5769 }
5770 case ISD::FADD:
5771 case ISD::FSUB:
5772 case ISD::FMUL:
5773 case ISD::FDIV:
5774 case ISD::FREM:
5775 case ISD::FMINNUM:
5776 case ISD::FMAXNUM:
5777 case ISD::FMINIMUM:
5778 case ISD::FMAXIMUM:
5779 case ISD::FMINIMUMNUM:
5780 case ISD::FMAXIMUMNUM:
5781 case ISD::FPOW:
5782 case ISD::FATAN2:
5783 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5784 if (!OVT.isVector() && NVT.isVector() &&
5785 NVT.getVectorElementType() == OVT) {
5786 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5787 Tmp2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5788 Tmp3 =
5789 DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, Flags: Node->getFlags());
5790 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Tmp3,
5791 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5792 break;
5793 }
5794 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5795 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5796 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5797 Results.push_back(
5798 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5799 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5800 break;
5801
5802 case ISD::STRICT_FMINIMUM:
5803 case ISD::STRICT_FMAXIMUM: {
5804 SDValue InChain = Node->getOperand(Num: 0);
5805 SDVTList VTs = DAG.getVTList(VT1: NVT, VT2: MVT::Other);
5806 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5807 N2: Node->getOperand(Num: 1));
5808 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, VTList: VTs, N1: InChain,
5809 N2: Node->getOperand(Num: 2));
5810 SmallVector<SDValue, 4> Ops = {InChain, Tmp1, Tmp2};
5811 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: VTs, Ops, Flags: Node->getFlags());
5812 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: OVT, VT2: MVT::Other),
5813 N1: InChain, N2: Tmp3,
5814 N3: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
5815 Results.push_back(Elt: Tmp4);
5816 Results.push_back(Elt: Tmp4.getValue(R: 1));
5817 break;
5818 }
5819
5820 case ISD::STRICT_FADD:
5821 case ISD::STRICT_FSUB:
5822 case ISD::STRICT_FMUL:
5823 case ISD::STRICT_FDIV:
5824 case ISD::STRICT_FMINNUM:
5825 case ISD::STRICT_FMAXNUM:
5826 case ISD::STRICT_FREM:
5827 case ISD::STRICT_FPOW:
5828 case ISD::STRICT_FATAN2:
5829 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5830 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5831 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5832 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5833 Tmp3 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5834 N2: Tmp2.getValue(R: 1));
5835 Tmp1 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5836 Ops: {Tmp3, Tmp1, Tmp2});
5837 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5838 Ops: {Tmp1.getValue(R: 1), Tmp1,
5839 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5840 Results.push_back(Elt: Tmp1);
5841 Results.push_back(Elt: Tmp1.getValue(R: 1));
5842 break;
5843 case ISD::FMA:
5844 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5845 if (!OVT.isVector() && NVT.isVector() &&
5846 NVT.getVectorElementType() == OVT) {
5847 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5848 Tmp2 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5849 Tmp3 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5850 SDValue Result = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3,
5851 Flags: Node->getFlags());
5852 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Result,
5853 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5854 break;
5855 }
5856 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5857 Tmp2 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 1));
5858 Tmp3 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 2));
5859 Results.push_back(
5860 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT,
5861 N1: DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2, N3: Tmp3),
5862 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5863 break;
5864 case ISD::STRICT_FMA:
5865 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5866 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5867 Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5868 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 2)});
5869 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5870 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 3)});
5871 Tmp4 = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Tmp1.getValue(R: 1),
5872 N2: Tmp2.getValue(R: 1), N3: Tmp3.getValue(R: 1));
5873 Tmp4 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5874 Ops: {Tmp4, Tmp1, Tmp2, Tmp3});
5875 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5876 Ops: {Tmp4.getValue(R: 1), Tmp4,
5877 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5878 Results.push_back(Elt: Tmp4);
5879 Results.push_back(Elt: Tmp4.getValue(R: 1));
5880 break;
5881 case ISD::FCOPYSIGN:
5882 case ISD::FLDEXP:
5883 case ISD::FPOWI: {
5884 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5885 Tmp2 = Node->getOperand(Num: 1);
5886 Tmp3 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, N1: Tmp1, N2: Tmp2);
5887
5888 // fcopysign doesn't change anything but the sign bit, so
5889 // (fp_round (fcopysign (fpext a), b))
5890 // is as precise as
5891 // (fp_round (fpext a))
5892 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5893 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5894 Results.push_back(
5895 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp3,
5896 N2: DAG.getIntPtrConstant(Val: isTrunc, DL: dl, /*isTarget=*/true)));
5897 break;
5898 }
5899 case ISD::STRICT_FLDEXP: {
5900 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5901 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5902 Tmp2 = Node->getOperand(Num: 2);
5903 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FLDEXP, DL: dl, ResultTys: {NVT, MVT::Other},
5904 Ops: {Tmp1.getValue(R: 1), Tmp1, Tmp2});
5905 Tmp4 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5906 Ops: {Tmp3.getValue(R: 1), Tmp3,
5907 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5908 Results.push_back(Elt: Tmp4);
5909 Results.push_back(Elt: Tmp4.getValue(R: 1));
5910 break;
5911 }
5912 case ISD::STRICT_FPOWI:
5913 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
5914 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
5915 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
5916 Ops: {Tmp1.getValue(R: 1), Tmp1, Node->getOperand(Num: 2)});
5917 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
5918 Ops: {Tmp2.getValue(R: 1), Tmp2,
5919 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
5920 Results.push_back(Elt: Tmp3);
5921 Results.push_back(Elt: Tmp3.getValue(R: 1));
5922 break;
5923 case ISD::FFREXP: {
5924 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5925 Tmp2 = DAG.getNode(Opcode: ISD::FFREXP, DL: dl, ResultTys: {NVT, Node->getValueType(ResNo: 1)}, Ops: Tmp1);
5926
5927 Results.push_back(
5928 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5929 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5930
5931 Results.push_back(Elt: Tmp2.getValue(R: 1));
5932 break;
5933 }
5934 case ISD::FMODF:
5935 case ISD::FSINCOS:
5936 case ISD::FSINCOSPI: {
5937 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5938 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VTList: DAG.getVTList(VT1: NVT, VT2: NVT), N: Tmp1);
5939 Tmp3 = DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true);
5940 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5941 Results.push_back(
5942 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2.getValue(R: ResNum), N2: Tmp3));
5943 break;
5944 }
5945 case ISD::FFLOOR:
5946 case ISD::FCEIL:
5947 case ISD::FRINT:
5948 case ISD::FNEARBYINT:
5949 case ISD::FROUND:
5950 case ISD::FROUNDEVEN:
5951 case ISD::FTRUNC:
5952 case ISD::FNEG:
5953 case ISD::FSQRT:
5954 case ISD::FSIN:
5955 case ISD::FCOS:
5956 case ISD::FTAN:
5957 case ISD::FASIN:
5958 case ISD::FACOS:
5959 case ISD::FATAN:
5960 case ISD::FSINH:
5961 case ISD::FCOSH:
5962 case ISD::FTANH:
5963 case ISD::FLOG:
5964 case ISD::FLOG2:
5965 case ISD::FLOG10:
5966 case ISD::FABS:
5967 case ISD::FEXP:
5968 case ISD::FEXP2:
5969 case ISD::FEXP10:
5970 case ISD::FCANONICALIZE:
5971 // Promote scalar operations to vector using SCALAR_TO_VECTOR
5972 if (!OVT.isVector() && NVT.isVector() &&
5973 NVT.getVectorElementType() == OVT) {
5974 Tmp1 = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5975 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1, Flags: Node->getFlags());
5976 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: OVT, N1: Tmp2,
5977 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32)));
5978 break;
5979 }
5980 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
5981 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
5982 Results.push_back(
5983 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
5984 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
5985 break;
5986 case ISD::STRICT_FFLOOR:
5987 case ISD::STRICT_FCEIL:
5988 case ISD::STRICT_FRINT:
5989 case ISD::STRICT_FNEARBYINT:
5990 case ISD::STRICT_FROUND:
5991 case ISD::STRICT_FROUNDEVEN:
5992 case ISD::STRICT_FTRUNC:
5993 case ISD::STRICT_FSQRT:
5994 case ISD::STRICT_FSIN:
5995 case ISD::STRICT_FCOS:
5996 case ISD::STRICT_FTAN:
5997 case ISD::STRICT_FASIN:
5998 case ISD::STRICT_FACOS:
5999 case ISD::STRICT_FATAN:
6000 case ISD::STRICT_FSINH:
6001 case ISD::STRICT_FCOSH:
6002 case ISD::STRICT_FTANH:
6003 case ISD::STRICT_FLOG:
6004 case ISD::STRICT_FLOG2:
6005 case ISD::STRICT_FLOG10:
6006 case ISD::STRICT_FEXP:
6007 case ISD::STRICT_FEXP2:
6008 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6009 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6010 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6011 Ops: {Tmp1.getValue(R: 1), Tmp1});
6012 Tmp3 = DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL: dl, ResultTys: {OVT, MVT::Other},
6013 Ops: {Tmp2.getValue(R: 1), Tmp2,
6014 DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)});
6015 Results.push_back(Elt: Tmp3);
6016 Results.push_back(Elt: Tmp3.getValue(R: 1));
6017 break;
6018 case ISD::LLROUND:
6019 case ISD::LROUND:
6020 case ISD::LRINT:
6021 case ISD::LLRINT:
6022 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NVT, Operand: Node->getOperand(Num: 0));
6023 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Tmp1);
6024 Results.push_back(Elt: Tmp2);
6025 break;
6026 case ISD::STRICT_LLROUND:
6027 case ISD::STRICT_LROUND:
6028 case ISD::STRICT_LRINT:
6029 case ISD::STRICT_LLRINT:
6030 Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl, ResultTys: {NVT, MVT::Other},
6031 Ops: {Node->getOperand(Num: 0), Node->getOperand(Num: 1)});
6032 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, ResultTys: {NVT, MVT::Other},
6033 Ops: {Tmp1.getValue(R: 1), Tmp1});
6034 Results.push_back(Elt: Tmp2);
6035 Results.push_back(Elt: Tmp2.getValue(R: 1));
6036 break;
6037 case ISD::BUILD_VECTOR: {
6038 MVT EltVT = OVT.getVectorElementType();
6039 MVT NewEltVT = NVT.getVectorElementType();
6040
6041 // Handle bitcasts to a different vector type with the same total bit size
6042 //
6043 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
6044 // =>
6045 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
6046
6047 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6048 "Invalid promote type for build_vector");
6049 assert(NewEltVT.bitsLE(EltVT) && "not handled");
6050
6051 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6052
6053 SmallVector<SDValue, 8> NewOps;
6054 for (const SDValue &Op : Node->op_values())
6055 NewOps.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(Op), VT: MidVT, Operand: Op));
6056
6057 SDLoc SL(Node);
6058 SDValue Concat =
6059 DAG.getNode(Opcode: MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
6060 DL: SL, VT: NVT, Ops: NewOps);
6061 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6062 Results.push_back(Elt: CvtVec);
6063 break;
6064 }
6065 case ISD::EXTRACT_VECTOR_ELT: {
6066 MVT EltVT = OVT.getVectorElementType();
6067 MVT NewEltVT = NVT.getVectorElementType();
6068
6069 // Handle bitcasts to a different vector type with the same total bit size.
6070 //
6071 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
6072 // =>
6073 // v4i32:castx = bitcast x:v2i64
6074 //
6075 // i64 = bitcast
6076 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
6077 // (i32 (extract_vector_elt castx, (2 * y + 1)))
6078 //
6079
6080 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6081 "Invalid promote type for extract_vector_elt");
6082 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6083
6084 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6085 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6086
6087 SDValue Idx = Node->getOperand(Num: 1);
6088 EVT IdxVT = Idx.getValueType();
6089 SDLoc SL(Node);
6090 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SL, VT: IdxVT);
6091 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6092
6093 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6094
6095 SmallVector<SDValue, 8> NewOps;
6096 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6097 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6098 SDValue TmpIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6099
6100 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6101 N1: CastVec, N2: TmpIdx);
6102 NewOps.push_back(Elt);
6103 }
6104
6105 SDValue NewVec = DAG.getBuildVector(VT: MidVT, DL: SL, Ops: NewOps);
6106 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: EltVT, Operand: NewVec));
6107 break;
6108 }
6109 case ISD::INSERT_VECTOR_ELT: {
6110 MVT EltVT = OVT.getVectorElementType();
6111 MVT NewEltVT = NVT.getVectorElementType();
6112
6113 // Handle bitcasts to a different vector type with the same total bit size
6114 //
6115 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
6116 // =>
6117 // v4i32:castx = bitcast x:v2i64
6118 // v2i32:casty = bitcast y:i64
6119 //
6120 // v2i64 = bitcast
6121 // (v4i32 insert_vector_elt
6122 // (v4i32 insert_vector_elt v4i32:castx,
6123 // (extract_vector_elt casty, 0), 2 * z),
6124 // (extract_vector_elt casty, 1), (2 * z + 1))
6125
6126 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
6127 "Invalid promote type for insert_vector_elt");
6128 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6129
6130 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6131 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
6132
6133 SDValue Val = Node->getOperand(Num: 1);
6134 SDValue Idx = Node->getOperand(Num: 2);
6135 EVT IdxVT = Idx.getValueType();
6136 SDLoc SL(Node);
6137
6138 SDValue Factor = DAG.getConstant(Val: NewEltsPerOldElt, DL: SDLoc(), VT: IdxVT);
6139 SDValue NewBaseIdx = DAG.getNode(Opcode: ISD::MUL, DL: SL, VT: IdxVT, N1: Idx, N2: Factor);
6140
6141 SDValue CastVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: Node->getOperand(Num: 0));
6142 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6143
6144 SDValue NewVec = CastVec;
6145 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6146 SDValue IdxOffset = DAG.getConstant(Val: I, DL: SL, VT: IdxVT);
6147 SDValue InEltIdx = DAG.getNode(Opcode: ISD::ADD, DL: SL, VT: IdxVT, N1: NewBaseIdx, N2: IdxOffset);
6148
6149 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: SL, VT: NewEltVT,
6150 N1: CastVal, N2: IdxOffset);
6151
6152 NewVec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SL, VT: NVT,
6153 N1: NewVec, N2: Elt, N3: InEltIdx);
6154 }
6155
6156 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewVec));
6157 break;
6158 }
6159 case ISD::SCALAR_TO_VECTOR: {
6160 MVT EltVT = OVT.getVectorElementType();
6161 MVT NewEltVT = NVT.getVectorElementType();
6162
6163 // Handle bitcasts to different vector type with the same total bit size.
6164 //
6165 // e.g. v2i64 = scalar_to_vector x:i64
6166 // =>
6167 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
6168 //
6169
6170 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
6171 SDValue Val = Node->getOperand(Num: 0);
6172 SDLoc SL(Node);
6173
6174 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: MidVT, Operand: Val);
6175 SDValue Undef = DAG.getUNDEF(VT: MidVT);
6176
6177 SmallVector<SDValue, 8> NewElts;
6178 NewElts.push_back(Elt: CastVal);
6179 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
6180 NewElts.push_back(Elt: Undef);
6181
6182 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SL, VT: NVT, Ops: NewElts);
6183 SDValue CvtVec = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Concat);
6184 Results.push_back(Elt: CvtVec);
6185 break;
6186 }
6187 case ISD::ATOMIC_SWAP:
6188 case ISD::ATOMIC_STORE: {
6189 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6190 SDLoc SL(Node);
6191 if (AM->getOpcode() == ISD::ATOMIC_STORE && NVT.bitsGT(VT: OVT)) {
6192 assert(NVT.isInteger() && "unexpected promotion type");
6193 MVT IntVT = MVT::getIntegerVT(BitWidth: OVT.getSizeInBits());
6194 EVT MemVT = AM->getMemoryVT().changeTypeToInteger();
6195 SDValue IntVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: IntVT, Operand: AM->getVal());
6196 SDValue ExtVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SL, VT: NVT, Operand: IntVal);
6197 Results.push_back(Elt: DAG.getAtomic(Opcode: ISD::ATOMIC_STORE, dl: SL, MemVT,
6198 Chain: AM->getChain(), Ptr: ExtVal, Val: AM->getBasePtr(),
6199 MMO: AM->getMemOperand()));
6200 break;
6201 }
6202
6203 SDValue CastVal = DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: NVT, Operand: AM->getVal());
6204 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6205 "unexpected promotion type");
6206 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6207 "unexpected atomic_swap with illegal type");
6208
6209 SDValue Op0 = AM->getBasePtr();
6210 SDValue Op1 = CastVal;
6211
6212 // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
6213 // but really it should merge with ISD::STORE.
6214 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6215 std::swap(a&: Op0, b&: Op1);
6216
6217 SDValue NewAtomic = DAG.getAtomic(Opcode: AM->getOpcode(), dl: SL, MemVT: NVT, Chain: AM->getChain(),
6218 Ptr: Op0, Val: Op1, MMO: AM->getMemOperand());
6219
6220 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6221 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6222 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6223 } else
6224 Results.push_back(Elt: NewAtomic);
6225 break;
6226 }
6227 case ISD::ATOMIC_LOAD: {
6228 AtomicSDNode *AM = cast<AtomicSDNode>(Val: Node);
6229 SDLoc SL(Node);
6230 if (NVT.bitsGT(VT: OVT)) {
6231 assert(NVT.isInteger() && "unexpected promotion type");
6232 MVT IntVT = MVT::getIntegerVT(BitWidth: OVT.getSizeInBits());
6233 EVT MemVT = AM->getMemoryVT().changeTypeToInteger();
6234 ISD::LoadExtType ExtType = AM->getExtensionType();
6235 if (ExtType == ISD::NON_EXTLOAD)
6236 ExtType = ISD::EXTLOAD;
6237 SDValue NewAtomic =
6238 DAG.getAtomicLoad(ExtType, dl: SL, MemVT, VT: NVT, Chain: AM->getChain(),
6239 Ptr: AM->getBasePtr(), MMO: AM->getMemOperand());
6240 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: IntVT, Operand: NewAtomic);
6241 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: Trunc));
6242 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6243 break;
6244 }
6245
6246 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
6247 "unexpected promotion type");
6248 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
6249 "unexpected atomic_load with illegal type");
6250
6251 SDValue NewAtomic =
6252 DAG.getAtomic(Opcode: ISD::ATOMIC_LOAD, dl: SL, MemVT: NVT, VTList: DAG.getVTList(VT1: NVT, VT2: MVT::Other),
6253 Ops: {AM->getChain(), AM->getBasePtr()}, MMO: AM->getMemOperand());
6254 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BITCAST, DL: SL, VT: OVT, Operand: NewAtomic));
6255 Results.push_back(Elt: NewAtomic.getValue(R: 1));
6256 break;
6257 }
6258 case ISD::SPLAT_VECTOR: {
6259 SDValue Scalar = Node->getOperand(Num: 0);
6260 MVT ScalarType = Scalar.getSimpleValueType();
6261 MVT NewScalarType = NVT.getVectorElementType();
6262 if (ScalarType.isInteger()) {
6263 Tmp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6264 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6265 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: OVT, Operand: Tmp2));
6266 break;
6267 }
6268 Tmp1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: NewScalarType, Operand: Scalar);
6269 Tmp2 = DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: NVT, Operand: Tmp1);
6270 Results.push_back(
6271 Elt: DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: OVT, N1: Tmp2,
6272 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)));
6273 break;
6274 }
6275 case ISD::VECREDUCE_FMAX:
6276 case ISD::VECREDUCE_FMIN:
6277 case ISD::VECREDUCE_FMAXIMUM:
6278 case ISD::VECREDUCE_FMINIMUM:
6279 case ISD::VECREDUCE_FMAXIMUMNUM:
6280 case ISD::VECREDUCE_FMINIMUMNUM:
6281 case ISD::VP_REDUCE_FMAX:
6282 case ISD::VP_REDUCE_FMIN:
6283 case ISD::VP_REDUCE_FMAXIMUM:
6284 case ISD::VP_REDUCE_FMINIMUM:
6285 Results.push_back(Elt: PromoteReduction(Node));
6286 break;
6287 }
6288
6289 // Replace the original node with the legalized result.
6290 if (!Results.empty()) {
6291 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6292 ReplaceNode(Old: Node, New: Results.data());
6293 } else
6294 LLVM_DEBUG(dbgs() << "Could not promote node\n");
6295}
6296
6297/// This is the entry point for the file.
6298void SelectionDAG::Legalize() {
6299 AssignTopologicalOrder();
6300
6301 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6302 // Use a delete listener to remove nodes which were deleted during
6303 // legalization from LegalizeNodes. This is needed to handle the situation
6304 // where a new node is allocated by the object pool to the same address of a
6305 // previously deleted node.
6306 DAGNodeDeletedListener DeleteListener(
6307 *this,
6308 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(Ptr: N); });
6309
6310 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6311
6312 // Visit all the nodes. We start in topological order, so that we see
6313 // nodes with their original operands intact. Legalization can produce
6314 // new nodes which may themselves need to be legalized. Iterate until all
6315 // nodes have been legalized.
6316 while (true) {
6317 bool AnyLegalized = false;
6318 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
6319 --NI;
6320
6321 SDNode *N = &*NI;
6322 if (N->use_empty() && N != getRoot().getNode()) {
6323 ++NI;
6324 DeleteNode(N);
6325 continue;
6326 }
6327
6328 if (LegalizedNodes.insert(Ptr: N).second) {
6329 AnyLegalized = true;
6330 Legalizer.LegalizeOp(Node: N);
6331
6332 if (N->use_empty() && N != getRoot().getNode()) {
6333 ++NI;
6334 DeleteNode(N);
6335 }
6336 }
6337 }
6338 if (!AnyLegalized)
6339 break;
6340
6341 }
6342
6343 // Remove dead nodes now.
6344 RemoveDeadNodes();
6345}
6346
6347bool SelectionDAG::LegalizeOp(SDNode *N,
6348 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
6349 SmallPtrSet<SDNode *, 16> LegalizedNodes;
6350 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6351
6352 // Directly insert the node in question, and legalize it. This will recurse
6353 // as needed through operands.
6354 LegalizedNodes.insert(Ptr: N);
6355 Legalizer.LegalizeOp(Node: N);
6356
6357 return LegalizedNodes.count(Ptr: N);
6358}
6359